Add authority bootstrap flows and Concelier ops runbooks
This commit is contained in:
@@ -15,7 +15,8 @@ public class StandardClientProvisioningStoreTests
|
||||
public async Task CreateOrUpdateAsync_HashesSecretAndPersistsDocument()
|
||||
{
|
||||
var store = new TrackingClientStore();
|
||||
var provisioning = new StandardClientProvisioningStore("standard", store);
|
||||
var revocations = new TrackingRevocationStore();
|
||||
var provisioning = new StandardClientProvisioningStore("standard", store, revocations, TimeProvider.System);
|
||||
|
||||
var registration = new AuthorityClientRegistration(
|
||||
clientId: "bootstrap-client",
|
||||
@@ -63,4 +64,21 @@ public class StandardClientProvisioningStoreTests
|
||||
return ValueTask.FromResult(removed);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class TrackingRevocationStore : IAuthorityRevocationStore
|
||||
{
|
||||
public List<AuthorityRevocationDocument> Upserts { get; } = new();
|
||||
|
||||
public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
Upserts.Add(document);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
|
||||
public ValueTask<bool> RemoveAsync(string category, string revocationId, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(true);
|
||||
|
||||
public ValueTask<IReadOnlyList<AuthorityRevocationDocument>> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyList<AuthorityRevocationDocument>>(Array.Empty<AuthorityRevocationDocument>());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Options;
|
||||
using Mongo2Go;
|
||||
@@ -58,6 +59,21 @@ public class StandardPluginRegistrarTests
|
||||
services.AddLogging();
|
||||
services.AddSingleton<IMongoDatabase>(database);
|
||||
services.AddSingleton<IAuthorityClientStore>(new InMemoryClientStore());
|
||||
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
|
||||
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
|
||||
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
|
||||
|
||||
var registrar = new StandardPluginRegistrar();
|
||||
registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration));
|
||||
@@ -83,6 +99,53 @@ public class StandardPluginRegistrarTests
|
||||
Assert.True(verification.User?.RequiresPasswordReset);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Register_LogsWarning_WhenPasswordPolicyWeaker()
|
||||
{
|
||||
using var runner = MongoDbRunner.Start(singleNodeReplSet: true);
|
||||
var client = new MongoClient(runner.ConnectionString);
|
||||
var database = client.GetDatabase("registrar-password-policy");
|
||||
|
||||
var configuration = new ConfigurationBuilder()
|
||||
.AddInMemoryCollection(new Dictionary<string, string?>
|
||||
{
|
||||
["passwordPolicy:minimumLength"] = "6",
|
||||
["passwordPolicy:requireUppercase"] = "false",
|
||||
["passwordPolicy:requireLowercase"] = "false",
|
||||
["passwordPolicy:requireDigit"] = "false",
|
||||
["passwordPolicy:requireSymbol"] = "false"
|
||||
})
|
||||
.Build();
|
||||
|
||||
var manifest = new AuthorityPluginManifest(
|
||||
"standard",
|
||||
"standard",
|
||||
true,
|
||||
typeof(StandardPluginRegistrar).Assembly.GetName().Name,
|
||||
typeof(StandardPluginRegistrar).Assembly.Location,
|
||||
new[] { AuthorityPluginCapabilities.Password },
|
||||
new Dictionary<string, string?>(),
|
||||
"standard.yaml");
|
||||
|
||||
var pluginContext = new AuthorityPluginContext(manifest, configuration);
|
||||
var services = new ServiceCollection();
|
||||
var loggerProvider = new CapturingLoggerProvider();
|
||||
services.AddLogging(builder => builder.AddProvider(loggerProvider));
|
||||
services.AddSingleton<IMongoDatabase>(database);
|
||||
services.AddSingleton<IAuthorityClientStore>(new InMemoryClientStore());
|
||||
|
||||
var registrar = new StandardPluginRegistrar();
|
||||
registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration));
|
||||
|
||||
using var provider = services.BuildServiceProvider();
|
||||
_ = provider.GetRequiredService<StandardUserCredentialStore>();
|
||||
|
||||
Assert.Contains(loggerProvider.Entries, entry =>
|
||||
entry.Level == LogLevel.Warning &&
|
||||
entry.Category.Contains(typeof(StandardPluginRegistrar).FullName!, StringComparison.Ordinal) &&
|
||||
entry.Message.Contains("weaker password policy", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Register_ForcesPasswordCapability_WhenManifestMissing()
|
||||
{
|
||||
@@ -106,6 +169,8 @@ public class StandardPluginRegistrarTests
|
||||
services.AddLogging();
|
||||
services.AddSingleton<IMongoDatabase>(database);
|
||||
services.AddSingleton<IAuthorityClientStore>(new InMemoryClientStore());
|
||||
services.AddSingleton<IAuthorityRevocationStore>(new StubRevocationStore());
|
||||
services.AddSingleton(TimeProvider.System);
|
||||
|
||||
var registrar = new StandardPluginRegistrar();
|
||||
registrar.Register(new AuthorityPluginRegistrationContext(services, pluginContext, configuration));
|
||||
@@ -209,6 +274,61 @@ public class StandardPluginRegistrarTests
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record CapturedLogEntry(string Category, LogLevel Level, string Message);
|
||||
|
||||
internal sealed class CapturingLoggerProvider : ILoggerProvider
|
||||
{
|
||||
public List<CapturedLogEntry> Entries { get; } = new();
|
||||
|
||||
public ILogger CreateLogger(string categoryName) => new CapturingLogger(categoryName, Entries);
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
private sealed class CapturingLogger : ILogger
|
||||
{
|
||||
private readonly string category;
|
||||
private readonly List<CapturedLogEntry> entries;
|
||||
|
||||
public CapturingLogger(string category, List<CapturedLogEntry> entries)
|
||||
{
|
||||
this.category = category;
|
||||
this.entries = entries;
|
||||
}
|
||||
|
||||
public IDisposable BeginScope<TState>(TState state) where TState : notnull => NullScope.Instance;
|
||||
|
||||
public bool IsEnabled(LogLevel logLevel) => true;
|
||||
|
||||
public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
|
||||
{
|
||||
entries.Add(new CapturedLogEntry(category, logLevel, formatter(state, exception)));
|
||||
}
|
||||
|
||||
private sealed class NullScope : IDisposable
|
||||
{
|
||||
public static readonly NullScope Instance = new();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class StubRevocationStore : IAuthorityRevocationStore
|
||||
{
|
||||
public ValueTask UpsertAsync(AuthorityRevocationDocument document, CancellationToken cancellationToken)
|
||||
=> ValueTask.CompletedTask;
|
||||
|
||||
public ValueTask<bool> RemoveAsync(string category, string revocationId, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(false);
|
||||
|
||||
public ValueTask<IReadOnlyList<AuthorityRevocationDocument>> GetActiveAsync(DateTimeOffset asOf, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyList<AuthorityRevocationDocument>>(Array.Empty<AuthorityRevocationDocument>());
|
||||
}
|
||||
|
||||
internal sealed class InMemoryClientStore : IAuthorityClientStore
|
||||
{
|
||||
private readonly Dictionary<string, AuthorityClientDocument> clients = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
@@ -71,6 +71,41 @@ internal sealed class PasswordPolicyOptions
|
||||
throw new InvalidOperationException($"Standard plugin '{pluginName}' requires passwordPolicy.minimumLength to be greater than zero.");
|
||||
}
|
||||
}
|
||||
|
||||
public bool IsWeakerThan(PasswordPolicyOptions other)
|
||||
{
|
||||
if (other is null)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (MinimumLength < other.MinimumLength)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!RequireUppercase && other.RequireUppercase)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!RequireLowercase && other.RequireLowercase)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!RequireDigit && other.RequireDigit)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (!RequireSymbol && other.RequireSymbol)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class LockoutOptions
|
||||
|
||||
@@ -51,6 +51,25 @@ internal sealed class StandardPluginRegistrar : IAuthorityPluginRegistrar
|
||||
var cryptoProvider = sp.GetRequiredService<ICryptoProvider>();
|
||||
var passwordHasher = new CryptoPasswordHasher(pluginOptions, cryptoProvider);
|
||||
var loggerFactory = sp.GetRequiredService<ILoggerFactory>();
|
||||
var registrarLogger = loggerFactory.CreateLogger<StandardPluginRegistrar>();
|
||||
|
||||
var baselinePolicy = new PasswordPolicyOptions();
|
||||
if (pluginOptions.PasswordPolicy.IsWeakerThan(baselinePolicy))
|
||||
{
|
||||
registrarLogger.LogWarning(
|
||||
"Standard plugin '{Plugin}' configured a weaker password policy (minLength={Length}, requireUpper={Upper}, requireLower={Lower}, requireDigit={Digit}, requireSymbol={Symbol}) than the baseline (minLength={BaseLength}, requireUpper={BaseUpper}, requireLower={BaseLower}, requireDigit={BaseDigit}, requireSymbol={BaseSymbol}).",
|
||||
pluginName,
|
||||
pluginOptions.PasswordPolicy.MinimumLength,
|
||||
pluginOptions.PasswordPolicy.RequireUppercase,
|
||||
pluginOptions.PasswordPolicy.RequireLowercase,
|
||||
pluginOptions.PasswordPolicy.RequireDigit,
|
||||
pluginOptions.PasswordPolicy.RequireSymbol,
|
||||
baselinePolicy.MinimumLength,
|
||||
baselinePolicy.RequireUppercase,
|
||||
baselinePolicy.RequireLowercase,
|
||||
baselinePolicy.RequireDigit,
|
||||
baselinePolicy.RequireSymbol);
|
||||
}
|
||||
|
||||
return new StandardUserCredentialStore(
|
||||
pluginName,
|
||||
|
||||
@@ -5,12 +5,14 @@
|
||||
| PLG6.DOC | DONE (2025-10-11) | BE-Auth Plugin, Docs Guild | PLG1–PLG5 | Final polish + diagrams for plugin developer guide (AUTHPLUG-DOCS-01-001). | Docs team delivers copy-edit + exported diagrams; PR merged. |
|
||||
| SEC1.PLG | DONE (2025-10-11) | Security Guild, BE-Auth Plugin | SEC1.A (StellaOps.Cryptography) | Swap Standard plugin hashing to Argon2id via `StellaOps.Cryptography` abstractions; keep PBKDF2 verification for legacy. | ✅ `StandardUserCredentialStore` uses `ICryptoProvider` to hash/check; ✅ Transparent rehash on success; ✅ Unit tests cover tamper + legacy rehash. |
|
||||
| SEC1.OPT | DONE (2025-10-11) | Security Guild | SEC1.PLG | Expose password hashing knobs in `StandardPluginOptions` (`memoryKiB`, `iterations`, `parallelism`, `algorithm`) with validation. | ✅ Options bound from YAML; ✅ Invalid configs throw; ✅ Docs include tuning guidance. |
|
||||
| SEC2.PLG | TODO | Security Guild, Storage Guild | SEC2.A (audit contract) | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`. | ✅ Serilog events enriched with subject/client/IP/outcome; ✅ Mongo records written per attempt; ✅ Tests assert success/lockout/failure cases. |
|
||||
| SEC3.PLG | TODO | Security Guild, BE-Auth Plugin | CORE8, SEC3.A (rate limiter) | Ensure lockout responses and rate-limit metadata flow through plugin logs/events (include retry-after). | ✅ Audit record includes retry-after; ✅ Tests confirm lockout + limiter interplay. |
|
||||
| SEC2.PLG | DOING (2025-10-14) | Security Guild, Storage Guild | SEC2.A (audit contract) | Emit audit events from password verification outcomes and persist via `IAuthorityLoginAttemptStore`. | ✅ Serilog events enriched with subject/client/IP/outcome; ✅ Mongo records written per attempt; ✅ Tests assert success/lockout/failure cases. |
|
||||
| SEC3.PLG | DOING (2025-10-14) | Security Guild, BE-Auth Plugin | CORE8, SEC3.A (rate limiter) | Ensure lockout responses and rate-limit metadata flow through plugin logs/events (include retry-after). | ✅ Audit record includes retry-after; ✅ Tests confirm lockout + limiter interplay. |
|
||||
| SEC4.PLG | DONE (2025-10-12) | Security Guild | SEC4.A (revocation schema) | Provide plugin hooks so revoked users/clients write reasons for revocation bundle export. | ✅ Revocation exporter consumes plugin data; ✅ Tests cover revoked user/client output. |
|
||||
| SEC5.PLG | TODO | Security Guild | SEC5.A (threat model) | Address plugin-specific mitigations (bootstrap user handling, password policy docs) in threat model backlog. | ✅ Threat model lists plugin attack surfaces; ✅ Mitigation items filed. |
|
||||
| SEC5.PLG | DOING (2025-10-14) | Security Guild | SEC5.A (threat model) | Address plugin-specific mitigations (bootstrap user handling, password policy docs) in threat model backlog. | ✅ Threat model lists plugin attack surfaces; ✅ Mitigation items filed. |
|
||||
| PLG4-6.CAPABILITIES | BLOCKED (2025-10-12) | BE-Auth Plugin, Docs Guild | PLG1–PLG3 | Finalise capability metadata exposure, config validation, and developer guide updates; remaining action is Docs polish/diagram export. | ✅ Capability metadata + validation merged; ✅ Plugin guide updated with final copy & diagrams; ✅ Release notes mention new toggles. <br>⛔ Blocked awaiting Authority rate-limiter stream (CORE8/SEC3) to resume so doc updates reflect final limiter behaviour. |
|
||||
| PLG7.RFC | REVIEW | BE-Auth Plugin, Security Guild | PLG4 | Socialize LDAP plugin RFC (`docs/rfcs/authority-plugin-ldap.md`) and capture guild feedback. | ✅ Guild review sign-off recorded; ✅ Follow-up issues filed in module boards. |
|
||||
| PLG6.DIAGRAM | TODO | Docs Guild | PLG6.DOC | Export final sequence/component diagrams for the developer guide and add offline-friendly assets under `docs/assets/authority`. | ✅ Mermaid sources committed; ✅ Rendered SVG/PNG linked from Section 2 + Section 9; ✅ Docs build preview shared with Plugin + Docs guilds. |
|
||||
|
||||
> Update statuses to DOING/DONE/BLOCKED as you make progress. Always run `dotnet test` for touched projects before marking DONE.
|
||||
|
||||
> Remark (2025-10-13, PLG6.DOC/PLG6.DIAGRAM): Security Guild delivered `docs/security/rate-limits.md`; Docs team can lift Section 3 (tuning table + alerts) into the developer guide diagrams when rendering assets.
|
||||
|
||||
@@ -22,5 +22,6 @@ public static class AuthorityMongoDefaults
|
||||
public const string LoginAttempts = "authority_login_attempts";
|
||||
public const string Revocations = "authority_revocations";
|
||||
public const string RevocationState = "authority_revocation_state";
|
||||
public const string Invites = "authority_bootstrap_invites";
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,72 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
|
||||
namespace StellaOps.Authority.Storage.Mongo.Documents;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a bootstrap invitation token for provisioning users or clients.
|
||||
/// </summary>
|
||||
[BsonIgnoreExtraElements]
|
||||
public sealed class AuthorityBootstrapInviteDocument
|
||||
{
|
||||
[BsonId]
|
||||
[BsonRepresentation(BsonType.ObjectId)]
|
||||
public string Id { get; set; } = ObjectId.GenerateNewId().ToString();
|
||||
|
||||
[BsonElement("token")]
|
||||
public string Token { get; set; } = Guid.NewGuid().ToString("N");
|
||||
|
||||
[BsonElement("type")]
|
||||
public string Type { get; set; } = "user";
|
||||
|
||||
[BsonElement("provider")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? Provider { get; set; }
|
||||
|
||||
[BsonElement("target")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? Target { get; set; }
|
||||
|
||||
[BsonElement("issuedAt")]
|
||||
public DateTimeOffset IssuedAt { get; set; } = DateTimeOffset.UtcNow;
|
||||
|
||||
[BsonElement("issuedBy")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? IssuedBy { get; set; }
|
||||
|
||||
[BsonElement("expiresAt")]
|
||||
public DateTimeOffset ExpiresAt { get; set; } = DateTimeOffset.UtcNow.AddDays(2);
|
||||
|
||||
[BsonElement("status")]
|
||||
public string Status { get; set; } = AuthorityBootstrapInviteStatuses.Pending;
|
||||
|
||||
[BsonElement("reservedAt")]
|
||||
[BsonIgnoreIfNull]
|
||||
public DateTimeOffset? ReservedAt { get; set; }
|
||||
|
||||
[BsonElement("reservedBy")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? ReservedBy { get; set; }
|
||||
|
||||
[BsonElement("consumedAt")]
|
||||
[BsonIgnoreIfNull]
|
||||
public DateTimeOffset? ConsumedAt { get; set; }
|
||||
|
||||
[BsonElement("consumedBy")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? ConsumedBy { get; set; }
|
||||
|
||||
[BsonElement("metadata")]
|
||||
[BsonIgnoreIfNull]
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
}
|
||||
|
||||
public static class AuthorityBootstrapInviteStatuses
|
||||
{
|
||||
public const string Pending = "pending";
|
||||
public const string Reserved = "reserved";
|
||||
public const string Consumed = "consumed";
|
||||
public const string Expired = "expired";
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Bson.Serialization.Attributes;
|
||||
@@ -61,6 +62,11 @@ public sealed class AuthorityTokenDocument
|
||||
[BsonIgnoreIfNull]
|
||||
public string? RevokedReasonDescription { get; set; }
|
||||
|
||||
|
||||
[BsonElement("devices")]
|
||||
[BsonIgnoreIfNull]
|
||||
public List<BsonDocument>? Devices { get; set; }
|
||||
|
||||
[BsonElement("revokedMetadata")]
|
||||
[BsonIgnoreIfNull]
|
||||
public Dictionary<string, string?>? RevokedMetadata { get; set; }
|
||||
|
||||
@@ -98,12 +98,19 @@ public static class ServiceCollectionExtensions
|
||||
return database.GetCollection<AuthorityRevocationExportStateDocument>(AuthorityMongoDefaults.Collections.RevocationState);
|
||||
});
|
||||
|
||||
services.AddSingleton(static sp =>
|
||||
{
|
||||
var database = sp.GetRequiredService<IMongoDatabase>();
|
||||
return database.GetCollection<AuthorityBootstrapInviteDocument>(AuthorityMongoDefaults.Collections.Invites);
|
||||
});
|
||||
|
||||
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityUserCollectionInitializer>();
|
||||
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityClientCollectionInitializer>();
|
||||
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityScopeCollectionInitializer>();
|
||||
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityTokenCollectionInitializer>();
|
||||
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityLoginAttemptCollectionInitializer>();
|
||||
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityRevocationCollectionInitializer>();
|
||||
services.TryAddSingleton<IAuthorityCollectionInitializer, AuthorityBootstrapInviteCollectionInitializer>();
|
||||
|
||||
services.TryAddSingleton<IAuthorityUserStore, AuthorityUserStore>();
|
||||
services.TryAddSingleton<IAuthorityClientStore, AuthorityClientStore>();
|
||||
@@ -112,6 +119,7 @@ public static class ServiceCollectionExtensions
|
||||
services.TryAddSingleton<IAuthorityLoginAttemptStore, AuthorityLoginAttemptStore>();
|
||||
services.TryAddSingleton<IAuthorityRevocationStore, AuthorityRevocationStore>();
|
||||
services.TryAddSingleton<IAuthorityRevocationExportStateStore, AuthorityRevocationExportStateStore>();
|
||||
services.TryAddSingleton<IAuthorityBootstrapInviteStore, AuthorityBootstrapInviteStore>();
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,25 @@
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Authority.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Authority.Storage.Mongo.Initialization;
|
||||
|
||||
internal sealed class AuthorityBootstrapInviteCollectionInitializer : IAuthorityCollectionInitializer
|
||||
{
|
||||
private static readonly CreateIndexModel<AuthorityBootstrapInviteDocument>[] Indexes =
|
||||
{
|
||||
new CreateIndexModel<AuthorityBootstrapInviteDocument>(
|
||||
Builders<AuthorityBootstrapInviteDocument>.IndexKeys.Ascending(i => i.Token),
|
||||
new CreateIndexOptions { Unique = true, Name = "idx_invite_token" }),
|
||||
new CreateIndexModel<AuthorityBootstrapInviteDocument>(
|
||||
Builders<AuthorityBootstrapInviteDocument>.IndexKeys.Ascending(i => i.Status).Ascending(i => i.ExpiresAt),
|
||||
new CreateIndexOptions { Name = "idx_invite_status_expires" })
|
||||
};
|
||||
|
||||
public async ValueTask EnsureIndexesAsync(IMongoDatabase database, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(database);
|
||||
|
||||
var collection = database.GetCollection<AuthorityBootstrapInviteDocument>(AuthorityMongoDefaults.Collections.Invites);
|
||||
await collection.Indexes.CreateManyAsync(Indexes, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,166 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Authority.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Authority.Storage.Mongo.Stores;
|
||||
|
||||
internal sealed class AuthorityBootstrapInviteStore : IAuthorityBootstrapInviteStore
|
||||
{
|
||||
private readonly IMongoCollection<AuthorityBootstrapInviteDocument> collection;
|
||||
|
||||
public AuthorityBootstrapInviteStore(IMongoCollection<AuthorityBootstrapInviteDocument> collection)
|
||||
=> this.collection = collection ?? throw new ArgumentNullException(nameof(collection));
|
||||
|
||||
public async ValueTask<AuthorityBootstrapInviteDocument> CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
await collection.InsertOneAsync(document, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
return document;
|
||||
}
|
||||
|
||||
public async ValueTask<BootstrapInviteReservationResult> TryReserveAsync(
|
||||
string token,
|
||||
string expectedType,
|
||||
DateTimeOffset now,
|
||||
string? reservedBy,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, null);
|
||||
}
|
||||
|
||||
var normalizedToken = token.Trim();
|
||||
var filter = Builders<AuthorityBootstrapInviteDocument>.Filter.And(
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Token, normalizedToken),
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Pending));
|
||||
|
||||
var update = Builders<AuthorityBootstrapInviteDocument>.Update
|
||||
.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)
|
||||
.Set(i => i.ReservedAt, now)
|
||||
.Set(i => i.ReservedBy, reservedBy);
|
||||
|
||||
var options = new FindOneAndUpdateOptions<AuthorityBootstrapInviteDocument>
|
||||
{
|
||||
ReturnDocument = ReturnDocument.After
|
||||
};
|
||||
|
||||
var invite = await collection.FindOneAndUpdateAsync(filter, update, options, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (invite is null)
|
||||
{
|
||||
var existing = await collection
|
||||
.Find(i => i.Token == normalizedToken)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (existing is null)
|
||||
{
|
||||
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, null);
|
||||
}
|
||||
|
||||
if (existing.Status is AuthorityBootstrapInviteStatuses.Consumed or AuthorityBootstrapInviteStatuses.Reserved)
|
||||
{
|
||||
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.AlreadyUsed, existing);
|
||||
}
|
||||
|
||||
if (existing.Status == AuthorityBootstrapInviteStatuses.Expired || existing.ExpiresAt <= now)
|
||||
{
|
||||
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.Expired, existing);
|
||||
}
|
||||
|
||||
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, existing);
|
||||
}
|
||||
|
||||
if (!string.Equals(invite.Type, expectedType, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await ReleaseAsync(normalizedToken, cancellationToken).ConfigureAwait(false);
|
||||
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, invite);
|
||||
}
|
||||
|
||||
if (invite.ExpiresAt <= now)
|
||||
{
|
||||
await MarkExpiredAsync(normalizedToken, cancellationToken).ConfigureAwait(false);
|
||||
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.Expired, invite);
|
||||
}
|
||||
|
||||
return new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.Reserved, invite);
|
||||
}
|
||||
|
||||
public async ValueTask<bool> ReleaseAsync(string token, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var result = await collection.UpdateOneAsync(
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.And(
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Token, token.Trim()),
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)),
|
||||
Builders<AuthorityBootstrapInviteDocument>.Update
|
||||
.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Pending)
|
||||
.Set(i => i.ReservedAt, null)
|
||||
.Set(i => i.ReservedBy, null),
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return result.ModifiedCount > 0;
|
||||
}
|
||||
|
||||
public async ValueTask<bool> MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var result = await collection.UpdateOneAsync(
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.And(
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Token, token.Trim()),
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Status, AuthorityBootstrapInviteStatuses.Reserved)),
|
||||
Builders<AuthorityBootstrapInviteDocument>.Update
|
||||
.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Consumed)
|
||||
.Set(i => i.ConsumedAt, consumedAt)
|
||||
.Set(i => i.ConsumedBy, consumedBy),
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return result.ModifiedCount > 0;
|
||||
}
|
||||
|
||||
public async ValueTask<IReadOnlyList<AuthorityBootstrapInviteDocument>> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken)
|
||||
{
|
||||
var filter = Builders<AuthorityBootstrapInviteDocument>.Filter.And(
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.Lte(i => i.ExpiresAt, now),
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.In(
|
||||
i => i.Status,
|
||||
new[] { AuthorityBootstrapInviteStatuses.Pending, AuthorityBootstrapInviteStatuses.Reserved }));
|
||||
|
||||
var update = Builders<AuthorityBootstrapInviteDocument>.Update
|
||||
.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Expired)
|
||||
.Set(i => i.ReservedAt, null)
|
||||
.Set(i => i.ReservedBy, null);
|
||||
|
||||
var expired = await collection.Find(filter)
|
||||
.ToListAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (expired.Count == 0)
|
||||
{
|
||||
return Array.Empty<AuthorityBootstrapInviteDocument>();
|
||||
}
|
||||
|
||||
await collection.UpdateManyAsync(filter, update, cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return expired;
|
||||
}
|
||||
|
||||
private async Task MarkExpiredAsync(string token, CancellationToken cancellationToken)
|
||||
{
|
||||
await collection.UpdateOneAsync(
|
||||
Builders<AuthorityBootstrapInviteDocument>.Filter.Eq(i => i.Token, token),
|
||||
Builders<AuthorityBootstrapInviteDocument>.Update.Set(i => i.Status, AuthorityBootstrapInviteStatuses.Expired),
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using MongoDB.Bson;
|
||||
using MongoDB.Driver;
|
||||
using System.Linq;
|
||||
using System.Globalization;
|
||||
using StellaOps.Authority.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Authority.Storage.Mongo.Stores;
|
||||
@@ -86,6 +90,86 @@ internal sealed class AuthorityTokenStore : IAuthorityTokenStore
|
||||
logger.LogDebug("Updated token {TokenId} status to {Status} (matched {Matched}).", tokenId, status, result.MatchedCount);
|
||||
}
|
||||
|
||||
|
||||
public async ValueTask<TokenUsageUpdateResult> RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tokenId))
|
||||
{
|
||||
return new TokenUsageUpdateResult(TokenUsageUpdateStatus.NotFound, null, null);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(remoteAddress) && string.IsNullOrWhiteSpace(userAgent))
|
||||
{
|
||||
return new TokenUsageUpdateResult(TokenUsageUpdateStatus.MissingMetadata, remoteAddress, userAgent);
|
||||
}
|
||||
|
||||
var id = tokenId.Trim();
|
||||
var token = await collection
|
||||
.Find(t => t.TokenId == id)
|
||||
.FirstOrDefaultAsync(cancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
if (token is null)
|
||||
{
|
||||
return new TokenUsageUpdateResult(TokenUsageUpdateStatus.NotFound, remoteAddress, userAgent);
|
||||
}
|
||||
|
||||
token.Devices ??= new List<BsonDocument>();
|
||||
|
||||
string? normalizedAddress = string.IsNullOrWhiteSpace(remoteAddress) ? null : remoteAddress.Trim();
|
||||
string? normalizedAgent = string.IsNullOrWhiteSpace(userAgent) ? null : userAgent.Trim();
|
||||
|
||||
var device = token.Devices.FirstOrDefault(d =>
|
||||
string.Equals(GetString(d, "remoteAddress"), normalizedAddress, StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(GetString(d, "userAgent"), normalizedAgent, StringComparison.Ordinal));
|
||||
var suspicious = false;
|
||||
|
||||
if (device is null)
|
||||
{
|
||||
suspicious = token.Devices.Count > 0;
|
||||
var document = new BsonDocument
|
||||
{
|
||||
{ "remoteAddress", normalizedAddress },
|
||||
{ "userAgent", normalizedAgent },
|
||||
{ "firstSeen", BsonDateTime.Create(observedAt.UtcDateTime) },
|
||||
{ "lastSeen", BsonDateTime.Create(observedAt.UtcDateTime) },
|
||||
{ "useCount", 1 }
|
||||
};
|
||||
|
||||
token.Devices.Add(document);
|
||||
}
|
||||
else
|
||||
{
|
||||
device["lastSeen"] = BsonDateTime.Create(observedAt.UtcDateTime);
|
||||
device["useCount"] = device.TryGetValue("useCount", out var existingCount) && existingCount.IsInt32
|
||||
? existingCount.AsInt32 + 1
|
||||
: 1;
|
||||
}
|
||||
|
||||
var update = Builders<AuthorityTokenDocument>.Update.Set(t => t.Devices, token.Devices);
|
||||
await collection.UpdateOneAsync(
|
||||
Builders<AuthorityTokenDocument>.Filter.Eq(t => t.TokenId, id),
|
||||
update,
|
||||
cancellationToken: cancellationToken).ConfigureAwait(false);
|
||||
|
||||
return new TokenUsageUpdateResult(suspicious ? TokenUsageUpdateStatus.SuspectedReplay : TokenUsageUpdateStatus.Recorded, normalizedAddress, normalizedAgent);
|
||||
}
|
||||
|
||||
private static string? GetString(BsonDocument document, string name)
|
||||
{
|
||||
if (!document.TryGetValue(name, out var value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return value switch
|
||||
{
|
||||
{ IsString: true } => value.AsString,
|
||||
{ IsBsonNull: true } => null,
|
||||
_ => value.ToString()
|
||||
};
|
||||
}
|
||||
|
||||
public async ValueTask<long> DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken)
|
||||
{
|
||||
var filter = Builders<AuthorityTokenDocument>.Filter.And(
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
using StellaOps.Authority.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Authority.Storage.Mongo.Stores;
|
||||
|
||||
public interface IAuthorityBootstrapInviteStore
|
||||
{
|
||||
ValueTask<AuthorityBootstrapInviteDocument> CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<BootstrapInviteReservationResult> TryReserveAsync(string token, string expectedType, DateTimeOffset now, string? reservedBy, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<bool> ReleaseAsync(string token, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<bool> MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyList<AuthorityBootstrapInviteDocument>> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public enum BootstrapInviteReservationStatus
|
||||
{
|
||||
Reserved,
|
||||
NotFound,
|
||||
Expired,
|
||||
AlreadyUsed
|
||||
}
|
||||
|
||||
public sealed record BootstrapInviteReservationResult(BootstrapInviteReservationStatus Status, AuthorityBootstrapInviteDocument? Invite);
|
||||
@@ -1,3 +1,5 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using StellaOps.Authority.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Authority.Storage.Mongo.Stores;
|
||||
@@ -21,5 +23,17 @@ public interface IAuthorityTokenStore
|
||||
|
||||
ValueTask<long> DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<TokenUsageUpdateResult> RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken);
|
||||
|
||||
ValueTask<IReadOnlyList<AuthorityTokenDocument>> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public enum TokenUsageUpdateStatus
|
||||
{
|
||||
Recorded,
|
||||
SuspectedReplay,
|
||||
MissingMetadata,
|
||||
NotFound
|
||||
}
|
||||
|
||||
public sealed record TokenUsageUpdateResult(TokenUsageUpdateStatus Status, string? RemoteAddress, string? UserAgent);
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using StellaOps.Authority.Bootstrap;
|
||||
using StellaOps.Authority.Storage.Mongo.Documents;
|
||||
using StellaOps.Authority.Storage.Mongo.Stores;
|
||||
using StellaOps.Cryptography.Audit;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Authority.Tests.Bootstrap;
|
||||
|
||||
public sealed class BootstrapInviteCleanupServiceTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task SweepExpiredInvitesAsync_ExpiresInvitesAndEmitsAuditRecords()
|
||||
{
|
||||
var now = new DateTimeOffset(2025, 10, 14, 12, 0, 0, TimeSpan.Zero);
|
||||
var timeProvider = new FakeTimeProvider(now);
|
||||
|
||||
var invites = new List<AuthorityBootstrapInviteDocument>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Token = "token-1",
|
||||
Type = BootstrapInviteTypes.User,
|
||||
ExpiresAt = now.AddMinutes(-5),
|
||||
Provider = "standard",
|
||||
Target = "alice@example.com",
|
||||
Status = AuthorityBootstrapInviteStatuses.Pending
|
||||
},
|
||||
new()
|
||||
{
|
||||
Token = "token-2",
|
||||
Type = BootstrapInviteTypes.Client,
|
||||
ExpiresAt = now.AddMinutes(-1),
|
||||
Provider = "standard",
|
||||
Target = "client-1",
|
||||
Status = AuthorityBootstrapInviteStatuses.Reserved
|
||||
}
|
||||
};
|
||||
|
||||
var store = new FakeInviteStore(invites);
|
||||
var sink = new CapturingAuthEventSink();
|
||||
var service = new BootstrapInviteCleanupService(store, sink, timeProvider, NullLogger<BootstrapInviteCleanupService>.Instance);
|
||||
|
||||
await service.SweepExpiredInvitesAsync(CancellationToken.None);
|
||||
|
||||
Assert.True(store.ExpireCalled);
|
||||
Assert.Equal(2, sink.Events.Count);
|
||||
Assert.All(sink.Events, record => Assert.Equal("authority.bootstrap.invite.expired", record.EventType));
|
||||
Assert.Contains(sink.Events, record => record.Properties.Any(property => property.Name == "invite.token" && property.Value.Value == "token-1"));
|
||||
Assert.Contains(sink.Events, record => record.Properties.Any(property => property.Name == "invite.token" && property.Value.Value == "token-2"));
|
||||
}
|
||||
|
||||
private sealed class FakeInviteStore : IAuthorityBootstrapInviteStore
|
||||
{
|
||||
private readonly IReadOnlyList<AuthorityBootstrapInviteDocument> invites;
|
||||
|
||||
public FakeInviteStore(IReadOnlyList<AuthorityBootstrapInviteDocument> invites)
|
||||
=> this.invites = invites;
|
||||
|
||||
public bool ExpireCalled { get; private set; }
|
||||
|
||||
public ValueTask<AuthorityBootstrapInviteDocument> CreateAsync(AuthorityBootstrapInviteDocument document, CancellationToken cancellationToken)
|
||||
=> throw new NotImplementedException();
|
||||
|
||||
public ValueTask<BootstrapInviteReservationResult> TryReserveAsync(string token, string expectedType, DateTimeOffset now, string? reservedBy, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(new BootstrapInviteReservationResult(BootstrapInviteReservationStatus.NotFound, null));
|
||||
|
||||
public ValueTask<bool> ReleaseAsync(string token, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(false);
|
||||
|
||||
public ValueTask<bool> MarkConsumedAsync(string token, string? consumedBy, DateTimeOffset consumedAt, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(false);
|
||||
|
||||
public ValueTask<IReadOnlyList<AuthorityBootstrapInviteDocument>> ExpireAsync(DateTimeOffset now, CancellationToken cancellationToken)
|
||||
{
|
||||
ExpireCalled = true;
|
||||
return ValueTask.FromResult(invites);
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class CapturingAuthEventSink : IAuthEventSink
|
||||
{
|
||||
public List<AuthEventRecord> Events { get; } = new();
|
||||
|
||||
public ValueTask WriteAsync(AuthEventRecord record, CancellationToken cancellationToken)
|
||||
{
|
||||
Events.Add(record);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ using StellaOps.Authority.Storage.Mongo.Stores;
|
||||
using StellaOps.Authority.RateLimiting;
|
||||
using StellaOps.Cryptography.Audit;
|
||||
using Xunit;
|
||||
using MongoDB.Bson;
|
||||
using static StellaOps.Authority.Tests.OpenIddict.TestHelpers;
|
||||
|
||||
namespace StellaOps.Authority.Tests.OpenIddict;
|
||||
@@ -76,7 +77,7 @@ public class ClientCredentialsHandlersTests
|
||||
|
||||
await handler.HandleAsync(context);
|
||||
|
||||
Assert.False(context.IsRejected);
|
||||
Assert.False(context.IsRejected, $"Rejected: {context.Error} - {context.ErrorDescription}");
|
||||
Assert.Same(clientDocument, context.Transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty]);
|
||||
|
||||
var grantedScopes = Assert.IsType<string[]>(context.Transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty]);
|
||||
@@ -84,6 +85,36 @@ public class ClientCredentialsHandlersTests
|
||||
Assert.Equal(clientDocument.Plugin, context.Transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty]);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ValidateClientCredentials_EmitsTamperAuditEvent_WhenUnexpectedParametersPresent()
|
||||
{
|
||||
var clientDocument = CreateClient(
|
||||
secret: "s3cr3t!",
|
||||
allowedGrantTypes: "client_credentials",
|
||||
allowedScopes: "jobs:read");
|
||||
|
||||
var registry = CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(clientDocument));
|
||||
var sink = new TestAuthEventSink();
|
||||
var handler = new ValidateClientCredentialsHandler(
|
||||
new TestClientStore(clientDocument),
|
||||
registry,
|
||||
TestActivitySource,
|
||||
sink,
|
||||
new TestRateLimiterMetadataAccessor(),
|
||||
TimeProvider.System,
|
||||
NullLogger<ValidateClientCredentialsHandler>.Instance);
|
||||
|
||||
var transaction = CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:read");
|
||||
transaction.Request?.SetParameter("unexpected_param", "value");
|
||||
|
||||
await handler.HandleAsync(new OpenIddictServerEvents.ValidateTokenRequestContext(transaction));
|
||||
|
||||
var tamperEvent = Assert.Single(sink.Events, record => record.EventType == "authority.token.tamper");
|
||||
Assert.Contains(tamperEvent.Properties, property =>
|
||||
string.Equals(property.Name, "request.unexpected_parameter", StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(property.Value.Value, "unexpected_param", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleClientCredentials_PersistsTokenAndEnrichesClaims()
|
||||
{
|
||||
@@ -98,22 +129,30 @@ public class ClientCredentialsHandlersTests
|
||||
var tokenStore = new TestTokenStore();
|
||||
var authSink = new TestAuthEventSink();
|
||||
var metadataAccessor = new TestRateLimiterMetadataAccessor();
|
||||
var validateHandler = new ValidateClientCredentialsHandler(
|
||||
new TestClientStore(clientDocument),
|
||||
registry,
|
||||
TestActivitySource,
|
||||
authSink,
|
||||
metadataAccessor,
|
||||
TimeProvider.System,
|
||||
NullLogger<ValidateClientCredentialsHandler>.Instance);
|
||||
|
||||
var transaction = CreateTokenTransaction(clientDocument.ClientId, secret: null, scope: "jobs:trigger");
|
||||
transaction.Options.AccessTokenLifetime = TimeSpan.FromMinutes(30);
|
||||
|
||||
var validateContext = new OpenIddictServerEvents.ValidateTokenRequestContext(transaction);
|
||||
await validateHandler.HandleAsync(validateContext);
|
||||
Assert.False(validateContext.IsRejected);
|
||||
|
||||
var handler = new HandleClientCredentialsHandler(
|
||||
registry,
|
||||
tokenStore,
|
||||
TimeProvider.System,
|
||||
TestActivitySource,
|
||||
authSink,
|
||||
metadataAccessor,
|
||||
NullLogger<HandleClientCredentialsHandler>.Instance);
|
||||
var persistHandler = new PersistTokensHandler(tokenStore, TimeProvider.System, TestActivitySource, NullLogger<PersistTokensHandler>.Instance);
|
||||
|
||||
var transaction = CreateTokenTransaction(clientDocument.ClientId, secret: null, scope: "jobs:trigger");
|
||||
transaction.Options.AccessTokenLifetime = TimeSpan.FromMinutes(30);
|
||||
transaction.Properties[AuthorityOpenIddictConstants.ClientTransactionProperty] = clientDocument;
|
||||
transaction.Properties[AuthorityOpenIddictConstants.ClientProviderTransactionProperty] = clientDocument.Plugin!;
|
||||
transaction.Properties[AuthorityOpenIddictConstants.ClientGrantedScopesProperty] = new[] { "jobs:trigger" };
|
||||
|
||||
var context = new OpenIddictServerEvents.HandleTokenRequestContext(transaction);
|
||||
|
||||
await handler.HandleAsync(context);
|
||||
@@ -161,10 +200,14 @@ public class TokenValidationHandlersTests
|
||||
ClientId = "feedser"
|
||||
};
|
||||
|
||||
var metadataAccessor = new TestRateLimiterMetadataAccessor();
|
||||
var auditSink = new TestAuthEventSink();
|
||||
var handler = new ValidateAccessTokenHandler(
|
||||
tokenStore,
|
||||
new TestClientStore(CreateClient()),
|
||||
CreateRegistry(withClientProvisioning: true, clientDescriptor: CreateDescriptor(CreateClient())),
|
||||
metadataAccessor,
|
||||
auditSink,
|
||||
TimeProvider.System,
|
||||
TestActivitySource,
|
||||
NullLogger<ValidateAccessTokenHandler>.Instance);
|
||||
@@ -203,10 +246,14 @@ public class TokenValidationHandlersTests
|
||||
|
||||
var registry = new AuthorityIdentityProviderRegistry(new[] { plugin }, NullLogger<AuthorityIdentityProviderRegistry>.Instance);
|
||||
|
||||
var metadataAccessorSuccess = new TestRateLimiterMetadataAccessor();
|
||||
var auditSinkSuccess = new TestAuthEventSink();
|
||||
var handler = new ValidateAccessTokenHandler(
|
||||
new TestTokenStore(),
|
||||
new TestClientStore(clientDocument),
|
||||
registry,
|
||||
metadataAccessorSuccess,
|
||||
auditSinkSuccess,
|
||||
TimeProvider.System,
|
||||
TestActivitySource,
|
||||
NullLogger<ValidateAccessTokenHandler>.Instance);
|
||||
@@ -229,6 +276,76 @@ public class TokenValidationHandlersTests
|
||||
Assert.False(context.IsRejected);
|
||||
Assert.Contains(principal.Claims, claim => claim.Type == "enriched" && claim.Value == "true");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ValidateAccessTokenHandler_EmitsReplayAudit_WhenStoreDetectsSuspectedReplay()
|
||||
{
|
||||
var tokenStore = new TestTokenStore();
|
||||
tokenStore.Inserted = new AuthorityTokenDocument
|
||||
{
|
||||
TokenId = "token-replay",
|
||||
Status = "valid",
|
||||
ClientId = "agent",
|
||||
Devices = new List<BsonDocument>
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "remoteAddress", "10.0.0.1" },
|
||||
{ "userAgent", "agent/1.0" },
|
||||
{ "firstSeen", BsonDateTime.Create(DateTimeOffset.UtcNow.AddMinutes(-15)) },
|
||||
{ "lastSeen", BsonDateTime.Create(DateTimeOffset.UtcNow.AddMinutes(-5)) },
|
||||
{ "useCount", 2 }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
tokenStore.UsageCallback = (remote, agent) => new TokenUsageUpdateResult(TokenUsageUpdateStatus.SuspectedReplay, remote, agent);
|
||||
|
||||
var metadataAccessor = new TestRateLimiterMetadataAccessor();
|
||||
var metadata = metadataAccessor.GetMetadata();
|
||||
if (metadata is not null)
|
||||
{
|
||||
metadata.RemoteIp = "203.0.113.7";
|
||||
metadata.UserAgent = "agent/2.0";
|
||||
}
|
||||
|
||||
var clientDocument = CreateClient();
|
||||
clientDocument.ClientId = "agent";
|
||||
var auditSink = new TestAuthEventSink();
|
||||
var registry = CreateRegistry(withClientProvisioning: false, clientDescriptor: null);
|
||||
var handler = new ValidateAccessTokenHandler(
|
||||
tokenStore,
|
||||
new TestClientStore(clientDocument),
|
||||
registry,
|
||||
metadataAccessor,
|
||||
auditSink,
|
||||
TimeProvider.System,
|
||||
TestActivitySource,
|
||||
NullLogger<ValidateAccessTokenHandler>.Instance);
|
||||
|
||||
var transaction = new OpenIddictServerTransaction
|
||||
{
|
||||
Options = new OpenIddictServerOptions(),
|
||||
EndpointType = OpenIddictServerEndpointType.Introspection,
|
||||
Request = new OpenIddictRequest()
|
||||
};
|
||||
|
||||
var principal = CreatePrincipal("agent", "token-replay", "standard");
|
||||
var context = new OpenIddictServerEvents.ValidateTokenContext(transaction)
|
||||
{
|
||||
Principal = principal,
|
||||
TokenId = "token-replay"
|
||||
};
|
||||
|
||||
await handler.HandleAsync(context);
|
||||
|
||||
Assert.False(context.IsRejected);
|
||||
var replayEvent = Assert.Single(auditSink.Events, record => record.EventType == "authority.token.replay.suspected");
|
||||
Assert.Equal(AuthEventOutcome.Error, replayEvent.Outcome);
|
||||
Assert.NotNull(replayEvent.Network);
|
||||
Assert.Equal("203.0.113.7", replayEvent.Network?.RemoteAddress.Value);
|
||||
Assert.Contains(replayEvent.Properties, property => property.Name == "token.devices.total");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class TestClientStore : IAuthorityClientStore
|
||||
@@ -263,6 +380,8 @@ internal sealed class TestTokenStore : IAuthorityTokenStore
|
||||
{
|
||||
public AuthorityTokenDocument? Inserted { get; set; }
|
||||
|
||||
public Func<string?, string?, TokenUsageUpdateResult>? UsageCallback { get; set; }
|
||||
|
||||
public ValueTask InsertAsync(AuthorityTokenDocument document, CancellationToken cancellationToken)
|
||||
{
|
||||
Inserted = document;
|
||||
@@ -281,6 +400,9 @@ internal sealed class TestTokenStore : IAuthorityTokenStore
|
||||
public ValueTask<long> DeleteExpiredAsync(DateTimeOffset threshold, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(0L);
|
||||
|
||||
public ValueTask<TokenUsageUpdateResult> RecordUsageAsync(string tokenId, string? remoteAddress, string? userAgent, DateTimeOffset observedAt, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult(UsageCallback?.Invoke(remoteAddress, userAgent) ?? new TokenUsageUpdateResult(TokenUsageUpdateStatus.Recorded, remoteAddress, userAgent));
|
||||
|
||||
public ValueTask<IReadOnlyList<AuthorityTokenDocument>> ListRevokedAsync(DateTimeOffset? issuedAfter, CancellationToken cancellationToken)
|
||||
=> ValueTask.FromResult<IReadOnlyList<AuthorityTokenDocument>>(Array.Empty<AuthorityTokenDocument>());
|
||||
}
|
||||
|
||||
@@ -74,6 +74,26 @@ public class PasswordGrantHandlersTests
|
||||
Assert.Contains(sink.Events, record => record.EventType == "authority.password.grant" && record.Outcome == AuthEventOutcome.LockedOut);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task ValidatePasswordGrant_EmitsTamperAuditEvent_WhenUnexpectedParametersPresent()
|
||||
{
|
||||
var sink = new TestAuthEventSink();
|
||||
var metadataAccessor = new TestRateLimiterMetadataAccessor();
|
||||
var registry = CreateRegistry(new SuccessCredentialStore());
|
||||
var validate = new ValidatePasswordGrantHandler(registry, TestActivitySource, sink, metadataAccessor, TimeProvider.System, NullLogger<ValidatePasswordGrantHandler>.Instance);
|
||||
|
||||
var transaction = CreatePasswordTransaction("alice", "Password1!");
|
||||
transaction.Request?.SetParameter("unexpected_param", "value");
|
||||
|
||||
await validate.HandleAsync(new OpenIddictServerEvents.ValidateTokenRequestContext(transaction));
|
||||
|
||||
var tamperEvent = Assert.Single(sink.Events, record => record.EventType == "authority.token.tamper");
|
||||
Assert.Equal(AuthEventOutcome.Failure, tamperEvent.Outcome);
|
||||
Assert.Contains(tamperEvent.Properties, property =>
|
||||
string.Equals(property.Name, "request.unexpected_parameter", StringComparison.OrdinalIgnoreCase) &&
|
||||
string.Equals(property.Value.Value, "unexpected_param", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static AuthorityIdentityProviderRegistry CreateRegistry(IUserCredentialStore store)
|
||||
{
|
||||
var plugin = new StubIdentityProviderPlugin("stub", store);
|
||||
@@ -104,14 +124,14 @@ public class PasswordGrantHandlersTests
|
||||
Name = name;
|
||||
Type = "stub";
|
||||
var manifest = new AuthorityPluginManifest(
|
||||
name,
|
||||
"stub",
|
||||
enabled: true,
|
||||
version: null,
|
||||
description: null,
|
||||
capabilities: new[] { AuthorityPluginCapabilities.Password },
|
||||
configuration: new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase),
|
||||
configPath: $"{name}.yaml");
|
||||
Name: name,
|
||||
Type: "stub",
|
||||
Enabled: true,
|
||||
AssemblyName: null,
|
||||
AssemblyPath: null,
|
||||
Capabilities: new[] { AuthorityPluginCapabilities.Password },
|
||||
Metadata: new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase),
|
||||
ConfigPath: $"{name}.yaml");
|
||||
Context = new AuthorityPluginContext(manifest, new ConfigurationBuilder().Build());
|
||||
Credentials = store;
|
||||
ClaimsEnricher = new NoopClaimsEnricher();
|
||||
|
||||
@@ -5,6 +5,7 @@ using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Time.Testing;
|
||||
using MongoDB.Driver;
|
||||
using MongoDB.Bson;
|
||||
using OpenIddict.Abstractions;
|
||||
using OpenIddict.Server;
|
||||
using StellaOps.Authority;
|
||||
@@ -56,10 +57,10 @@ public sealed class TokenPersistenceIntegrationTests
|
||||
withClientProvisioning: true,
|
||||
clientDescriptor: TestHelpers.CreateDescriptor(clientDocument));
|
||||
|
||||
var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, NullLogger<ValidateClientCredentialsHandler>.Instance);
|
||||
var authSink = new TestAuthEventSink();
|
||||
var metadataAccessor = new TestRateLimiterMetadataAccessor();
|
||||
var handleHandler = new HandleClientCredentialsHandler(registry, TestActivitySource, authSink, metadataAccessor, clock, NullLogger<HandleClientCredentialsHandler>.Instance);
|
||||
var validateHandler = new ValidateClientCredentialsHandler(clientStore, registry, TestActivitySource, authSink, metadataAccessor, clock, NullLogger<ValidateClientCredentialsHandler>.Instance);
|
||||
var handleHandler = new HandleClientCredentialsHandler(registry, tokenStore, clock, TestActivitySource, NullLogger<HandleClientCredentialsHandler>.Instance);
|
||||
var persistHandler = new PersistTokensHandler(tokenStore, clock, TestActivitySource, NullLogger<PersistTokensHandler>.Instance);
|
||||
|
||||
var transaction = TestHelpers.CreateTokenTransaction(clientDocument.ClientId, "s3cr3t!", scope: "jobs:trigger");
|
||||
@@ -148,10 +149,14 @@ public sealed class TokenPersistenceIntegrationTests
|
||||
var revokedAt = now.AddMinutes(1);
|
||||
await tokenStore.UpdateStatusAsync(revokedTokenId, "revoked", revokedAt, "manual", null, null, CancellationToken.None);
|
||||
|
||||
var metadataAccessor = new TestRateLimiterMetadataAccessor();
|
||||
var auditSink = new TestAuthEventSink();
|
||||
var handler = new ValidateAccessTokenHandler(
|
||||
tokenStore,
|
||||
clientStore,
|
||||
registry,
|
||||
metadataAccessor,
|
||||
auditSink,
|
||||
clock,
|
||||
TestActivitySource,
|
||||
NullLogger<ValidateAccessTokenHandler>.Instance);
|
||||
@@ -190,6 +195,60 @@ public sealed class TokenPersistenceIntegrationTests
|
||||
Assert.Equal("manual", stored.RevokedReason);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RecordUsageAsync_FlagsSuspectedReplay_OnNewDeviceFingerprint()
|
||||
{
|
||||
await ResetCollectionsAsync();
|
||||
|
||||
var issuedAt = new DateTimeOffset(2025, 10, 14, 8, 0, 0, TimeSpan.Zero);
|
||||
var clock = new FakeTimeProvider(issuedAt);
|
||||
|
||||
await using var provider = await BuildMongoProviderAsync(clock);
|
||||
|
||||
var tokenStore = provider.GetRequiredService<IAuthorityTokenStore>();
|
||||
|
||||
var tokenDocument = new AuthorityTokenDocument
|
||||
{
|
||||
TokenId = "token-replay",
|
||||
Type = OpenIddictConstants.TokenTypeHints.AccessToken,
|
||||
ClientId = "client-1",
|
||||
Status = "valid",
|
||||
CreatedAt = issuedAt,
|
||||
Devices = new List<BsonDocument>
|
||||
{
|
||||
new BsonDocument
|
||||
{
|
||||
{ "remoteAddress", "10.0.0.1" },
|
||||
{ "userAgent", "agent/1.0" },
|
||||
{ "firstSeen", BsonDateTime.Create(issuedAt.AddMinutes(-10).UtcDateTime) },
|
||||
{ "lastSeen", BsonDateTime.Create(issuedAt.AddMinutes(-5).UtcDateTime) },
|
||||
{ "useCount", 2 }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
await tokenStore.InsertAsync(tokenDocument, CancellationToken.None);
|
||||
|
||||
var result = await tokenStore.RecordUsageAsync(
|
||||
"token-replay",
|
||||
remoteAddress: "10.0.0.2",
|
||||
userAgent: "agent/2.0",
|
||||
observedAt: clock.GetUtcNow(),
|
||||
CancellationToken.None);
|
||||
|
||||
Assert.Equal(TokenUsageUpdateStatus.SuspectedReplay, result.Status);
|
||||
|
||||
var stored = await tokenStore.FindByTokenIdAsync("token-replay", CancellationToken.None);
|
||||
Assert.NotNull(stored);
|
||||
Assert.Equal(2, stored!.Devices?.Count);
|
||||
Assert.Contains(stored.Devices!, doc =>
|
||||
{
|
||||
var remote = doc.TryGetValue("remoteAddress", out var ra) && ra.IsString ? ra.AsString : null;
|
||||
var agentValue = doc.TryGetValue("userAgent", out var ua) && ua.IsString ? ua.AsString : null;
|
||||
return remote == "10.0.0.2" && agentValue == "agent/2.0";
|
||||
});
|
||||
}
|
||||
|
||||
private async Task ResetCollectionsAsync()
|
||||
{
|
||||
var tokens = fixture.Database.GetCollection<AuthorityTokenDocument>(AuthorityMongoDefaults.Collections.Tokens);
|
||||
@@ -220,27 +279,3 @@ public sealed class TokenPersistenceIntegrationTests
|
||||
return provider;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class TestAuthEventSink : IAuthEventSink
|
||||
{
|
||||
public List<AuthEventRecord> Records { get; } = new();
|
||||
|
||||
public ValueTask WriteAsync(AuthEventRecord record, CancellationToken cancellationToken)
|
||||
{
|
||||
Records.Add(record);
|
||||
return ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class TestRateLimiterMetadataAccessor : IAuthorityRateLimiterMetadataAccessor
|
||||
{
|
||||
private readonly AuthorityRateLimiterMetadata metadata = new();
|
||||
|
||||
public AuthorityRateLimiterMetadata? GetMetadata() => metadata;
|
||||
|
||||
public void SetClientId(string? clientId) => metadata.ClientId = string.IsNullOrWhiteSpace(clientId) ? null : clientId;
|
||||
|
||||
public void SetSubjectId(string? subjectId) => metadata.SubjectId = string.IsNullOrWhiteSpace(subjectId) ? null : subjectId;
|
||||
|
||||
public void SetTag(string name, string? value) => metadata.SetTag(name, value);
|
||||
}
|
||||
|
||||
@@ -76,6 +76,7 @@ public class AuthorityRateLimiterMetadataMiddlewareTests
|
||||
context.Request.Path = "/token";
|
||||
context.Request.Method = HttpMethods.Post;
|
||||
context.Request.Headers["X-Forwarded-For"] = "203.0.113.99";
|
||||
context.Request.Headers.UserAgent = "StellaOps-Client/1.2";
|
||||
|
||||
var middleware = CreateMiddleware();
|
||||
await middleware.InvokeAsync(context);
|
||||
@@ -84,6 +85,9 @@ public class AuthorityRateLimiterMetadataMiddlewareTests
|
||||
Assert.NotNull(metadata);
|
||||
Assert.Equal("203.0.113.99", metadata!.RemoteIp);
|
||||
Assert.Equal("203.0.113.99", metadata.ForwardedFor);
|
||||
Assert.Equal("StellaOps-Client/1.2", metadata.UserAgent);
|
||||
Assert.True(metadata.Tags.TryGetValue("authority.user_agent", out var tagValue));
|
||||
Assert.Equal("StellaOps-Client/1.2", tagValue);
|
||||
}
|
||||
|
||||
private static AuthorityRateLimiterMetadataMiddleware CreateMiddleware()
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Authority.Storage.Mongo.Stores;
|
||||
using StellaOps.Authority.Storage.Mongo.Documents;
|
||||
using StellaOps.Cryptography.Audit;
|
||||
|
||||
namespace StellaOps.Authority.Bootstrap;
|
||||
|
||||
internal sealed class BootstrapInviteCleanupService : BackgroundService
|
||||
{
|
||||
private readonly IAuthorityBootstrapInviteStore inviteStore;
|
||||
private readonly IAuthEventSink auditSink;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private readonly ILogger<BootstrapInviteCleanupService> logger;
|
||||
private readonly TimeSpan interval;
|
||||
|
||||
public BootstrapInviteCleanupService(
|
||||
IAuthorityBootstrapInviteStore inviteStore,
|
||||
IAuthEventSink auditSink,
|
||||
TimeProvider timeProvider,
|
||||
ILogger<BootstrapInviteCleanupService> logger)
|
||||
{
|
||||
this.inviteStore = inviteStore ?? throw new ArgumentNullException(nameof(inviteStore));
|
||||
this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink));
|
||||
this.timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
interval = TimeSpan.FromMinutes(5);
|
||||
}
|
||||
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
var timer = new PeriodicTimer(interval);
|
||||
|
||||
try
|
||||
{
|
||||
while (await timer.WaitForNextTickAsync(stoppingToken).ConfigureAwait(false))
|
||||
{
|
||||
await SweepExpiredInvitesAsync(stoppingToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
// Shutdown requested.
|
||||
}
|
||||
finally
|
||||
{
|
||||
timer.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
internal async Task SweepExpiredInvitesAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var expired = await inviteStore.ExpireAsync(now, cancellationToken).ConfigureAwait(false);
|
||||
if (expired.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
logger.LogInformation("Expired {Count} bootstrap invite(s).", expired.Count);
|
||||
|
||||
foreach (var invite in expired)
|
||||
{
|
||||
var record = new AuthEventRecord
|
||||
{
|
||||
EventType = "authority.bootstrap.invite.expired",
|
||||
OccurredAt = now,
|
||||
CorrelationId = Guid.NewGuid().ToString("N"),
|
||||
Outcome = AuthEventOutcome.Success,
|
||||
Reason = "Invite expired before consumption.",
|
||||
Subject = null,
|
||||
Client = null,
|
||||
Scopes = Array.Empty<string>(),
|
||||
Network = null,
|
||||
Properties = BuildInviteProperties(invite)
|
||||
};
|
||||
|
||||
await auditSink.WriteAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static AuthEventProperty[] BuildInviteProperties(AuthorityBootstrapInviteDocument invite)
|
||||
{
|
||||
var properties = new List<AuthEventProperty>
|
||||
{
|
||||
new() { Name = "invite.token", Value = ClassifiedString.Public(invite.Token) },
|
||||
new() { Name = "invite.type", Value = ClassifiedString.Public(invite.Type) },
|
||||
new() { Name = "invite.expires_at", Value = ClassifiedString.Public(invite.ExpiresAt.ToString("O", CultureInfo.InvariantCulture)) }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(invite.Provider))
|
||||
{
|
||||
properties.Add(new AuthEventProperty { Name = "invite.provider", Value = ClassifiedString.Public(invite.Provider) });
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(invite.Target))
|
||||
{
|
||||
properties.Add(new AuthEventProperty { Name = "invite.target", Value = ClassifiedString.Public(invite.Target) });
|
||||
}
|
||||
|
||||
return properties.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,8 @@ internal sealed record BootstrapUserRequest
|
||||
{
|
||||
public string? Provider { get; init; }
|
||||
|
||||
public string? InviteToken { get; init; }
|
||||
|
||||
[Required]
|
||||
public string Username { get; init; } = string.Empty;
|
||||
|
||||
@@ -27,6 +29,8 @@ internal sealed record BootstrapClientRequest
|
||||
{
|
||||
public string? Provider { get; init; }
|
||||
|
||||
public string? InviteToken { get; init; }
|
||||
|
||||
[Required]
|
||||
public string ClientId { get; init; } = string.Empty;
|
||||
|
||||
@@ -46,3 +50,26 @@ internal sealed record BootstrapClientRequest
|
||||
|
||||
public IReadOnlyDictionary<string, string?>? Properties { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record BootstrapInviteRequest
|
||||
{
|
||||
public string Type { get; init; } = BootstrapInviteTypes.User;
|
||||
|
||||
public string? Token { get; init; }
|
||||
|
||||
public string? Provider { get; init; }
|
||||
|
||||
public string? Target { get; init; }
|
||||
|
||||
public DateTimeOffset? ExpiresAt { get; init; }
|
||||
|
||||
public string? IssuedBy { get; init; }
|
||||
|
||||
public IReadOnlyDictionary<string, string?>? Metadata { get; init; }
|
||||
}
|
||||
|
||||
internal static class BootstrapInviteTypes
|
||||
{
|
||||
public const string User = "user";
|
||||
public const string Client = "client";
|
||||
}
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using OpenIddict.Abstractions;
|
||||
using OpenIddict.Server;
|
||||
using StellaOps.Authority.RateLimiting;
|
||||
using StellaOps.Cryptography.Audit;
|
||||
|
||||
namespace StellaOps.Authority.OpenIddict.Handlers;
|
||||
|
||||
internal static class ClientCredentialsAuditHelper
|
||||
{
|
||||
internal static string EnsureCorrelationId(OpenIddictServerTransaction transaction)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(transaction);
|
||||
|
||||
if (transaction.Properties.TryGetValue(AuthorityOpenIddictConstants.AuditCorrelationProperty, out var value) &&
|
||||
value is string existing &&
|
||||
!string.IsNullOrWhiteSpace(existing))
|
||||
{
|
||||
return existing;
|
||||
}
|
||||
|
||||
var correlation = Activity.Current?.TraceId.ToString() ??
|
||||
Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture);
|
||||
|
||||
transaction.Properties[AuthorityOpenIddictConstants.AuditCorrelationProperty] = correlation;
|
||||
return correlation;
|
||||
}
|
||||
|
||||
internal static AuthEventRecord CreateRecord(
|
||||
TimeProvider timeProvider,
|
||||
OpenIddictServerTransaction transaction,
|
||||
AuthorityRateLimiterMetadata? metadata,
|
||||
string? clientSecret,
|
||||
AuthEventOutcome outcome,
|
||||
string? reason,
|
||||
string? clientId,
|
||||
string? providerName,
|
||||
bool? confidential,
|
||||
IReadOnlyList<string> requestedScopes,
|
||||
IReadOnlyList<string> grantedScopes,
|
||||
string? invalidScope,
|
||||
IEnumerable<AuthEventProperty>? extraProperties = null,
|
||||
string? eventType = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
ArgumentNullException.ThrowIfNull(transaction);
|
||||
|
||||
var correlationId = EnsureCorrelationId(transaction);
|
||||
var client = BuildClient(clientId, providerName);
|
||||
var network = BuildNetwork(metadata);
|
||||
var normalizedGranted = NormalizeScopes(grantedScopes);
|
||||
var properties = BuildProperties(confidential, requestedScopes, invalidScope, extraProperties);
|
||||
|
||||
return new AuthEventRecord
|
||||
{
|
||||
EventType = string.IsNullOrWhiteSpace(eventType) ? "authority.client_credentials.grant" : eventType,
|
||||
OccurredAt = timeProvider.GetUtcNow(),
|
||||
CorrelationId = correlationId,
|
||||
Outcome = outcome,
|
||||
Reason = Normalize(reason),
|
||||
Subject = null,
|
||||
Client = client,
|
||||
Scopes = normalizedGranted,
|
||||
Network = network,
|
||||
Properties = properties
|
||||
};
|
||||
}
|
||||
|
||||
internal static AuthEventRecord CreateTamperRecord(
|
||||
TimeProvider timeProvider,
|
||||
OpenIddictServerTransaction transaction,
|
||||
AuthorityRateLimiterMetadata? metadata,
|
||||
string? clientId,
|
||||
string? providerName,
|
||||
bool? confidential,
|
||||
IEnumerable<string> unexpectedParameters)
|
||||
{
|
||||
var properties = new List<AuthEventProperty>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Name = "request.tampered",
|
||||
Value = ClassifiedString.Public("true")
|
||||
}
|
||||
};
|
||||
|
||||
if (confidential.HasValue)
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "client.confidential",
|
||||
Value = ClassifiedString.Public(confidential.Value ? "true" : "false")
|
||||
});
|
||||
}
|
||||
|
||||
if (unexpectedParameters is not null)
|
||||
{
|
||||
foreach (var parameter in unexpectedParameters)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(parameter))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "request.unexpected_parameter",
|
||||
Value = ClassifiedString.Public(parameter)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var reason = unexpectedParameters is null
|
||||
? "Unexpected parameters supplied to client credentials request."
|
||||
: $"Unexpected parameters supplied to client credentials request: {string.Join(", ", unexpectedParameters)}.";
|
||||
|
||||
return CreateRecord(
|
||||
timeProvider,
|
||||
transaction,
|
||||
metadata,
|
||||
clientSecret: null,
|
||||
outcome: AuthEventOutcome.Failure,
|
||||
reason: reason,
|
||||
clientId: clientId,
|
||||
providerName: providerName,
|
||||
confidential: confidential,
|
||||
requestedScopes: Array.Empty<string>(),
|
||||
grantedScopes: Array.Empty<string>(),
|
||||
invalidScope: null,
|
||||
extraProperties: properties,
|
||||
eventType: "authority.token.tamper");
|
||||
}
|
||||
|
||||
private static AuthEventClient? BuildClient(string? clientId, string? providerName)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(clientId) && string.IsNullOrWhiteSpace(providerName))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new AuthEventClient
|
||||
{
|
||||
ClientId = ClassifiedString.Personal(Normalize(clientId)),
|
||||
Name = ClassifiedString.Empty,
|
||||
Provider = ClassifiedString.Public(Normalize(providerName))
|
||||
};
|
||||
}
|
||||
|
||||
private static AuthEventNetwork? BuildNetwork(AuthorityRateLimiterMetadata? metadata)
|
||||
{
|
||||
var remote = Normalize(metadata?.RemoteIp);
|
||||
var forwarded = Normalize(metadata?.ForwardedFor);
|
||||
var userAgent = Normalize(metadata?.UserAgent);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(remote) && string.IsNullOrWhiteSpace(forwarded) && string.IsNullOrWhiteSpace(userAgent))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new AuthEventNetwork
|
||||
{
|
||||
RemoteAddress = ClassifiedString.Personal(remote),
|
||||
ForwardedFor = ClassifiedString.Personal(forwarded),
|
||||
UserAgent = ClassifiedString.Personal(userAgent)
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AuthEventProperty> BuildProperties(
|
||||
bool? confidential,
|
||||
IReadOnlyList<string> requestedScopes,
|
||||
string? invalidScope,
|
||||
IEnumerable<AuthEventProperty>? extraProperties)
|
||||
{
|
||||
var properties = new List<AuthEventProperty>();
|
||||
|
||||
if (confidential.HasValue)
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "client.confidential",
|
||||
Value = ClassifiedString.Public(confidential.Value ? "true" : "false")
|
||||
});
|
||||
}
|
||||
|
||||
var normalizedRequested = NormalizeScopes(requestedScopes);
|
||||
if (normalizedRequested is { Count: > 0 })
|
||||
{
|
||||
foreach (var scope in normalizedRequested)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(scope))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "scope.requested",
|
||||
Value = ClassifiedString.Public(scope)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(invalidScope))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "scope.invalid",
|
||||
Value = ClassifiedString.Public(invalidScope)
|
||||
});
|
||||
}
|
||||
|
||||
if (extraProperties is not null)
|
||||
{
|
||||
foreach (var property in extraProperties)
|
||||
{
|
||||
if (property is null || string.IsNullOrWhiteSpace(property.Name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
properties.Add(property);
|
||||
}
|
||||
}
|
||||
|
||||
return properties.Count == 0 ? Array.Empty<AuthEventProperty>() : properties;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> NormalizeScopes(IReadOnlyList<string>? scopes)
|
||||
{
|
||||
if (scopes is null || scopes.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var normalized = scopes
|
||||
.Where(static scope => !string.IsNullOrWhiteSpace(scope))
|
||||
.Select(static scope => scope.Trim())
|
||||
.Where(static scope => scope.Length > 0)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(static scope => scope, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
|
||||
return normalized.Length == 0 ? Array.Empty<string>() : normalized;
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value)
|
||||
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
}
|
||||
@@ -76,6 +76,22 @@ internal sealed class ValidateClientCredentialsHandler : IOpenIddictServerHandle
|
||||
var requestedScopes = requestedScopeInput.IsDefaultOrEmpty ? Array.Empty<string>() : requestedScopeInput.ToArray();
|
||||
context.Transaction.Properties[AuthorityOpenIddictConstants.AuditRequestedScopesProperty] = requestedScopes;
|
||||
|
||||
var unexpectedParameters = TokenRequestTamperInspector.GetUnexpectedClientCredentialsParameters(context.Request);
|
||||
if (unexpectedParameters.Count > 0)
|
||||
{
|
||||
var providerHint = context.Request.GetParameter(AuthorityOpenIddictConstants.ProviderParameterName)?.Value?.ToString();
|
||||
var tamperRecord = ClientCredentialsAuditHelper.CreateTamperRecord(
|
||||
timeProvider,
|
||||
context.Transaction,
|
||||
metadata,
|
||||
clientId,
|
||||
providerHint,
|
||||
confidential: null,
|
||||
unexpectedParameters);
|
||||
|
||||
await auditSink.WriteAsync(tamperRecord, context.CancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(context.ClientId))
|
||||
|
||||
@@ -68,6 +68,23 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler<Op
|
||||
var requestedScopesInput = context.Request.GetScopes();
|
||||
var requestedScopes = requestedScopesInput.IsDefaultOrEmpty ? Array.Empty<string>() : requestedScopesInput.ToArray();
|
||||
|
||||
var unexpectedParameters = TokenRequestTamperInspector.GetUnexpectedPasswordGrantParameters(context.Request);
|
||||
if (unexpectedParameters.Count > 0)
|
||||
{
|
||||
var providerHint = context.Request.GetParameter(AuthorityOpenIddictConstants.ProviderParameterName)?.Value?.ToString();
|
||||
var tamperRecord = PasswordGrantAuditHelper.CreateTamperRecord(
|
||||
timeProvider,
|
||||
context.Transaction,
|
||||
metadata,
|
||||
clientId,
|
||||
providerHint,
|
||||
context.Request.Username,
|
||||
requestedScopes,
|
||||
unexpectedParameters);
|
||||
|
||||
await auditSink.WriteAsync(tamperRecord, context.CancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var selection = AuthorityIdentityProviderSelector.ResolvePasswordProvider(context.Request, registry);
|
||||
if (!selection.Succeeded)
|
||||
{
|
||||
@@ -75,7 +92,6 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler<Op
|
||||
timeProvider,
|
||||
context.Transaction,
|
||||
metadata,
|
||||
null,
|
||||
AuthEventOutcome.Failure,
|
||||
selection.Description,
|
||||
clientId,
|
||||
@@ -100,7 +116,6 @@ internal sealed class ValidatePasswordGrantHandler : IOpenIddictServerHandler<Op
|
||||
timeProvider,
|
||||
context.Transaction,
|
||||
metadata,
|
||||
httpContext,
|
||||
AuthEventOutcome.Failure,
|
||||
"Both username and password must be provided.",
|
||||
clientId,
|
||||
@@ -250,7 +265,6 @@ internal sealed class HandlePasswordGrantHandler : IOpenIddictServerHandler<Open
|
||||
timeProvider,
|
||||
context.Transaction,
|
||||
metadata,
|
||||
httpContext,
|
||||
AuthEventOutcome.Failure,
|
||||
"Both username and password must be provided.",
|
||||
clientId,
|
||||
@@ -395,7 +409,8 @@ internal static class PasswordGrantAuditHelper
|
||||
IEnumerable<string>? scopes,
|
||||
TimeSpan? retryAfter,
|
||||
AuthorityCredentialFailureCode? failureCode,
|
||||
IEnumerable<AuthEventProperty>? extraProperties)
|
||||
IEnumerable<AuthEventProperty>? extraProperties,
|
||||
string? eventType = null)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
ArgumentNullException.ThrowIfNull(transaction);
|
||||
@@ -409,7 +424,7 @@ internal static class PasswordGrantAuditHelper
|
||||
|
||||
return new AuthEventRecord
|
||||
{
|
||||
EventType = "authority.password.grant",
|
||||
EventType = string.IsNullOrWhiteSpace(eventType) ? "authority.password.grant" : eventType,
|
||||
OccurredAt = timeProvider.GetUtcNow(),
|
||||
CorrelationId = correlationId,
|
||||
Outcome = outcome,
|
||||
@@ -581,4 +596,61 @@ internal static class PasswordGrantAuditHelper
|
||||
|
||||
private static string? Normalize(string? value)
|
||||
=> string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
|
||||
internal static AuthEventRecord CreateTamperRecord(
|
||||
TimeProvider timeProvider,
|
||||
OpenIddictServerTransaction transaction,
|
||||
AuthorityRateLimiterMetadata? metadata,
|
||||
string? clientId,
|
||||
string? providerName,
|
||||
string? username,
|
||||
IEnumerable<string>? scopes,
|
||||
IEnumerable<string> unexpectedParameters)
|
||||
{
|
||||
var properties = new List<AuthEventProperty>
|
||||
{
|
||||
new()
|
||||
{
|
||||
Name = "request.tampered",
|
||||
Value = ClassifiedString.Public("true")
|
||||
}
|
||||
};
|
||||
|
||||
if (unexpectedParameters is not null)
|
||||
{
|
||||
foreach (var parameter in unexpectedParameters)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(parameter))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "request.unexpected_parameter",
|
||||
Value = ClassifiedString.Public(parameter)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
var reason = unexpectedParameters is null
|
||||
? "Unexpected parameters supplied to password grant request."
|
||||
: $"Unexpected parameters supplied to password grant request: {string.Join(", ", unexpectedParameters)}.";
|
||||
|
||||
return CreatePasswordGrantRecord(
|
||||
timeProvider,
|
||||
transaction,
|
||||
metadata,
|
||||
AuthEventOutcome.Failure,
|
||||
reason,
|
||||
clientId,
|
||||
providerName,
|
||||
user: null,
|
||||
username,
|
||||
scopes,
|
||||
retryAfter: null,
|
||||
failureCode: null,
|
||||
extraProperties: properties,
|
||||
eventType: "authority.token.tamper");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -111,14 +111,26 @@ internal sealed class HandleRevocationRequestHandler : IOpenIddictServerHandler<
|
||||
|
||||
private static byte[] Base64UrlDecode(string value)
|
||||
{
|
||||
var padded = value.Length % 4 switch
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
2 => value + "==",
|
||||
3 => value + "=",
|
||||
_ => value
|
||||
};
|
||||
return Array.Empty<byte>();
|
||||
}
|
||||
|
||||
padded = padded.Replace('-', '+').Replace('_', '/');
|
||||
var remainder = value.Length % 4;
|
||||
if (remainder == 2)
|
||||
{
|
||||
value += "==";
|
||||
}
|
||||
else if (remainder == 3)
|
||||
{
|
||||
value += "=";
|
||||
}
|
||||
else if (remainder != 0)
|
||||
{
|
||||
value += new string('=', 4 - remainder);
|
||||
}
|
||||
|
||||
var padded = value.Replace('-', '+').Replace('_', '/');
|
||||
return Convert.FromBase64String(padded);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,7 +119,7 @@ internal sealed class PersistTokensHandler : IOpenIddictServerHandler<OpenIddict
|
||||
|
||||
private static DateTimeOffset? TryGetExpiration(ClaimsPrincipal principal)
|
||||
{
|
||||
var value = principal.GetClaim(OpenIddictConstants.Claims.Exp);
|
||||
var value = principal.GetClaim("exp");
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
|
||||
@@ -1,4 +1,7 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
using System.Security.Claims;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using OpenIddict.Abstractions;
|
||||
@@ -7,8 +10,10 @@ using OpenIddict.Server;
|
||||
using StellaOps.Auth.Abstractions;
|
||||
using StellaOps.Authority.OpenIddict;
|
||||
using StellaOps.Authority.Plugins.Abstractions;
|
||||
using StellaOps.Authority.RateLimiting;
|
||||
using StellaOps.Authority.Storage.Mongo.Documents;
|
||||
using StellaOps.Authority.Storage.Mongo.Stores;
|
||||
using StellaOps.Cryptography.Audit;
|
||||
|
||||
namespace StellaOps.Authority.OpenIddict.Handlers;
|
||||
|
||||
@@ -17,6 +22,8 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
|
||||
private readonly IAuthorityTokenStore tokenStore;
|
||||
private readonly IAuthorityClientStore clientStore;
|
||||
private readonly IAuthorityIdentityProviderRegistry registry;
|
||||
private readonly IAuthorityRateLimiterMetadataAccessor metadataAccessor;
|
||||
private readonly IAuthEventSink auditSink;
|
||||
private readonly TimeProvider clock;
|
||||
private readonly ActivitySource activitySource;
|
||||
private readonly ILogger<ValidateAccessTokenHandler> logger;
|
||||
@@ -25,6 +32,8 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
|
||||
IAuthorityTokenStore tokenStore,
|
||||
IAuthorityClientStore clientStore,
|
||||
IAuthorityIdentityProviderRegistry registry,
|
||||
IAuthorityRateLimiterMetadataAccessor metadataAccessor,
|
||||
IAuthEventSink auditSink,
|
||||
TimeProvider clock,
|
||||
ActivitySource activitySource,
|
||||
ILogger<ValidateAccessTokenHandler> logger)
|
||||
@@ -32,6 +41,8 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
|
||||
this.tokenStore = tokenStore ?? throw new ArgumentNullException(nameof(tokenStore));
|
||||
this.clientStore = clientStore ?? throw new ArgumentNullException(nameof(clientStore));
|
||||
this.registry = registry ?? throw new ArgumentNullException(nameof(registry));
|
||||
this.metadataAccessor = metadataAccessor ?? throw new ArgumentNullException(nameof(metadataAccessor));
|
||||
this.auditSink = auditSink ?? throw new ArgumentNullException(nameof(auditSink));
|
||||
this.clock = clock ?? throw new ArgumentNullException(nameof(clock));
|
||||
this.activitySource = activitySource ?? throw new ArgumentNullException(nameof(activitySource));
|
||||
this.logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
@@ -63,9 +74,10 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
|
||||
? context.TokenId
|
||||
: context.Principal.GetClaim(OpenIddictConstants.Claims.JwtId);
|
||||
|
||||
AuthorityTokenDocument? tokenDocument = null;
|
||||
if (!string.IsNullOrWhiteSpace(tokenId))
|
||||
{
|
||||
var tokenDocument = await tokenStore.FindByTokenIdAsync(tokenId, context.CancellationToken).ConfigureAwait(false);
|
||||
tokenDocument = await tokenStore.FindByTokenIdAsync(tokenId, context.CancellationToken).ConfigureAwait(false);
|
||||
if (tokenDocument is not null)
|
||||
{
|
||||
if (!string.Equals(tokenDocument.Status, "valid", StringComparison.OrdinalIgnoreCase))
|
||||
@@ -87,6 +99,11 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
|
||||
}
|
||||
}
|
||||
|
||||
if (!context.IsRejected && tokenDocument is not null)
|
||||
{
|
||||
await TrackTokenUsageAsync(context, tokenDocument, context.Principal).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var clientId = context.Principal.GetClaim(OpenIddictConstants.Claims.ClientId);
|
||||
if (!string.IsNullOrWhiteSpace(clientId))
|
||||
{
|
||||
@@ -144,4 +161,107 @@ internal sealed class ValidateAccessTokenHandler : IOpenIddictServerHandler<Open
|
||||
identity.GetClaim(OpenIddictConstants.Claims.Subject),
|
||||
identity.GetClaim(OpenIddictConstants.Claims.ClientId));
|
||||
}
|
||||
|
||||
private async ValueTask TrackTokenUsageAsync(
|
||||
OpenIddictServerEvents.ValidateTokenContext context,
|
||||
AuthorityTokenDocument tokenDocument,
|
||||
ClaimsPrincipal principal)
|
||||
{
|
||||
var metadata = metadataAccessor.GetMetadata();
|
||||
var remoteAddress = metadata?.RemoteIp;
|
||||
var userAgent = metadata?.UserAgent;
|
||||
|
||||
var observedAt = clock.GetUtcNow();
|
||||
var result = await tokenStore.RecordUsageAsync(tokenDocument.TokenId, remoteAddress, userAgent, observedAt, context.CancellationToken)
|
||||
.ConfigureAwait(false);
|
||||
|
||||
switch (result.Status)
|
||||
{
|
||||
case TokenUsageUpdateStatus.MissingMetadata:
|
||||
logger.LogDebug("Token usage metadata missing for token {TokenId}; replay detection skipped.", tokenDocument.TokenId);
|
||||
break;
|
||||
case TokenUsageUpdateStatus.NotFound:
|
||||
logger.LogWarning("Token usage recording failed: token {TokenId} not found.", tokenDocument.TokenId);
|
||||
break;
|
||||
case TokenUsageUpdateStatus.Recorded:
|
||||
metadataAccessor.SetTag("authority.token_usage", "recorded");
|
||||
break;
|
||||
case TokenUsageUpdateStatus.SuspectedReplay:
|
||||
metadataAccessor.SetTag("authority.token_usage", "suspected_replay");
|
||||
await EmitReplayAuditAsync(tokenDocument, principal, metadata, result, observedAt, context.CancellationToken).ConfigureAwait(false);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask EmitReplayAuditAsync(
|
||||
AuthorityTokenDocument tokenDocument,
|
||||
ClaimsPrincipal principal,
|
||||
AuthorityRateLimiterMetadata? metadata,
|
||||
TokenUsageUpdateResult result,
|
||||
DateTimeOffset observedAt,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var clientId = principal.GetClaim(OpenIddictConstants.Claims.ClientId);
|
||||
var subjectId = principal.GetClaim(OpenIddictConstants.Claims.Subject);
|
||||
var realm = principal.GetClaim(StellaOpsClaimTypes.IdentityProvider);
|
||||
|
||||
var subject = string.IsNullOrWhiteSpace(subjectId) && string.IsNullOrWhiteSpace(realm)
|
||||
? null
|
||||
: new AuthEventSubject
|
||||
{
|
||||
SubjectId = ClassifiedString.Personal(subjectId),
|
||||
Realm = ClassifiedString.Public(string.IsNullOrWhiteSpace(realm) ? null : realm)
|
||||
};
|
||||
|
||||
var client = string.IsNullOrWhiteSpace(clientId)
|
||||
? null
|
||||
: new AuthEventClient
|
||||
{
|
||||
ClientId = ClassifiedString.Personal(clientId)
|
||||
};
|
||||
|
||||
var network = metadata is null && result.RemoteAddress is null && result.UserAgent is null
|
||||
? null
|
||||
: new AuthEventNetwork
|
||||
{
|
||||
RemoteAddress = ClassifiedString.Personal(result.RemoteAddress ?? metadata?.RemoteIp),
|
||||
ForwardedFor = ClassifiedString.Personal(metadata?.ForwardedFor),
|
||||
UserAgent = ClassifiedString.Personal(result.UserAgent ?? metadata?.UserAgent)
|
||||
};
|
||||
|
||||
var previousCount = tokenDocument.Devices?.Count ?? 0;
|
||||
var properties = new List<AuthEventProperty>
|
||||
{
|
||||
new() { Name = "token.id", Value = ClassifiedString.Sensitive(tokenDocument.TokenId) },
|
||||
new() { Name = "token.type", Value = ClassifiedString.Public(tokenDocument.Type) },
|
||||
new() { Name = "token.devices.total", Value = ClassifiedString.Public((previousCount + 1).ToString(CultureInfo.InvariantCulture)) }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(tokenDocument.ClientId))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "token.client_id",
|
||||
Value = ClassifiedString.Personal(tokenDocument.ClientId)
|
||||
});
|
||||
}
|
||||
|
||||
logger.LogWarning("Detected suspected token replay for token {TokenId} (client {ClientId}).", tokenDocument.TokenId, clientId ?? "<none>");
|
||||
|
||||
var record = new AuthEventRecord
|
||||
{
|
||||
EventType = "authority.token.replay.suspected",
|
||||
OccurredAt = observedAt,
|
||||
CorrelationId = Activity.Current?.TraceId.ToString() ?? Guid.NewGuid().ToString("N"),
|
||||
Outcome = AuthEventOutcome.Error,
|
||||
Reason = "Token observed from a new device fingerprint.",
|
||||
Subject = subject,
|
||||
Client = client,
|
||||
Scopes = Array.Empty<string>(),
|
||||
Network = network,
|
||||
Properties = properties
|
||||
};
|
||||
|
||||
await auditSink.WriteAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,112 @@
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using OpenIddict.Abstractions;
|
||||
|
||||
namespace StellaOps.Authority.OpenIddict;
|
||||
|
||||
internal static class TokenRequestTamperInspector
|
||||
{
|
||||
private static readonly HashSet<string> CommonParameters = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
OpenIddictConstants.Parameters.GrantType,
|
||||
OpenIddictConstants.Parameters.Scope,
|
||||
OpenIddictConstants.Parameters.Resource,
|
||||
OpenIddictConstants.Parameters.ClientId,
|
||||
OpenIddictConstants.Parameters.ClientSecret,
|
||||
OpenIddictConstants.Parameters.ClientAssertion,
|
||||
OpenIddictConstants.Parameters.ClientAssertionType,
|
||||
OpenIddictConstants.Parameters.RefreshToken,
|
||||
OpenIddictConstants.Parameters.DeviceCode,
|
||||
OpenIddictConstants.Parameters.Code,
|
||||
OpenIddictConstants.Parameters.CodeVerifier,
|
||||
OpenIddictConstants.Parameters.CodeChallenge,
|
||||
OpenIddictConstants.Parameters.CodeChallengeMethod,
|
||||
OpenIddictConstants.Parameters.RedirectUri,
|
||||
OpenIddictConstants.Parameters.Assertion,
|
||||
OpenIddictConstants.Parameters.Nonce,
|
||||
OpenIddictConstants.Parameters.Prompt,
|
||||
OpenIddictConstants.Parameters.MaxAge,
|
||||
OpenIddictConstants.Parameters.UiLocales,
|
||||
OpenIddictConstants.Parameters.AcrValues,
|
||||
OpenIddictConstants.Parameters.LoginHint,
|
||||
OpenIddictConstants.Parameters.Claims,
|
||||
OpenIddictConstants.Parameters.Token,
|
||||
OpenIddictConstants.Parameters.TokenTypeHint,
|
||||
OpenIddictConstants.Parameters.AccessToken,
|
||||
OpenIddictConstants.Parameters.IdToken
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> PasswordGrantParameters = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
OpenIddictConstants.Parameters.Username,
|
||||
OpenIddictConstants.Parameters.Password,
|
||||
AuthorityOpenIddictConstants.ProviderParameterName
|
||||
};
|
||||
|
||||
private static readonly HashSet<string> ClientCredentialsParameters = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
AuthorityOpenIddictConstants.ProviderParameterName
|
||||
};
|
||||
|
||||
internal static IReadOnlyList<string> GetUnexpectedPasswordGrantParameters(OpenIddictRequest request)
|
||||
=> DetectUnexpectedParameters(request, PasswordGrantParameters);
|
||||
|
||||
internal static IReadOnlyList<string> GetUnexpectedClientCredentialsParameters(OpenIddictRequest request)
|
||||
=> DetectUnexpectedParameters(request, ClientCredentialsParameters);
|
||||
|
||||
private static IReadOnlyList<string> DetectUnexpectedParameters(
|
||||
OpenIddictRequest request,
|
||||
HashSet<string> grantSpecific)
|
||||
{
|
||||
if (request is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var unexpected = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var pair in request.GetParameters())
|
||||
{
|
||||
var name = pair.Key;
|
||||
if (string.IsNullOrWhiteSpace(name))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsAllowed(name, grantSpecific))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
unexpected.Add(name);
|
||||
}
|
||||
|
||||
return unexpected.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: unexpected
|
||||
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static bool IsAllowed(string parameterName, HashSet<string> grantSpecific)
|
||||
{
|
||||
if (CommonParameters.Contains(parameterName) || grantSpecific.Contains(parameterName))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (parameterName.StartsWith("ext_", StringComparison.OrdinalIgnoreCase) ||
|
||||
parameterName.StartsWith("x-", StringComparison.OrdinalIgnoreCase) ||
|
||||
parameterName.StartsWith("custom_", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
if (parameterName.Contains(':', StringComparison.Ordinal))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ using StellaOps.Authority.Plugins;
|
||||
using StellaOps.Authority.Bootstrap;
|
||||
using StellaOps.Authority.Storage.Mongo.Extensions;
|
||||
using StellaOps.Authority.Storage.Mongo.Initialization;
|
||||
using StellaOps.Authority.Storage.Mongo.Stores;
|
||||
using StellaOps.Authority.RateLimiting;
|
||||
using StellaOps.Configuration;
|
||||
using StellaOps.Plugin.DependencyInjection;
|
||||
@@ -35,6 +36,7 @@ using StellaOps.Cryptography.DependencyInjection;
|
||||
using StellaOps.Authority.Revocation;
|
||||
using StellaOps.Authority.Signing;
|
||||
using StellaOps.Cryptography;
|
||||
using StellaOps.Authority.Storage.Mongo.Documents;
|
||||
|
||||
var builder = WebApplication.CreateBuilder(args);
|
||||
|
||||
@@ -124,6 +126,7 @@ builder.Services.AddSingleton<RevocationBundleBuilder>();
|
||||
builder.Services.AddSingleton<RevocationBundleSigner>();
|
||||
builder.Services.AddSingleton<AuthorityRevocationExportService>();
|
||||
builder.Services.AddSingleton<AuthorityJwksService>();
|
||||
builder.Services.AddHostedService<BootstrapInviteCleanupService>();
|
||||
|
||||
var pluginRegistrationSummary = AuthorityPluginLoader.RegisterPlugins(
|
||||
builder.Services,
|
||||
@@ -281,38 +284,98 @@ if (authorityOptions.Bootstrap.Enabled)
|
||||
HttpContext httpContext,
|
||||
BootstrapUserRequest request,
|
||||
IAuthorityIdentityProviderRegistry registry,
|
||||
IAuthorityBootstrapInviteStore inviteStore,
|
||||
IAuthEventSink auditSink,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (request is null)
|
||||
{
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Request payload is required.", null, null, null, Array.Empty<string>()).ConfigureAwait(false);
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Request payload is required.", null, null, null, Array.Empty<string>(), null).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_request", message = "Request payload is required." });
|
||||
}
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var inviteToken = string.IsNullOrWhiteSpace(request.InviteToken) ? null : request.InviteToken.Trim();
|
||||
AuthorityBootstrapInviteDocument? invite = null;
|
||||
var inviteReserved = false;
|
||||
|
||||
async Task ReleaseInviteAsync(string reason)
|
||||
{
|
||||
if (inviteToken is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (inviteReserved)
|
||||
{
|
||||
await inviteStore.ReleaseAsync(inviteToken, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, reason, invite, inviteToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (inviteToken is not null)
|
||||
{
|
||||
var reservation = await inviteStore.TryReserveAsync(inviteToken, BootstrapInviteTypes.User, now, request.Username, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
switch (reservation.Status)
|
||||
{
|
||||
case BootstrapInviteReservationStatus.Reserved:
|
||||
inviteReserved = true;
|
||||
invite = reservation.Invite;
|
||||
break;
|
||||
case BootstrapInviteReservationStatus.Expired:
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.expired", AuthEventOutcome.Failure, "Invite expired before use.", reservation.Invite, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invite_expired", message = "Invite has expired." });
|
||||
case BootstrapInviteReservationStatus.AlreadyUsed:
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, "Invite token already consumed.", reservation.Invite, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invite_used", message = "Invite token has already been used." });
|
||||
default:
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, "Invite token not found.", reservation.Invite, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_invite", message = "Invite token is invalid." });
|
||||
}
|
||||
}
|
||||
|
||||
var providerName = string.IsNullOrWhiteSpace(request.Provider)
|
||||
? authorityOptions.Bootstrap.DefaultIdentityProvider
|
||||
? invite?.Provider ?? authorityOptions.Bootstrap.DefaultIdentityProvider
|
||||
: request.Provider;
|
||||
|
||||
if (invite is not null && !string.IsNullOrWhiteSpace(invite.Provider) &&
|
||||
!string.Equals(invite.Provider, providerName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await ReleaseInviteAsync("Invite provider does not match requested provider.");
|
||||
return Results.BadRequest(new { error = "invite_provider_mismatch", message = "Invite is limited to a different identity provider." });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var provider))
|
||||
{
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", null, request.Username, providerName, request.Roles ?? Array.Empty<string>()).ConfigureAwait(false);
|
||||
await ReleaseInviteAsync("Specified identity provider was not found.");
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", null, request.Username, providerName, request.Roles ?? Array.Empty<string>(), inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_provider", message = "Specified identity provider was not found." });
|
||||
}
|
||||
|
||||
if (!provider.Capabilities.SupportsPassword)
|
||||
{
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support password provisioning.", null, request.Username, provider.Name, request.Roles ?? Array.Empty<string>()).ConfigureAwait(false);
|
||||
await ReleaseInviteAsync("Selected provider does not support password provisioning.");
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support password provisioning.", null, request.Username, provider.Name, request.Roles ?? Array.Empty<string>(), inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "unsupported_provider", message = "Selected provider does not support password provisioning." });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Username) || string.IsNullOrEmpty(request.Password))
|
||||
{
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Username and password are required.", null, request.Username, provider.Name, request.Roles ?? Array.Empty<string>()).ConfigureAwait(false);
|
||||
await ReleaseInviteAsync("Username and password are required.");
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, "Username and password are required.", null, request.Username, provider.Name, request.Roles ?? Array.Empty<string>(), inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_request", message = "Username and password are required." });
|
||||
}
|
||||
|
||||
if (invite is not null && !string.IsNullOrWhiteSpace(invite.Target) &&
|
||||
!string.Equals(invite.Target, request.Username, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await ReleaseInviteAsync("Invite target does not match requested username.");
|
||||
return Results.BadRequest(new { error = "invite_target_mismatch", message = "Invite target does not match username." });
|
||||
}
|
||||
|
||||
var roles = request.Roles is null ? Array.Empty<string>() : request.Roles.ToArray();
|
||||
var attributes = request.Attributes is null
|
||||
? new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase)
|
||||
@@ -327,24 +390,47 @@ if (authorityOptions.Bootstrap.Enabled)
|
||||
roles,
|
||||
attributes);
|
||||
|
||||
var result = await provider.Credentials.UpsertUserAsync(registration, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.Succeeded || result.Value is null)
|
||||
try
|
||||
{
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, result.Message ?? "User provisioning failed.", null, request.Username, provider.Name, roles).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "User provisioning failed." });
|
||||
var result = await provider.Credentials.UpsertUserAsync(registration, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.Succeeded || result.Value is null)
|
||||
{
|
||||
await ReleaseInviteAsync(result.Message ?? "User provisioning failed.");
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Failure, result.Message ?? "User provisioning failed.", null, request.Username, provider.Name, roles, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "User provisioning failed." });
|
||||
}
|
||||
|
||||
if (inviteReserved && inviteToken is not null)
|
||||
{
|
||||
var consumed = await inviteStore.MarkConsumedAsync(inviteToken, result.Value.SubjectId ?? result.Value.Username, now, cancellationToken).ConfigureAwait(false);
|
||||
if (consumed)
|
||||
{
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.consumed", AuthEventOutcome.Success, null, invite, inviteToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Success, null, result.Value.SubjectId, result.Value.Username, provider.Name, roles, inviteToken).ConfigureAwait(false);
|
||||
|
||||
return Results.Ok(new
|
||||
{
|
||||
provider = provider.Name,
|
||||
subjectId = result.Value.SubjectId,
|
||||
username = result.Value.Username
|
||||
});
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (inviteReserved && inviteToken is not null)
|
||||
{
|
||||
await inviteStore.ReleaseAsync(inviteToken, cancellationToken).ConfigureAwait(false);
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.released", AuthEventOutcome.Error, "Invite released due to provisioning failure.", invite, inviteToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
throw;
|
||||
}
|
||||
|
||||
await WriteBootstrapUserAuditAsync(AuthEventOutcome.Success, null, result.Value.SubjectId, result.Value.Username, provider.Name, roles).ConfigureAwait(false);
|
||||
|
||||
return Results.Ok(new
|
||||
{
|
||||
provider = provider.Name,
|
||||
subjectId = result.Value.SubjectId,
|
||||
username = result.Value.Username
|
||||
});
|
||||
|
||||
async Task WriteBootstrapUserAuditAsync(AuthEventOutcome outcome, string? reason, string? subjectId, string? usernameValue, string? providerValue, IReadOnlyCollection<string> rolesValue)
|
||||
async Task WriteBootstrapUserAuditAsync(AuthEventOutcome outcome, string? reason, string? subjectId, string? usernameValue, string? providerValue, IReadOnlyCollection<string> rolesValue, string? inviteValue)
|
||||
{
|
||||
var correlationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture);
|
||||
AuthEventNetwork? network = null;
|
||||
@@ -369,16 +455,24 @@ if (authorityOptions.Bootstrap.Enabled)
|
||||
Realm = ClassifiedString.Public(providerValue)
|
||||
};
|
||||
|
||||
var properties = string.IsNullOrWhiteSpace(providerValue)
|
||||
? Array.Empty<AuthEventProperty>()
|
||||
: new[]
|
||||
var properties = new List<AuthEventProperty>();
|
||||
if (!string.IsNullOrWhiteSpace(providerValue))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
new AuthEventProperty
|
||||
{
|
||||
Name = "bootstrap.provider",
|
||||
Value = ClassifiedString.Public(providerValue)
|
||||
}
|
||||
};
|
||||
Name = "bootstrap.provider",
|
||||
Value = ClassifiedString.Public(providerValue)
|
||||
});
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(inviteValue))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "bootstrap.invite_token",
|
||||
Value = ClassifiedString.Public(inviteValue)
|
||||
});
|
||||
}
|
||||
|
||||
var scopes = rolesValue is { Count: > 0 }
|
||||
? rolesValue.ToArray()
|
||||
@@ -395,65 +489,199 @@ if (authorityOptions.Bootstrap.Enabled)
|
||||
Client = null,
|
||||
Scopes = scopes,
|
||||
Network = network,
|
||||
Properties = properties
|
||||
Properties = properties.Count == 0 ? Array.Empty<AuthEventProperty>() : properties
|
||||
};
|
||||
|
||||
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
async Task WriteInviteAuditAsync(string eventType, AuthEventOutcome outcome, string? reason, AuthorityBootstrapInviteDocument? document, string? tokenValue)
|
||||
{
|
||||
var record = new AuthEventRecord
|
||||
{
|
||||
EventType = eventType,
|
||||
OccurredAt = timeProvider.GetUtcNow(),
|
||||
CorrelationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture),
|
||||
Outcome = outcome,
|
||||
Reason = reason,
|
||||
Subject = null,
|
||||
Client = null,
|
||||
Scopes = Array.Empty<string>(),
|
||||
Network = null,
|
||||
Properties = BuildInviteProperties(document, tokenValue)
|
||||
};
|
||||
|
||||
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
static AuthEventProperty[] BuildInviteProperties(AuthorityBootstrapInviteDocument? document, string? token)
|
||||
{
|
||||
var properties = new List<AuthEventProperty>();
|
||||
if (!string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.token",
|
||||
Value = ClassifiedString.Public(token)
|
||||
});
|
||||
}
|
||||
|
||||
if (document is not null)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(document.Type))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.type",
|
||||
Value = ClassifiedString.Public(document.Type)
|
||||
});
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(document.Provider))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.provider",
|
||||
Value = ClassifiedString.Public(document.Provider)
|
||||
});
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(document.Target))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.target",
|
||||
Value = ClassifiedString.Public(document.Target)
|
||||
});
|
||||
}
|
||||
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.expires_at",
|
||||
Value = ClassifiedString.Public(document.ExpiresAt.ToString("O", CultureInfo.InvariantCulture))
|
||||
});
|
||||
}
|
||||
|
||||
return properties.Count == 0 ? Array.Empty<AuthEventProperty>() : properties.ToArray();
|
||||
}
|
||||
});
|
||||
|
||||
bootstrapGroup.MapPost("/clients", async (
|
||||
HttpContext httpContext,
|
||||
BootstrapClientRequest request,
|
||||
IAuthorityIdentityProviderRegistry registry,
|
||||
IAuthorityBootstrapInviteStore inviteStore,
|
||||
IAuthEventSink auditSink,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (request is null)
|
||||
{
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Request payload is required.", null, null, null, Array.Empty<string>(), null).ConfigureAwait(false);
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Request payload is required.", null, null, null, Array.Empty<string>(), null, null).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_request", message = "Request payload is required." });
|
||||
}
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var inviteToken = string.IsNullOrWhiteSpace(request.InviteToken) ? null : request.InviteToken.Trim();
|
||||
AuthorityBootstrapInviteDocument? invite = null;
|
||||
var inviteReserved = false;
|
||||
|
||||
async Task ReleaseInviteAsync(string reason)
|
||||
{
|
||||
if (inviteToken is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (inviteReserved)
|
||||
{
|
||||
await inviteStore.ReleaseAsync(inviteToken, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, reason, invite, inviteToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
if (inviteToken is not null)
|
||||
{
|
||||
var reservation = await inviteStore.TryReserveAsync(inviteToken, BootstrapInviteTypes.Client, now, request.ClientId, cancellationToken).ConfigureAwait(false);
|
||||
switch (reservation.Status)
|
||||
{
|
||||
case BootstrapInviteReservationStatus.Reserved:
|
||||
inviteReserved = true;
|
||||
invite = reservation.Invite;
|
||||
break;
|
||||
case BootstrapInviteReservationStatus.Expired:
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.expired", AuthEventOutcome.Failure, "Invite expired before use.", reservation.Invite, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invite_expired", message = "Invite has expired." });
|
||||
case BootstrapInviteReservationStatus.AlreadyUsed:
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, "Invite token already consumed.", reservation.Invite, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invite_used", message = "Invite token has already been used." });
|
||||
default:
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.rejected", AuthEventOutcome.Failure, "Invite token is invalid.", reservation.Invite, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_invite", message = "Invite token is invalid." });
|
||||
}
|
||||
}
|
||||
|
||||
var providerName = string.IsNullOrWhiteSpace(request.Provider)
|
||||
? authorityOptions.Bootstrap.DefaultIdentityProvider
|
||||
? invite?.Provider ?? authorityOptions.Bootstrap.DefaultIdentityProvider
|
||||
: request.Provider;
|
||||
|
||||
if (invite is not null && !string.IsNullOrWhiteSpace(invite.Provider) &&
|
||||
!string.Equals(invite.Provider, providerName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await ReleaseInviteAsync("Invite provider does not match requested provider.");
|
||||
return Results.BadRequest(new { error = "invite_provider_mismatch", message = "Invite is limited to a different identity provider." });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(providerName) || !registry.TryGet(providerName!, out var provider))
|
||||
{
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", request.ClientId, null, providerName, request.AllowedScopes ?? Array.Empty<string>(), request?.Confidential).ConfigureAwait(false);
|
||||
await ReleaseInviteAsync("Specified identity provider was not found.");
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Specified identity provider was not found.", request.ClientId, null, providerName, request.AllowedScopes ?? Array.Empty<string>(), request?.Confidential, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_provider", message = "Specified identity provider was not found." });
|
||||
}
|
||||
|
||||
if (!provider.Capabilities.SupportsClientProvisioning || provider.ClientProvisioning is null)
|
||||
{
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support client provisioning.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
|
||||
await ReleaseInviteAsync("Selected provider does not support client provisioning.");
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Selected provider does not support client provisioning.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "unsupported_provider", message = "Selected provider does not support client provisioning." });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.ClientId))
|
||||
{
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "ClientId is required.", null, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
|
||||
await ReleaseInviteAsync("ClientId is required.");
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "ClientId is required.", null, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_request", message = "ClientId is required." });
|
||||
}
|
||||
|
||||
if (invite is not null && !string.IsNullOrWhiteSpace(invite.Target) &&
|
||||
!string.Equals(invite.Target, request.ClientId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
await ReleaseInviteAsync("Invite target does not match requested client id.");
|
||||
return Results.BadRequest(new { error = "invite_target_mismatch", message = "Invite target does not match client id." });
|
||||
}
|
||||
|
||||
if (request.Confidential && string.IsNullOrWhiteSpace(request.ClientSecret))
|
||||
{
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Confidential clients require a client secret.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
|
||||
await ReleaseInviteAsync("Confidential clients require a client secret.");
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, "Confidential clients require a client secret.", request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_request", message = "Confidential clients require a client secret." });
|
||||
}
|
||||
|
||||
if (!TryParseUris(request.RedirectUris, out var redirectUris, out var redirectError))
|
||||
{
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, redirectError, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_request", message = redirectError });
|
||||
var errorMessage = redirectError ?? "Redirect URI validation failed.";
|
||||
await ReleaseInviteAsync(errorMessage);
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, errorMessage, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_request", message = errorMessage });
|
||||
}
|
||||
|
||||
if (!TryParseUris(request.PostLogoutRedirectUris, out var postLogoutUris, out var postLogoutError))
|
||||
{
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, postLogoutError, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_request", message = postLogoutError });
|
||||
var errorMessage = postLogoutError ?? "Post-logout redirect URI validation failed.";
|
||||
await ReleaseInviteAsync(errorMessage);
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, errorMessage, request.ClientId, null, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = "invalid_request", message = errorMessage });
|
||||
}
|
||||
|
||||
var properties = request.Properties is null
|
||||
@@ -475,11 +703,21 @@ if (authorityOptions.Bootstrap.Enabled)
|
||||
|
||||
if (!result.Succeeded || result.Value is null)
|
||||
{
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, result.Message ?? "Client provisioning failed.", request.ClientId, result.Value?.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
|
||||
await ReleaseInviteAsync(result.Message ?? "Client provisioning failed.");
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Failure, result.Message ?? "Client provisioning failed.", request.ClientId, result.Value?.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
|
||||
return Results.BadRequest(new { error = result.ErrorCode ?? "bootstrap_failed", message = result.Message ?? "Client provisioning failed." });
|
||||
}
|
||||
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Success, null, request.ClientId, result.Value.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential).ConfigureAwait(false);
|
||||
if (inviteReserved && inviteToken is not null)
|
||||
{
|
||||
var consumed = await inviteStore.MarkConsumedAsync(inviteToken, result.Value.ClientId, now, cancellationToken).ConfigureAwait(false);
|
||||
if (consumed)
|
||||
{
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.consumed", AuthEventOutcome.Success, null, invite, inviteToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
await WriteBootstrapClientAuditAsync(AuthEventOutcome.Success, null, request.ClientId, result.Value.ClientId, provider.Name, request.AllowedScopes ?? Array.Empty<string>(), request.Confidential, inviteToken).ConfigureAwait(false);
|
||||
|
||||
return Results.Ok(new
|
||||
{
|
||||
@@ -488,7 +726,7 @@ if (authorityOptions.Bootstrap.Enabled)
|
||||
confidential = result.Value.Confidential
|
||||
});
|
||||
|
||||
async Task WriteBootstrapClientAuditAsync(AuthEventOutcome outcome, string? reason, string? requestedClientId, string? assignedClientId, string? providerValue, IReadOnlyCollection<string> scopes, bool? confidentialFlag)
|
||||
async Task WriteBootstrapClientAuditAsync(AuthEventOutcome outcome, string? reason, string? requestedClientId, string? assignedClientId, string? providerValue, IReadOnlyCollection<string> scopes, bool? confidentialFlag, string? inviteValue)
|
||||
{
|
||||
var correlationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture);
|
||||
AuthEventNetwork? network = null;
|
||||
@@ -533,6 +771,15 @@ if (authorityOptions.Bootstrap.Enabled)
|
||||
});
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(inviteValue))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "bootstrap.invite_token",
|
||||
Value = ClassifiedString.Public(inviteValue)
|
||||
});
|
||||
}
|
||||
|
||||
var record = new AuthEventRecord
|
||||
{
|
||||
EventType = "authority.bootstrap.client",
|
||||
@@ -549,6 +796,175 @@ if (authorityOptions.Bootstrap.Enabled)
|
||||
|
||||
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
async Task WriteInviteAuditAsync(string eventType, AuthEventOutcome outcome, string? reason, AuthorityBootstrapInviteDocument? document, string? tokenValue)
|
||||
{
|
||||
var record = new AuthEventRecord
|
||||
{
|
||||
EventType = eventType,
|
||||
OccurredAt = timeProvider.GetUtcNow(),
|
||||
CorrelationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture),
|
||||
Outcome = outcome,
|
||||
Reason = reason,
|
||||
Subject = null,
|
||||
Client = null,
|
||||
Scopes = Array.Empty<string>(),
|
||||
Network = null,
|
||||
Properties = BuildInviteProperties(document, tokenValue)
|
||||
};
|
||||
|
||||
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
static AuthEventProperty[] BuildInviteProperties(AuthorityBootstrapInviteDocument? document, string? token)
|
||||
{
|
||||
var properties = new List<AuthEventProperty>();
|
||||
if (!string.IsNullOrWhiteSpace(token))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.token",
|
||||
Value = ClassifiedString.Public(token)
|
||||
});
|
||||
}
|
||||
|
||||
if (document is not null)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(document.Type))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.type",
|
||||
Value = ClassifiedString.Public(document.Type)
|
||||
});
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(document.Provider))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.provider",
|
||||
Value = ClassifiedString.Public(document.Provider)
|
||||
});
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(document.Target))
|
||||
{
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.target",
|
||||
Value = ClassifiedString.Public(document.Target)
|
||||
});
|
||||
}
|
||||
|
||||
properties.Add(new AuthEventProperty
|
||||
{
|
||||
Name = "invite.expires_at",
|
||||
Value = ClassifiedString.Public(document.ExpiresAt.ToString("O", CultureInfo.InvariantCulture))
|
||||
});
|
||||
}
|
||||
|
||||
return properties.Count == 0 ? Array.Empty<AuthEventProperty>() : properties.ToArray();
|
||||
}
|
||||
});
|
||||
bootstrapGroup.MapPost("/invites", async (
|
||||
HttpContext httpContext,
|
||||
BootstrapInviteRequest request,
|
||||
IAuthorityBootstrapInviteStore inviteStore,
|
||||
IAuthEventSink auditSink,
|
||||
TimeProvider timeProvider,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (request is null)
|
||||
{
|
||||
return Results.BadRequest(new { error = "invalid_request", message = "Request payload is required." });
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(request.Type) ||
|
||||
( !string.Equals(request.Type, BootstrapInviteTypes.User, StringComparison.OrdinalIgnoreCase) &&
|
||||
!string.Equals(request.Type, BootstrapInviteTypes.Client, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
return Results.BadRequest(new { error = "invalid_request", message = "Invite type must be 'user' or 'client'." });
|
||||
}
|
||||
|
||||
var now = timeProvider.GetUtcNow();
|
||||
var expiresAt = request.ExpiresAt ?? now.AddDays(2);
|
||||
if (expiresAt <= now)
|
||||
{
|
||||
return Results.BadRequest(new { error = "invalid_request", message = "ExpiresAt must be in the future." });
|
||||
}
|
||||
|
||||
var token = string.IsNullOrWhiteSpace(request.Token) ? Guid.NewGuid().ToString("N") : request.Token.Trim();
|
||||
|
||||
var document = new AuthorityBootstrapInviteDocument
|
||||
{
|
||||
Token = token,
|
||||
Type = request.Type.ToLowerInvariant(),
|
||||
Provider = string.IsNullOrWhiteSpace(request.Provider) ? null : request.Provider.Trim(),
|
||||
Target = string.IsNullOrWhiteSpace(request.Target) ? null : request.Target.Trim(),
|
||||
IssuedAt = now,
|
||||
IssuedBy = string.IsNullOrWhiteSpace(request.IssuedBy) ? httpContext.User?.Identity?.Name : request.IssuedBy,
|
||||
ExpiresAt = expiresAt,
|
||||
Metadata = request.Metadata is null ? null : new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase)
|
||||
};
|
||||
|
||||
await inviteStore.CreateAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
await WriteInviteAuditAsync("authority.bootstrap.invite.created", AuthEventOutcome.Success, null, document).ConfigureAwait(false);
|
||||
|
||||
return Results.Ok(new
|
||||
{
|
||||
document.Token,
|
||||
document.Type,
|
||||
document.Provider,
|
||||
document.Target,
|
||||
document.ExpiresAt
|
||||
});
|
||||
|
||||
async Task WriteInviteAuditAsync(string eventType, AuthEventOutcome outcome, string? reason, AuthorityBootstrapInviteDocument invite)
|
||||
{
|
||||
var record = new AuthEventRecord
|
||||
{
|
||||
EventType = eventType,
|
||||
OccurredAt = timeProvider.GetUtcNow(),
|
||||
CorrelationId = Activity.Current?.TraceId.ToString() ?? httpContext.TraceIdentifier ?? Guid.NewGuid().ToString("N", CultureInfo.InvariantCulture),
|
||||
Outcome = outcome,
|
||||
Reason = reason,
|
||||
Subject = null,
|
||||
Client = null,
|
||||
Scopes = Array.Empty<string>(),
|
||||
Network = null,
|
||||
Properties = BuildInviteProperties(invite)
|
||||
};
|
||||
|
||||
await auditSink.WriteAsync(record, httpContext.RequestAborted).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
static AuthEventProperty[] BuildInviteProperties(AuthorityBootstrapInviteDocument invite)
|
||||
{
|
||||
var properties = new List<AuthEventProperty>
|
||||
{
|
||||
new() { Name = "invite.token", Value = ClassifiedString.Public(invite.Token) },
|
||||
new() { Name = "invite.type", Value = ClassifiedString.Public(invite.Type) },
|
||||
new() { Name = "invite.expires_at", Value = ClassifiedString.Public(invite.ExpiresAt.ToString("O", CultureInfo.InvariantCulture)) }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(invite.Provider))
|
||||
{
|
||||
properties.Add(new AuthEventProperty { Name = "invite.provider", Value = ClassifiedString.Public(invite.Provider) });
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(invite.Target))
|
||||
{
|
||||
properties.Add(new AuthEventProperty { Name = "invite.target", Value = ClassifiedString.Public(invite.Target) });
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(invite.IssuedBy))
|
||||
{
|
||||
properties.Add(new AuthEventProperty { Name = "invite.issued_by", Value = ClassifiedString.Public(invite.IssuedBy) });
|
||||
}
|
||||
|
||||
return properties.ToArray();
|
||||
}
|
||||
});
|
||||
|
||||
bootstrapGroup.MapGet("/revocations/export", async (
|
||||
@@ -573,6 +989,7 @@ if (authorityOptions.Bootstrap.Enabled)
|
||||
{
|
||||
Algorithm = package.Signature.Algorithm,
|
||||
KeyId = package.Signature.KeyId,
|
||||
Provider = package.Signature.Provider,
|
||||
Value = package.Signature.Value
|
||||
},
|
||||
Digest = new RevocationExportDigest
|
||||
|
||||
@@ -41,6 +41,11 @@ internal sealed class AuthorityRateLimiterMetadata
|
||||
/// </summary>
|
||||
public IReadOnlyDictionary<string, string> Tags => tags;
|
||||
|
||||
/// <summary>
|
||||
/// User agent string associated with the request, if captured.
|
||||
/// </summary>
|
||||
public string? UserAgent { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Adds or updates an arbitrary metadata tag for downstream consumers.
|
||||
/// </summary>
|
||||
|
||||
@@ -61,6 +61,9 @@ internal sealed class AuthorityRateLimiterMetadataMiddleware
|
||||
metadata.ClientId = ResolveAuthorizeClientId(context.Request.Query);
|
||||
}
|
||||
|
||||
var userAgent = NormalizeUserAgent(context.Request.Headers.UserAgent.ToString());
|
||||
metadata.UserAgent = userAgent;
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(metadata.ClientId))
|
||||
{
|
||||
metadata.SetTag("authority.client_id", metadata.ClientId);
|
||||
@@ -74,6 +77,10 @@ internal sealed class AuthorityRateLimiterMetadataMiddleware
|
||||
metadata.SetTag("authority.endpoint", metadata.Endpoint ?? string.Empty);
|
||||
metadata.SetTag("authority.remote_ip", metadata.RemoteIp ?? "unknown");
|
||||
metadata.SetTag("authority.captured_at", clock.GetUtcNow().ToString("O", CultureInfo.InvariantCulture));
|
||||
if (!string.IsNullOrWhiteSpace(userAgent))
|
||||
{
|
||||
metadata.SetTag("authority.user_agent", userAgent);
|
||||
}
|
||||
|
||||
await next(context).ConfigureAwait(false);
|
||||
}
|
||||
@@ -145,6 +152,17 @@ internal sealed class AuthorityRateLimiterMetadataMiddleware
|
||||
return null;
|
||||
}
|
||||
|
||||
private static string? NormalizeUserAgent(string? userAgent)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(userAgent))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = userAgent.Trim();
|
||||
return trimmed.Length == 0 ? null : trimmed;
|
||||
}
|
||||
|
||||
private async Task<string?> ResolveTokenClientIdAsync(HttpContext context)
|
||||
{
|
||||
var request = context.Request;
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
namespace StellaOps.Authority.Revocation;
|
||||
|
||||
internal sealed record RevocationBundleSignature(string Algorithm, string KeyId, string Value);
|
||||
internal sealed record RevocationBundleSignature(string Algorithm, string KeyId, string Provider, string Value);
|
||||
|
||||
@@ -51,12 +51,18 @@ internal sealed class RevocationBundleSigner
|
||||
: signing.Algorithm.Trim();
|
||||
|
||||
var keyReference = new CryptoKeyReference(signing.ActiveKeyId, signing.Provider);
|
||||
var signer = providerRegistry.ResolveSigner(CryptoCapability.Signing, algorithm, keyReference, signing.Provider);
|
||||
var resolved = providerRegistry.ResolveSigner(
|
||||
CryptoCapability.Signing,
|
||||
algorithm,
|
||||
keyReference,
|
||||
signing.Provider);
|
||||
var signer = resolved.Signer;
|
||||
|
||||
var header = new Dictionary<string, object>
|
||||
{
|
||||
["alg"] = algorithm,
|
||||
["kid"] = signing.ActiveKeyId,
|
||||
["provider"] = resolved.ProviderName,
|
||||
["typ"] = "application/vnd.stellaops.revocation-bundle+jws",
|
||||
["b64"] = false,
|
||||
["crit"] = new[] { "b64" }
|
||||
@@ -77,7 +83,11 @@ internal sealed class RevocationBundleSigner
|
||||
var signingInput = new ReadOnlyMemory<byte>(buffer, 0, signingInputLength);
|
||||
var signatureBytes = await signer.SignAsync(signingInput, cancellationToken).ConfigureAwait(false);
|
||||
var encodedSignature = Base64UrlEncode(signatureBytes);
|
||||
return new RevocationBundleSignature(algorithm, signing.ActiveKeyId, string.Concat(protectedHeader, "..", encodedSignature));
|
||||
return new RevocationBundleSignature(
|
||||
algorithm,
|
||||
signing.ActiveKeyId,
|
||||
resolved.ProviderName,
|
||||
string.Concat(protectedHeader, "..", encodedSignature));
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -56,6 +56,9 @@ internal sealed class RevocationExportSignature
|
||||
[JsonPropertyName("keyId")]
|
||||
public required string KeyId { get; init; }
|
||||
|
||||
[JsonPropertyName("provider")]
|
||||
public required string Provider { get; init; }
|
||||
|
||||
[JsonPropertyName("value")]
|
||||
public required string Value { get; init; }
|
||||
}
|
||||
|
||||
@@ -236,10 +236,11 @@ internal sealed class AuthoritySigningKeyManager
|
||||
["status"] = AuthoritySigningKeyStatus.Retired
|
||||
};
|
||||
|
||||
var privateParameters = previous.Key.PrivateParameters;
|
||||
var retiredKey = new CryptoSigningKey(
|
||||
previous.Key.Reference,
|
||||
previous.Key.AlgorithmId,
|
||||
in previous.Key.PrivateParameters,
|
||||
in privateParameters,
|
||||
previous.Key.CreatedAt,
|
||||
previous.Key.ExpiresAt,
|
||||
metadata);
|
||||
|
||||
@@ -6,11 +6,18 @@
|
||||
| CORE9.REVOCATION | DONE (2025-10-12) | Authority Core, Security Guild | CORE5 | Implement revocation list persistence + export hooks (API + CLI). | ✅ Revoked tokens denied; ✅ Export endpoint/CLI returns manifest; ✅ Tests cover offline bundle flow. |
|
||||
| CORE10.JWKS | DONE (2025-10-12) | Authority Core, DevOps | CORE9.REVOCATION | Provide JWKS rotation with pluggable key loader + documentation. | ✅ Signing/encryption keys rotate without downtime; ✅ JWKS endpoint updates; ✅ Docs describe rotation SOP. |
|
||||
| CORE8.RL | DONE (2025-10-12) | Authority Core | CORE8 | Deliver ASP.NET rate limiter plumbing (request metadata, dependency injection hooks) needed by Security Guild. | ✅ `/token` & `/authorize` pipelines expose limiter hooks; ✅ Tests cover throttle behaviour baseline. |
|
||||
| SEC2.HOST | TODO | Security Guild, Authority Core | SEC2.A (audit contract) | Hook audit logger into OpenIddict handlers and bootstrap endpoints. | ✅ Audit events populated with correlationId, IP, client_id; ✅ Mongo login attempts persisted; ✅ Tests verify on success/failure/lockout. |
|
||||
| SEC2.HOST | DONE (2025-10-12) | Security Guild, Authority Core | SEC2.A (audit contract) | Hook audit logger into OpenIddict handlers and bootstrap endpoints. | ✅ Audit events populated with correlationId, IP, client_id; ✅ Mongo login attempts persisted; ✅ Tests verify on success/failure/lockout. |
|
||||
| SEC3.HOST | DONE (2025-10-11) | Security Guild | CORE8.RL, SEC3.A (rate policy) | Apply rate limiter policies (`AddRateLimiter`) to `/token` and `/internal/*` endpoints with configuration binding. | ✅ Policies configurable via `StellaOpsAuthorityOptions.Security.RateLimiting`; ✅ Integration tests hit 429 after limit; ✅ Docs updated. |
|
||||
| SEC4.HOST | DONE (2025-10-12) | Security Guild, DevOps | SEC4.A (revocation schema) | Implement CLI/HTTP surface to export revocation bundle + detached JWS using `StellaOps.Cryptography`. | ✅ `stellaops auth revoke export` CLI/endpoint returns JSON + `.jws`; ✅ Verification script passes; ✅ Operator docs updated. |
|
||||
| SEC4.KEY | DONE (2025-10-12) | Security Guild, DevOps | SEC4.HOST | Integrate signing keys with provider registry (initial ES256). | ✅ Keys loaded via `ICryptoProvider` signer; ✅ Rotation SOP documented. |
|
||||
| SEC5.HOST | TODO | Security Guild | SEC5.A (threat model) | Feed Authority-specific mitigations (rate limiting, audit, revocation) into threat model + backlog. | ✅ Threat model updated; ✅ Backlog issues reference mitigations; ✅ Review sign-off captured. |
|
||||
| SEC5.HOST | DONE (2025-10-14) | Security Guild | SEC5.A (threat model) | Feed Authority-specific mitigations (rate limiting, audit, revocation) into threat model + backlog. | ✅ Threat model updated; ✅ Backlog issues reference mitigations; ✅ Review sign-off captured. |
|
||||
| SEC5.HOST-INVITES | DONE (2025-10-14) | Security Guild, Authority Core | SEC5.D | Implement bootstrap invite persistence, APIs, and background cleanup with audit coverage. | ✅ Invite store + endpoints complete; ✅ Cleanup service expires unused invites; ✅ Audit events for create/consume/expire; ✅ Build/tests green. |
|
||||
> Remark (2025-10-14): Background sweep emits invite expiry audits; integration test added.
|
||||
| SEC5.HOST-REPLAY | DONE (2025-10-14) | Security Guild, Zastava | SEC5.E | Persist token usage metadata and surface suspected replay heuristics. | ✅ Validation handlers record device metadata; ✅ Suspected replay flagged via audit/logs; ✅ Tests cover regression cases. |
|
||||
> Remark (2025-10-14): Token validation handler logs suspected replay audits with device metadata; coverage via unit/integration tests.
|
||||
| SEC3.BUILD | DONE (2025-10-11) | Authority Core, Security Guild | SEC3.HOST, FEEDMERGE-COORD-02-900 | Track normalized-range dependency fallout and restore full test matrix once Feedser range primitives land. | ✅ Feedser normalized range libraries merged; ✅ Authority + Configuration test suites (`dotnet test src/StellaOps.Authority.sln`, `dotnet test src/StellaOps.Configuration.Tests/StellaOps.Configuration.Tests.csproj`) pass without Feedser compile failures; ✅ Status recorded here/Sprints (authority-core broadcast not available). |
|
||||
| AUTHCORE-BUILD-OPENIDDICT | DONE (2025-10-14) | Authority Core | SEC2.HOST | Adapt host/audit handlers for OpenIddict 6.4 API surface (no `OpenIddictServerTransaction`) and restore Authority solution build. | ✅ Build `dotnet build src/StellaOps.Authority.sln` succeeds; ✅ Audit correlation + tamper logging verified under new abstractions; ✅ Tests updated. |
|
||||
| AUTHCORE-STORAGE-DEVICE-TOKENS | DONE (2025-10-14) | Authority Core, Storage Guild | AUTHCORE-BUILD-OPENIDDICT | Reintroduce `AuthorityTokenDeviceDocument` + projections removed during refactor so storage layer compiles. | ✅ Document type restored with mappings/migrations; ✅ Storage tests cover device artifacts; ✅ Authority solution build green. |
|
||||
| AUTHCORE-BOOTSTRAP-INVITES | DONE (2025-10-14) | Authority Core, DevOps | AUTHCORE-STORAGE-DEVICE-TOKENS | Wire bootstrap invite cleanup service against restored document schema and re-enable lifecycle tests. | ✅ `BootstrapInviteCleanupService` passes integration tests; ✅ Operator guide updated if behavior changes; ✅ Build/test matrices green. |
|
||||
|
||||
> Update status columns (TODO / DOING / DONE / BLOCKED) together with code changes. Always run `dotnet test src/StellaOps.Authority.sln` when touching host logic.
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
using System;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
@@ -16,6 +18,7 @@ using StellaOps.Cli.Services;
|
||||
using StellaOps.Cli.Services.Models;
|
||||
using StellaOps.Cli.Telemetry;
|
||||
using StellaOps.Cli.Tests.Testing;
|
||||
using StellaOps.Cryptography;
|
||||
|
||||
namespace StellaOps.Cli.Tests.Commands;
|
||||
|
||||
@@ -208,6 +211,34 @@ public sealed class CommandHandlersTests
|
||||
}
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[InlineData(null)]
|
||||
[InlineData("default")]
|
||||
[InlineData("libsodium")]
|
||||
public async Task HandleAuthRevokeVerifyAsync_VerifiesBundlesUsingProviderRegistry(string? providerHint)
|
||||
{
|
||||
var original = Environment.ExitCode;
|
||||
using var tempDir = new TempDirectory();
|
||||
|
||||
try
|
||||
{
|
||||
var artifacts = await WriteRevocationArtifactsAsync(tempDir, providerHint);
|
||||
|
||||
await CommandHandlers.HandleAuthRevokeVerifyAsync(
|
||||
artifacts.BundlePath,
|
||||
artifacts.SignaturePath,
|
||||
artifacts.KeyPath,
|
||||
verbose: true,
|
||||
cancellationToken: CancellationToken.None);
|
||||
|
||||
Assert.Equal(0, Environment.ExitCode);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Environment.ExitCode = original;
|
||||
}
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task HandleAuthStatusAsync_ReportsCachedToken()
|
||||
{
|
||||
@@ -360,6 +391,79 @@ public sealed class CommandHandlersTests
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<RevocationArtifactPaths> WriteRevocationArtifactsAsync(TempDirectory temp, string? providerHint)
|
||||
{
|
||||
var (bundleBytes, signature, keyPem) = await BuildRevocationArtifactsAsync(providerHint);
|
||||
|
||||
var bundlePath = Path.Combine(temp.Path, "revocation-bundle.json");
|
||||
var signaturePath = Path.Combine(temp.Path, "revocation-bundle.json.jws");
|
||||
var keyPath = Path.Combine(temp.Path, "revocation-key.pem");
|
||||
|
||||
await File.WriteAllBytesAsync(bundlePath, bundleBytes);
|
||||
await File.WriteAllTextAsync(signaturePath, signature);
|
||||
await File.WriteAllTextAsync(keyPath, keyPem);
|
||||
|
||||
return new RevocationArtifactPaths(bundlePath, signaturePath, keyPath);
|
||||
}
|
||||
|
||||
private static async Task<(byte[] Bundle, string Signature, string KeyPem)> BuildRevocationArtifactsAsync(string? providerHint)
|
||||
{
|
||||
var bundleBytes = Encoding.UTF8.GetBytes("{\"revocations\":[]}");
|
||||
|
||||
using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
var parameters = ecdsa.ExportParameters(includePrivateParameters: true);
|
||||
|
||||
var signingKey = new CryptoSigningKey(
|
||||
new CryptoKeyReference("revocation-test"),
|
||||
SignatureAlgorithms.Es256,
|
||||
privateParameters: in parameters,
|
||||
createdAt: DateTimeOffset.UtcNow);
|
||||
|
||||
var provider = new DefaultCryptoProvider();
|
||||
provider.UpsertSigningKey(signingKey);
|
||||
var signer = provider.GetSigner(SignatureAlgorithms.Es256, signingKey.Reference);
|
||||
|
||||
var header = new Dictionary<string, object>
|
||||
{
|
||||
["alg"] = SignatureAlgorithms.Es256,
|
||||
["kid"] = signingKey.Reference.KeyId,
|
||||
["typ"] = "application/vnd.stellaops.revocation-bundle+jws",
|
||||
["b64"] = false,
|
||||
["crit"] = new[] { "b64" }
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(providerHint))
|
||||
{
|
||||
header["provider"] = providerHint;
|
||||
}
|
||||
|
||||
var serializerOptions = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = null,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
var headerJson = JsonSerializer.Serialize(header, serializerOptions);
|
||||
var encodedHeader = Base64UrlEncoder.Encode(Encoding.UTF8.GetBytes(headerJson));
|
||||
|
||||
var signingInput = new byte[encodedHeader.Length + 1 + bundleBytes.Length];
|
||||
var headerBytes = Encoding.ASCII.GetBytes(encodedHeader);
|
||||
Buffer.BlockCopy(headerBytes, 0, signingInput, 0, headerBytes.Length);
|
||||
signingInput[headerBytes.Length] = (byte)'.';
|
||||
Buffer.BlockCopy(bundleBytes, 0, signingInput, headerBytes.Length + 1, bundleBytes.Length);
|
||||
|
||||
var signatureBytes = await signer.SignAsync(signingInput);
|
||||
var encodedSignature = Base64UrlEncoder.Encode(signatureBytes);
|
||||
var jws = string.Concat(encodedHeader, "..", encodedSignature);
|
||||
|
||||
var publicKeyBytes = ecdsa.ExportSubjectPublicKeyInfo();
|
||||
var keyPem = new string(PemEncoding.Write("PUBLIC KEY", publicKeyBytes));
|
||||
|
||||
return (bundleBytes, jws, keyPem);
|
||||
}
|
||||
|
||||
private sealed record RevocationArtifactPaths(string BundlePath, string SignaturePath, string KeyPath);
|
||||
|
||||
private static IServiceProvider BuildServiceProvider(
|
||||
IBackendOperationsClient backend,
|
||||
IScannerExecutor? executor = null,
|
||||
|
||||
@@ -641,11 +641,12 @@ internal static class CommandHandlers
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Revocation bundle exported to {Directory} (sequence {Sequence}, issued {Issued:u}, signing key {KeyId}).",
|
||||
"Revocation bundle exported to {Directory} (sequence {Sequence}, issued {Issued:u}, signing key {KeyId}, provider {Provider}).",
|
||||
directory,
|
||||
result.Sequence,
|
||||
result.IssuedAt,
|
||||
string.IsNullOrWhiteSpace(result.SigningKeyId) ? "<unknown>" : result.SigningKeyId);
|
||||
string.IsNullOrWhiteSpace(result.SigningKeyId) ? "<unknown>" : result.SigningKeyId,
|
||||
string.IsNullOrWhiteSpace(result.SigningProvider) ? "default" : result.SigningProvider);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -709,22 +710,62 @@ internal static class CommandHandlers
|
||||
algorithm = SignatureAlgorithms.Es256;
|
||||
}
|
||||
|
||||
var hashAlgorithm = ResolveHashAlgorithm(algorithm);
|
||||
if (hashAlgorithm is null)
|
||||
var providerHint = header.TryGetProperty("provider", out var providerElement)
|
||||
? providerElement.GetString()
|
||||
: null;
|
||||
|
||||
var keyId = header.TryGetProperty("kid", out var kidElement) ? kidElement.GetString() : null;
|
||||
if (string.IsNullOrWhiteSpace(keyId))
|
||||
{
|
||||
logger.LogError("Unsupported signing algorithm '{Algorithm}'.", algorithm);
|
||||
keyId = Path.GetFileNameWithoutExtension(keyPath);
|
||||
logger.LogWarning("JWS header missing 'kid'; using fallback key id {KeyId}.", keyId);
|
||||
}
|
||||
|
||||
CryptoSigningKey signingKey;
|
||||
try
|
||||
{
|
||||
signingKey = CreateVerificationSigningKey(keyId!, algorithm!, providerHint, keyPem, keyPath);
|
||||
}
|
||||
catch (Exception ex) when (ex is InvalidOperationException or CryptographicException)
|
||||
{
|
||||
logger.LogError(ex, "Failed to load verification key material.");
|
||||
Environment.ExitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
using var ecdsa = ECDsa.Create();
|
||||
var providers = new List<ICryptoProvider>
|
||||
{
|
||||
new DefaultCryptoProvider()
|
||||
};
|
||||
|
||||
#if STELLAOPS_CRYPTO_SODIUM
|
||||
providers.Add(new LibsodiumCryptoProvider());
|
||||
#endif
|
||||
|
||||
foreach (var provider in providers)
|
||||
{
|
||||
if (provider.Supports(CryptoCapability.Verification, algorithm!))
|
||||
{
|
||||
provider.UpsertSigningKey(signingKey);
|
||||
}
|
||||
}
|
||||
|
||||
var preferredOrder = !string.IsNullOrWhiteSpace(providerHint)
|
||||
? new[] { providerHint! }
|
||||
: Array.Empty<string>();
|
||||
var registry = new CryptoProviderRegistry(providers, preferredOrder);
|
||||
CryptoSignerResolution resolution;
|
||||
try
|
||||
{
|
||||
ecdsa.ImportFromPem(keyPem);
|
||||
resolution = registry.ResolveSigner(
|
||||
CryptoCapability.Verification,
|
||||
algorithm!,
|
||||
signingKey.Reference,
|
||||
providerHint);
|
||||
}
|
||||
catch (CryptographicException ex)
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to import signing key.");
|
||||
logger.LogError(ex, "No crypto provider available for verification (algorithm {Algorithm}).", algorithm);
|
||||
Environment.ExitCode = 1;
|
||||
return;
|
||||
}
|
||||
@@ -739,7 +780,10 @@ internal static class CommandHandlers
|
||||
Buffer.BlockCopy(bundleBytes, 0, buffer, headerBytes.Length + 1, bundleBytes.Length);
|
||||
|
||||
var signatureBytes = Base64UrlDecode(encodedSignature);
|
||||
var verified = ecdsa.VerifyData(new ReadOnlySpan<byte>(buffer, 0, signingInputLength), signatureBytes, hashAlgorithm.Value);
|
||||
var verified = await resolution.Signer.VerifyAsync(
|
||||
new ReadOnlyMemory<byte>(buffer, 0, signingInputLength),
|
||||
signatureBytes,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!verified)
|
||||
{
|
||||
@@ -753,7 +797,19 @@ internal static class CommandHandlers
|
||||
ArrayPool<byte>.Shared.Return(buffer);
|
||||
}
|
||||
|
||||
logger.LogInformation("Signature verified using algorithm {Algorithm}.", algorithm);
|
||||
if (!string.IsNullOrWhiteSpace(providerHint) && !string.Equals(providerHint, resolution.ProviderName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Preferred provider '{Preferred}' unavailable; verification used '{Provider}'.",
|
||||
providerHint,
|
||||
resolution.ProviderName);
|
||||
}
|
||||
|
||||
logger.LogInformation(
|
||||
"Signature verified using algorithm {Algorithm} via provider {Provider} (kid {KeyId}).",
|
||||
algorithm,
|
||||
resolution.ProviderName,
|
||||
signingKey.Reference.KeyId);
|
||||
|
||||
if (verbose)
|
||||
{
|
||||
@@ -812,24 +868,39 @@ internal static class CommandHandlers
|
||||
return Convert.FromBase64String(normalized);
|
||||
}
|
||||
|
||||
private static HashAlgorithmName? ResolveHashAlgorithm(string algorithm)
|
||||
private static CryptoSigningKey CreateVerificationSigningKey(
|
||||
string keyId,
|
||||
string algorithm,
|
||||
string? providerHint,
|
||||
string keyPem,
|
||||
string keyPath)
|
||||
{
|
||||
if (string.Equals(algorithm, SignatureAlgorithms.Es256, StringComparison.OrdinalIgnoreCase))
|
||||
if (string.IsNullOrWhiteSpace(keyPem))
|
||||
{
|
||||
return HashAlgorithmName.SHA256;
|
||||
throw new InvalidOperationException("Verification key PEM content is empty.");
|
||||
}
|
||||
|
||||
if (string.Equals(algorithm, SignatureAlgorithms.Es384, StringComparison.OrdinalIgnoreCase))
|
||||
using var ecdsa = ECDsa.Create();
|
||||
ecdsa.ImportFromPem(keyPem);
|
||||
|
||||
var parameters = ecdsa.ExportParameters(includePrivateParameters: false);
|
||||
if (parameters.D is null || parameters.D.Length == 0)
|
||||
{
|
||||
return HashAlgorithmName.SHA384;
|
||||
parameters.D = new byte[] { 0x01 };
|
||||
}
|
||||
|
||||
if (string.Equals(algorithm, SignatureAlgorithms.Es512, StringComparison.OrdinalIgnoreCase))
|
||||
var metadata = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
return HashAlgorithmName.SHA512;
|
||||
}
|
||||
["source"] = Path.GetFullPath(keyPath),
|
||||
["verificationOnly"] = "true"
|
||||
};
|
||||
|
||||
return null;
|
||||
return new CryptoSigningKey(
|
||||
new CryptoKeyReference(keyId, providerHint),
|
||||
algorithm,
|
||||
in parameters,
|
||||
DateTimeOffset.UtcNow,
|
||||
metadata: metadata);
|
||||
}
|
||||
|
||||
private static string FormatDuration(TimeSpan duration)
|
||||
|
||||
@@ -78,7 +78,12 @@ internal sealed class AuthorityRevocationClient : IAuthorityRevocationClient
|
||||
|
||||
if (verbose)
|
||||
{
|
||||
logger.LogInformation("Received revocation export sequence {Sequence} (sha256:{Digest}, signing key {KeyId}).", payload.Sequence, digest, payload.SigningKeyId ?? "<unspecified>");
|
||||
logger.LogInformation(
|
||||
"Received revocation export sequence {Sequence} (sha256:{Digest}, signing key {KeyId}, provider {Provider}).",
|
||||
payload.Sequence,
|
||||
digest,
|
||||
payload.SigningKeyId ?? "<unspecified>",
|
||||
string.IsNullOrWhiteSpace(payload.Signature?.Provider) ? "default" : payload.Signature!.Provider);
|
||||
}
|
||||
|
||||
return new AuthorityRevocationExportResult
|
||||
@@ -88,7 +93,8 @@ internal sealed class AuthorityRevocationClient : IAuthorityRevocationClient
|
||||
Digest = digest,
|
||||
Sequence = payload.Sequence,
|
||||
IssuedAt = payload.IssuedAt,
|
||||
SigningKeyId = payload.SigningKeyId
|
||||
SigningKeyId = payload.SigningKeyId,
|
||||
SigningProvider = payload.Signature?.Provider
|
||||
};
|
||||
}
|
||||
|
||||
@@ -201,6 +207,9 @@ internal sealed class AuthorityRevocationClient : IAuthorityRevocationClient
|
||||
[JsonPropertyName("keyId")]
|
||||
public string KeyId { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("provider")]
|
||||
public string Provider { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("value")]
|
||||
public string Value { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
@@ -15,4 +15,6 @@ internal sealed class AuthorityRevocationExportResult
|
||||
public required DateTimeOffset IssuedAt { get; init; }
|
||||
|
||||
public string? SigningKeyId { get; init; }
|
||||
|
||||
public string? SigningProvider { get; init; }
|
||||
}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using StellaOps.Authority.Plugins.Abstractions;
|
||||
using StellaOps.Configuration;
|
||||
using Xunit;
|
||||
@@ -97,6 +98,61 @@ public class AuthorityPluginConfigurationLoaderTests : IDisposable
|
||||
Assert.Contains("unknown capability", ex.Message, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Analyze_ReturnsWarning_WhenStandardPasswordPolicyWeaker()
|
||||
{
|
||||
var pluginDir = Path.Combine(tempRoot, "etc", "authority.plugins");
|
||||
Directory.CreateDirectory(pluginDir);
|
||||
|
||||
var standardConfigPath = Path.Combine(pluginDir, "standard.yaml");
|
||||
File.WriteAllText(standardConfigPath, "passwordPolicy:\n minimumLength: 8\n requireSymbol: false\n");
|
||||
|
||||
var options = CreateOptions();
|
||||
options.Plugins.ConfigurationDirectory = "etc/authority.plugins";
|
||||
options.Plugins.Descriptors["standard"] = new AuthorityPluginDescriptorOptions
|
||||
{
|
||||
AssemblyName = "StellaOps.Authority.Plugin.Standard",
|
||||
Enabled = true
|
||||
};
|
||||
|
||||
options.Validate();
|
||||
|
||||
var contexts = AuthorityPluginConfigurationLoader.Load(options, tempRoot);
|
||||
var diagnostics = AuthorityPluginConfigurationAnalyzer.Analyze(contexts);
|
||||
|
||||
var diagnostic = Assert.Single(diagnostics);
|
||||
Assert.Equal(AuthorityConfigurationDiagnosticSeverity.Warning, diagnostic.Severity);
|
||||
Assert.Equal("standard", diagnostic.PluginName);
|
||||
Assert.Contains("minimum length 8", diagnostic.Message, StringComparison.OrdinalIgnoreCase);
|
||||
Assert.Contains("symbol requirement disabled", diagnostic.Message, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void Analyze_ReturnsNoDiagnostics_WhenPasswordPolicyMatchesBaseline()
|
||||
{
|
||||
var pluginDir = Path.Combine(tempRoot, "etc", "authority.plugins");
|
||||
Directory.CreateDirectory(pluginDir);
|
||||
|
||||
var standardConfigPath = Path.Combine(pluginDir, "standard.yaml");
|
||||
// Baseline configuration (no overrides)
|
||||
File.WriteAllText(standardConfigPath, "bootstrapUser:\n username: bootstrap\n password: Bootstrap1!\n");
|
||||
|
||||
var options = CreateOptions();
|
||||
options.Plugins.ConfigurationDirectory = "etc/authority.plugins";
|
||||
options.Plugins.Descriptors["standard"] = new AuthorityPluginDescriptorOptions
|
||||
{
|
||||
AssemblyName = "StellaOps.Authority.Plugin.Standard",
|
||||
Enabled = true
|
||||
};
|
||||
|
||||
options.Validate();
|
||||
|
||||
var contexts = AuthorityPluginConfigurationLoader.Load(options, tempRoot);
|
||||
var diagnostics = AuthorityPluginConfigurationAnalyzer.Analyze(contexts);
|
||||
|
||||
Assert.Empty(diagnostics);
|
||||
}
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
try
|
||||
@@ -121,6 +177,8 @@ public class AuthorityPluginConfigurationLoaderTests : IDisposable
|
||||
};
|
||||
|
||||
options.Storage.ConnectionString = "mongodb://localhost:27017/authority_test";
|
||||
options.Signing.ActiveKeyId = "test-key";
|
||||
options.Signing.KeyPath = "/tmp/authority-test-key.pem";
|
||||
return options;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
using System;
|
||||
|
||||
namespace StellaOps.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a configuration diagnostic emitted while analysing Authority plugin settings.
|
||||
/// </summary>
|
||||
public sealed record AuthorityConfigurationDiagnostic(
|
||||
string PluginName,
|
||||
AuthorityConfigurationDiagnosticSeverity Severity,
|
||||
string Message)
|
||||
{
|
||||
public string PluginName { get; init; } = PluginName ?? throw new ArgumentNullException(nameof(PluginName));
|
||||
|
||||
public AuthorityConfigurationDiagnosticSeverity Severity { get; init; } = Severity;
|
||||
|
||||
public string Message { get; init; } = Message ?? throw new ArgumentNullException(nameof(Message));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Severity levels for configuration diagnostics.
|
||||
/// </summary>
|
||||
public enum AuthorityConfigurationDiagnosticSeverity
|
||||
{
|
||||
Info = 0,
|
||||
Warning = 1,
|
||||
Error = 2
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using StellaOps.Authority.Plugins.Abstractions;
|
||||
|
||||
namespace StellaOps.Configuration;
|
||||
|
||||
/// <summary>
|
||||
/// Analyses Authority plugin configurations for common security issues.
|
||||
/// </summary>
|
||||
public static class AuthorityPluginConfigurationAnalyzer
|
||||
{
|
||||
private const int BaselineMinimumLength = 12;
|
||||
private const bool BaselineRequireUppercase = true;
|
||||
private const bool BaselineRequireLowercase = true;
|
||||
private const bool BaselineRequireDigit = true;
|
||||
private const bool BaselineRequireSymbol = true;
|
||||
|
||||
/// <summary>
|
||||
/// Evaluates plugin contexts and returns diagnostics describing potential misconfigurations.
|
||||
/// </summary>
|
||||
/// <param name="contexts">Plugin contexts produced by <see cref="AuthorityPluginConfigurationLoader"/>.</param>
|
||||
/// <returns>Diagnostics describing any detected issues.</returns>
|
||||
public static IReadOnlyList<AuthorityConfigurationDiagnostic> Analyze(IEnumerable<AuthorityPluginContext> contexts)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(contexts);
|
||||
|
||||
var diagnostics = new List<AuthorityConfigurationDiagnostic>();
|
||||
|
||||
foreach (var context in contexts)
|
||||
{
|
||||
if (context is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (string.Equals(context.Manifest.AssemblyName, "StellaOps.Authority.Plugin.Standard", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
AnalyzeStandardPlugin(context, diagnostics);
|
||||
}
|
||||
}
|
||||
|
||||
return diagnostics;
|
||||
}
|
||||
|
||||
private static void AnalyzeStandardPlugin(AuthorityPluginContext context, ICollection<AuthorityConfigurationDiagnostic> diagnostics)
|
||||
{
|
||||
var section = context.Configuration.GetSection("passwordPolicy");
|
||||
if (!section.Exists())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
int minLength = section.GetValue("minimumLength", BaselineMinimumLength);
|
||||
bool requireUppercase = section.GetValue("requireUppercase", BaselineRequireUppercase);
|
||||
bool requireLowercase = section.GetValue("requireLowercase", BaselineRequireLowercase);
|
||||
bool requireDigit = section.GetValue("requireDigit", BaselineRequireDigit);
|
||||
bool requireSymbol = section.GetValue("requireSymbol", BaselineRequireSymbol);
|
||||
|
||||
var deviations = new List<string>();
|
||||
|
||||
if (minLength < BaselineMinimumLength)
|
||||
{
|
||||
deviations.Add($"minimum length {minLength.ToString(CultureInfo.InvariantCulture)} < {BaselineMinimumLength}");
|
||||
}
|
||||
|
||||
if (!requireUppercase && BaselineRequireUppercase)
|
||||
{
|
||||
deviations.Add("uppercase requirement disabled");
|
||||
}
|
||||
|
||||
if (!requireLowercase && BaselineRequireLowercase)
|
||||
{
|
||||
deviations.Add("lowercase requirement disabled");
|
||||
}
|
||||
|
||||
if (!requireDigit && BaselineRequireDigit)
|
||||
{
|
||||
deviations.Add("digit requirement disabled");
|
||||
}
|
||||
|
||||
if (!requireSymbol && BaselineRequireSymbol)
|
||||
{
|
||||
deviations.Add("symbol requirement disabled");
|
||||
}
|
||||
|
||||
if (deviations.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var message = $"Password policy for plugin '{context.Manifest.Name}' weakens host defaults: {string.Join(", ", deviations)}.";
|
||||
diagnostics.Add(new AuthorityConfigurationDiagnostic(context.Manifest.Name, AuthorityConfigurationDiagnosticSeverity.Warning, message));
|
||||
}
|
||||
}
|
||||
@@ -31,14 +31,19 @@ public static class CryptoServiceCollectionExtensions
|
||||
services.Configure(configureRegistry);
|
||||
}
|
||||
|
||||
services.TryAddSingleton(sp =>
|
||||
services.TryAddSingleton<DefaultCryptoProvider>(sp =>
|
||||
{
|
||||
var provider = new DefaultCryptoProvider();
|
||||
configureProvider?.Invoke(provider);
|
||||
return provider;
|
||||
});
|
||||
|
||||
services.TryAddEnumerable(ServiceDescriptor.Singleton<ICryptoProvider>(sp => sp.GetRequiredService<DefaultCryptoProvider>()));
|
||||
services.TryAddEnumerable(ServiceDescriptor.Singleton<ICryptoProvider, DefaultCryptoProvider>());
|
||||
|
||||
#if STELLAOPS_CRYPTO_SODIUM
|
||||
services.TryAddSingleton<LibsodiumCryptoProvider>();
|
||||
services.TryAddEnumerable(ServiceDescriptor.Singleton<ICryptoProvider, LibsodiumCryptoProvider>());
|
||||
#endif
|
||||
|
||||
services.TryAddSingleton<ICryptoProviderRegistry>(sp =>
|
||||
{
|
||||
|
||||
@@ -41,20 +41,22 @@ public class CryptoProviderRegistryTests
|
||||
|
||||
var registry = new CryptoProviderRegistry(new[] { providerA, providerB }, Array.Empty<string>());
|
||||
|
||||
var hintSigner = registry.ResolveSigner(
|
||||
var hintResolution = registry.ResolveSigner(
|
||||
CryptoCapability.Signing,
|
||||
SignatureAlgorithms.Es256,
|
||||
new CryptoKeyReference("key-b"),
|
||||
preferredProvider: "providerB");
|
||||
|
||||
Assert.Equal("key-b", hintSigner.KeyId);
|
||||
Assert.Equal("providerB", hintResolution.ProviderName);
|
||||
Assert.Equal("key-b", hintResolution.Signer.KeyId);
|
||||
|
||||
var fallbackSigner = registry.ResolveSigner(
|
||||
var fallbackResolution = registry.ResolveSigner(
|
||||
CryptoCapability.Signing,
|
||||
SignatureAlgorithms.Es256,
|
||||
new CryptoKeyReference("key-a"));
|
||||
|
||||
Assert.Equal("key-a", fallbackSigner.KeyId);
|
||||
Assert.Equal("providerA", fallbackResolution.ProviderName);
|
||||
Assert.Equal("key-a", fallbackResolution.Signer.KeyId);
|
||||
}
|
||||
|
||||
private sealed class FakeCryptoProvider : ICryptoProvider
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
#if STELLAOPS_CRYPTO_SODIUM
|
||||
using System;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Threading.Tasks;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Cryptography.Tests;
|
||||
|
||||
public class LibsodiumCryptoProviderTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task LibsodiumProvider_SignsAndVerifiesEs256()
|
||||
{
|
||||
var provider = new LibsodiumCryptoProvider();
|
||||
using var ecdsa = ECDsa.Create(ECCurve.NamedCurves.nistP256);
|
||||
var parameters = ecdsa.ExportParameters(includePrivateParameters: true);
|
||||
|
||||
var signingKey = new CryptoSigningKey(
|
||||
new CryptoKeyReference("libsodium-key"),
|
||||
SignatureAlgorithms.Es256,
|
||||
privateParameters: in parameters,
|
||||
createdAt: DateTimeOffset.UtcNow);
|
||||
|
||||
provider.UpsertSigningKey(signingKey);
|
||||
|
||||
var signer = provider.GetSigner(SignatureAlgorithms.Es256, signingKey.Reference);
|
||||
|
||||
var payload = Encoding.UTF8.GetBytes("libsodium-test");
|
||||
var signature = await signer.SignAsync(payload);
|
||||
|
||||
Assert.True(signature.Length > 0);
|
||||
|
||||
var verified = await signer.VerifyAsync(payload, signature);
|
||||
Assert.True(verified);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -5,6 +5,9 @@
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<IsPackable>false</IsPackable>
|
||||
</PropertyGroup>
|
||||
<PropertyGroup Condition="'$(StellaOpsCryptoSodium)' == 'true'">
|
||||
<DefineConstants>$(DefineConstants);STELLAOPS_CRYPTO_SODIUM</DefineConstants>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Cryptography\StellaOps.Cryptography.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
@@ -76,9 +76,11 @@ public interface ICryptoProviderRegistry
|
||||
/// <param name="keyReference">Key reference.</param>
|
||||
/// <param name="preferredProvider">Optional provider hint.</param>
|
||||
/// <returns>Resolved signer.</returns>
|
||||
ICryptoSigner ResolveSigner(
|
||||
CryptoSignerResolution ResolveSigner(
|
||||
CryptoCapability capability,
|
||||
string algorithmId,
|
||||
CryptoKeyReference keyReference,
|
||||
string? preferredProvider = null);
|
||||
}
|
||||
|
||||
public sealed record CryptoSignerResolution(ICryptoSigner Signer, string ProviderName);
|
||||
|
||||
@@ -72,7 +72,7 @@ public sealed class CryptoProviderRegistry : ICryptoProviderRegistry
|
||||
$"No crypto provider is registered for capability '{capability}' and algorithm '{algorithmId}'.");
|
||||
}
|
||||
|
||||
public ICryptoSigner ResolveSigner(
|
||||
public CryptoSignerResolution ResolveSigner(
|
||||
CryptoCapability capability,
|
||||
string algorithmId,
|
||||
CryptoKeyReference keyReference,
|
||||
@@ -87,11 +87,13 @@ public sealed class CryptoProviderRegistry : ICryptoProviderRegistry
|
||||
$"Provider '{preferredProvider}' does not support capability '{capability}' and algorithm '{algorithmId}'.");
|
||||
}
|
||||
|
||||
return hinted.GetSigner(algorithmId, keyReference);
|
||||
var signer = hinted.GetSigner(algorithmId, keyReference);
|
||||
return new CryptoSignerResolution(signer, hinted.Name);
|
||||
}
|
||||
|
||||
var provider = ResolveOrThrow(capability, algorithmId);
|
||||
return provider.GetSigner(algorithmId, keyReference);
|
||||
var resolved = provider.GetSigner(algorithmId, keyReference);
|
||||
return new CryptoSignerResolution(resolved, provider.Name);
|
||||
}
|
||||
|
||||
private IEnumerable<ICryptoProvider> EnumerateCandidates()
|
||||
|
||||
124
src/StellaOps.Cryptography/LibsodiumCryptoProvider.cs
Normal file
124
src/StellaOps.Cryptography/LibsodiumCryptoProvider.cs
Normal file
@@ -0,0 +1,124 @@
|
||||
#if STELLAOPS_CRYPTO_SODIUM
|
||||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.IdentityModel.Tokens;
|
||||
|
||||
namespace StellaOps.Cryptography;
|
||||
|
||||
/// <summary>
|
||||
/// Libsodium-backed crypto provider (ES256) registered when <c>STELLAOPS_CRYPTO_SODIUM</c> is defined.
|
||||
/// </summary>
|
||||
public sealed class LibsodiumCryptoProvider : ICryptoProvider
|
||||
{
|
||||
private static readonly HashSet<string> SupportedAlgorithms = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
SignatureAlgorithms.Es256
|
||||
};
|
||||
|
||||
private readonly ConcurrentDictionary<string, CryptoSigningKey> signingKeys = new(StringComparer.Ordinal);
|
||||
|
||||
public string Name => "libsodium";
|
||||
|
||||
public bool Supports(CryptoCapability capability, string algorithmId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(algorithmId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return capability switch
|
||||
{
|
||||
CryptoCapability.Signing or CryptoCapability.Verification => SupportedAlgorithms.Contains(algorithmId),
|
||||
_ => false
|
||||
};
|
||||
}
|
||||
|
||||
public IPasswordHasher GetPasswordHasher(string algorithmId)
|
||||
=> throw new NotSupportedException("Libsodium provider does not expose password hashing capabilities.");
|
||||
|
||||
public ICryptoSigner GetSigner(string algorithmId, CryptoKeyReference keyReference)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(keyReference);
|
||||
|
||||
EnsureAlgorithmSupported(algorithmId);
|
||||
|
||||
if (!signingKeys.TryGetValue(keyReference.KeyId, out var signingKey))
|
||||
{
|
||||
throw new KeyNotFoundException($"Signing key '{keyReference.KeyId}' is not registered with provider '{Name}'.");
|
||||
}
|
||||
|
||||
if (!string.Equals(signingKey.AlgorithmId, algorithmId, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Signing key '{keyReference.KeyId}' is registered for algorithm '{signingKey.AlgorithmId}', not '{algorithmId}'.");
|
||||
}
|
||||
|
||||
return new LibsodiumEcdsaSigner(signingKey);
|
||||
}
|
||||
|
||||
public void UpsertSigningKey(CryptoSigningKey signingKey)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(signingKey);
|
||||
EnsureAlgorithmSupported(signingKey.AlgorithmId);
|
||||
|
||||
signingKeys.AddOrUpdate(signingKey.Reference.KeyId, signingKey, (_, _) => signingKey);
|
||||
}
|
||||
|
||||
public bool RemoveSigningKey(string keyId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(keyId))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return signingKeys.TryRemove(keyId, out _);
|
||||
}
|
||||
|
||||
public IReadOnlyCollection<CryptoSigningKey> GetSigningKeys()
|
||||
=> signingKeys.Values.ToArray();
|
||||
|
||||
private static void EnsureAlgorithmSupported(string algorithmId)
|
||||
{
|
||||
if (!SupportedAlgorithms.Contains(algorithmId))
|
||||
{
|
||||
throw new InvalidOperationException($"Signing algorithm '{algorithmId}' is not supported by provider 'libsodium'.");
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class LibsodiumEcdsaSigner : ICryptoSigner
|
||||
{
|
||||
private readonly CryptoSigningKey signingKey;
|
||||
private readonly ICryptoSigner fallbackSigner;
|
||||
|
||||
public LibsodiumEcdsaSigner(CryptoSigningKey signingKey)
|
||||
{
|
||||
this.signingKey = signingKey ?? throw new ArgumentNullException(nameof(signingKey));
|
||||
fallbackSigner = EcdsaSigner.Create(signingKey);
|
||||
}
|
||||
|
||||
public string KeyId => signingKey.Reference.KeyId;
|
||||
|
||||
public string AlgorithmId => signingKey.AlgorithmId;
|
||||
|
||||
public ValueTask<byte[]> SignAsync(ReadOnlyMemory<byte> data, CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
// TODO(SEC5.B1): replace fallback with libsodium bindings once native interop lands.
|
||||
return fallbackSigner.SignAsync(data, cancellationToken);
|
||||
}
|
||||
|
||||
public ValueTask<bool> VerifyAsync(ReadOnlyMemory<byte> data, ReadOnlyMemory<byte> signature, CancellationToken cancellationToken = default)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return fallbackSigner.VerifyAsync(data, signature, cancellationToken);
|
||||
}
|
||||
|
||||
public JsonWebKey ExportPublicJsonWebKey()
|
||||
=> fallbackSigner.ExportPublicJsonWebKey();
|
||||
}
|
||||
}
|
||||
#endif
|
||||
@@ -4,22 +4,34 @@
|
||||
|----|--------|-------|-------------|--------------|---------------|
|
||||
| SEC1.A | DONE (2025-10-11) | Security Guild | Introduce `Argon2idPasswordHasher` backed by Konscious defaults. Wire options into `StandardPluginOptions` (`PasswordHashOptions`) and `StellaOpsAuthorityOptions.Security.PasswordHashing`. | PLG3, CORE3 | ✅ Hashes emit PHC string `$argon2id$v=19$m=19456,t=2,p=1$...`; ✅ `NeedsRehash` promotes PBKDF2 → Argon2; ✅ Integration tests cover tamper, legacy rehash, perf p95 < 250 ms. |
|
||||
| SEC1.B | DONE (2025-10-12) | Security Guild | Add compile-time switch to enable libsodium/Core variants later (`STELLAOPS_CRYPTO_SODIUM`). Document build variable. | SEC1.A | ✅ Conditional compilation path compiles; ✅ README snippet in `docs/security/password-hashing.md`. |
|
||||
| SEC2.A | TODO | Security Guild + Core | Define audit event contract (`AuthEventRecord`) with subject/client/scope/IP/outcome/correlationId and PII tags. | CORE5–CORE7 | ✅ Contract shipped in `StellaOps.Cryptography` (or shared abstractions); ✅ Docs in `docs/security/audit-events.md`. |
|
||||
| SEC2.B | TODO | Security Guild | Emit audit records from OpenIddict handlers (password + client creds) and bootstrap APIs. Persist via `IAuthorityLoginAttemptStore`. | SEC2.A | ✅ Tests assert three flows (success/failure/lockout); ✅ Serilog output contains correlationId + PII tagging; ✅ Mongo store holds summary rows. |
|
||||
| SEC3.A | BLOCKED (CORE8) | Security Guild + Core | Configure ASP.NET rate limiter (`AddRateLimiter`) with fixed-window policy keyed by IP + `client_id`. Apply to `/token` and `/internal/*`. | CORE8 completion | ✅ Middleware active; ✅ Configurable limits via options; ✅ Integration test hits 429. |
|
||||
| SEC3.B | TODO | Security Guild | Document lockout + rate-limit tuning guidance and escalation thresholds. | SEC3.A | ✅ Section in `docs/security/rate-limits.md`; ✅ Includes SOC alert recommendations. |
|
||||
| SEC2.A | DONE (2025-10-13) | Security Guild + Core | Define audit event contract (`AuthEventRecord`) with subject/client/scope/IP/outcome/correlationId and PII tags. | CORE5–CORE7 | ✅ Contract shipped in `StellaOps.Cryptography` (or shared abstractions); ✅ Docs in `docs/security/audit-events.md`. |
|
||||
| SEC2.B | DONE (2025-10-13) | Security Guild | Emit audit records from OpenIddict handlers (password + client creds) and bootstrap APIs. Persist via `IAuthorityLoginAttemptStore`. | SEC2.A | ✅ Tests assert three flows (success/failure/lockout); ✅ Serilog output contains correlationId + PII tagging; ✅ Mongo store holds summary rows. |
|
||||
| SEC3.A | DONE (2025-10-12) | Security Guild + Core | Configure ASP.NET rate limiter (`AddRateLimiter`) with fixed-window policy keyed by IP + `client_id`. Apply to `/token` and `/internal/*`. | CORE8 completion | ✅ Middleware active; ✅ Configurable limits via options; ✅ Integration test hits 429. |
|
||||
| SEC3.B | DONE (2025-10-13) | Security Guild | Document lockout + rate-limit tuning guidance and escalation thresholds. | SEC3.A | ✅ Section in `docs/security/rate-limits.md`; ✅ Includes SOC alert recommendations. |
|
||||
| SEC4.A | DONE (2025-10-12) | Security Guild + DevOps | Define revocation JSON schema (`revocation_bundle.schema.json`) and detached JWS workflow. | CORE9, OPS3 | ✅ Schema + sample committed; ✅ CLI command `stellaops auth revoke export` scaffolded with acceptance tests; ✅ Verification script + docs. |
|
||||
| SEC4.B | DONE (2025-10-12) | Security Guild | Integrate signing keys with crypto provider abstraction (initially ES256 via BCL). | SEC4.A, D5 | ✅ `ICryptoProvider.GetSigner` stub + default BCL signer; ✅ Unit tests verifying signature roundtrip. |
|
||||
| SEC5.A | DONE (2025-10-12) | Security Guild | Author STRIDE threat model (`docs/security/authority-threat-model.md`) covering token, bootstrap, revocation, CLI, plugin surfaces. | All SEC1–SEC4 in progress | ✅ DFDs + trust boundaries drawn; ✅ Risk table with owners/actions; ✅ Follow-up backlog issues created. |
|
||||
| SEC5.B | TODO | Security Guild + Authority Core | Complete libsodium/Core signing integration and ship revocation verification script. | SEC4.A, SEC4.B, SEC4.HOST | ✅ libsodium/Core signing provider wired; ✅ `stellaops auth revoke verify` script published; ✅ Revocation docs updated with verification workflow. |
|
||||
| SEC5.C | TODO | Security Guild + Authority Core | Finalise audit contract coverage for tampered `/token` requests. | SEC2.A, SEC2.B | ✅ Tamper attempts logged with correlationId/PII tags; ✅ SOC runbook updated; ✅ Threat model status reviewed. |
|
||||
| SEC5.D | TODO | Security Guild | Enforce bootstrap invite expiration and audit unused invites. | SEC5.A | ✅ Bootstrap tokens auto-expire; ✅ Audit entries emitted for expiration/reuse attempts; ✅ Operator docs updated. |
|
||||
| SEC5.E | TODO | Security Guild + Zastava | Detect stolen agent token replay via device binding heuristics. | SEC4.A | ✅ Device binding guidance published; ✅ Alerting pipeline raises stale revocation acknowledgements; ✅ Tests cover replay detection. |
|
||||
| SEC5.F | TODO | Security Guild + DevOps | Warn when plug-in password policy overrides weaken host defaults. | SEC1.A, PLG3 | ✅ Static analyser flags weaker overrides; ✅ Runtime warning surfaced; ✅ Docs call out mitigation. |
|
||||
| SEC5.G | TODO | Security Guild + Ops | Extend Offline Kit with attested manifest and verification CLI sample. | OPS3 | ✅ Offline Kit build signs manifest with detached JWS; ✅ Verification CLI documented; ✅ Supply-chain attestation recorded. |
|
||||
| SEC5.H | TODO | Security Guild + Authority Core | Ensure `/token` denials persist audit records with correlation IDs. | SEC2.A, SEC2.B | ✅ Audit store captures denials; ✅ Tests cover success/failure/lockout; ✅ Threat model review updated. |
|
||||
| SEC5.B | DONE (2025-10-14) | Security Guild + Authority Core | Complete libsodium/Core signing integration and ship revocation verification script. | SEC4.A, SEC4.B, SEC4.HOST | ✅ libsodium/Core signing provider wired; ✅ `stellaops auth revoke verify` script published; ✅ Revocation docs updated with verification workflow. |
|
||||
| SEC5.B1 | DONE (2025-10-14) | Security Guild + Authority Core | Introduce `LibsodiumCryptoProvider` implementing ECDSA signing/verification via libsodium, register under feature flag, and validate against existing ES256 fixtures. | SEC5.B | ✅ Provider resolves via `ICryptoProviderRegistry`; ✅ Integration tests cover sign/verify parity with default provider; ✅ Fallback to managed provider documented. |
|
||||
| SEC5.B2 | DONE (2025-10-14) | Security Guild + DevEx/CLI | Extend `stellaops auth revoke verify` to detect provider metadata, reuse registry for verification, and document CLI workflow. | SEC5.B | ✅ CLI uses registry signers for verification; ✅ End-to-end test invokes verify against sample bundle; ✅ docs/11_AUTHORITY.md references CLI procedure. |
|
||||
| SEC5.C | DONE (2025-10-14) | Security Guild + Authority Core | Finalise audit contract coverage for tampered `/token` requests. | SEC2.A, SEC2.B | ✅ Tamper attempts logged with correlationId/PII tags; ✅ SOC runbook updated; ✅ Threat model status reviewed. |
|
||||
| SEC5.D | DONE (2025-10-14) | Security Guild | Enforce bootstrap invite expiration and audit unused invites. | SEC5.A | ✅ Bootstrap tokens auto-expire; ✅ Audit entries emitted for expiration/reuse attempts; ✅ Operator docs updated. |
|
||||
> Remark (2025-10-14): Cleanup service wired to store; background sweep + invite audit tests added.
|
||||
| SEC5.E | DONE (2025-10-14) | Security Guild + Zastava | Detect stolen agent token replay via device binding heuristics. | SEC4.A | ✅ Device binding guidance published; ✅ Alerting pipeline raises stale revocation acknowledgements; ✅ Tests cover replay detection. |
|
||||
> Remark (2025-10-14): Token usage metadata persisted with replay audits + handler/unit coverage.
|
||||
| SEC5.F | DONE (2025-10-14) | Security Guild + DevOps | Warn when plug-in password policy overrides weaken host defaults. | SEC1.A, PLG3 | ✅ Static analyser flags weaker overrides; ✅ Runtime warning surfaced; ✅ Docs call out mitigation. |
|
||||
> Remark (2025-10-14): Analyzer surfaces warnings during CLI load; docs updated with mitigation steps.
|
||||
| SEC5.G | DONE (2025-10-14) | Security Guild + Ops | Extend Offline Kit with attested manifest and verification CLI sample. | OPS3 | ✅ Offline Kit build signs manifest with detached JWS; ✅ Verification CLI documented; ✅ Supply-chain attestation recorded. |
|
||||
> Remark (2025-10-14): Offline kit docs include manifest verification workflow; attestation artifacts referenced.
|
||||
| SEC5.H | DONE (2025-10-13) | Security Guild + Authority Core | Ensure `/token` denials persist audit records with correlation IDs. | SEC2.A, SEC2.B | ✅ Audit store captures denials; ✅ Tests cover success/failure/lockout; ✅ Threat model review updated. |
|
||||
| D5.A | DONE (2025-10-12) | Security Guild | Flesh out `StellaOps.Cryptography` provider registry, policy, and DI helpers enabling sovereign crypto selection. | SEC1.A, SEC4.B | ✅ `ICryptoProviderRegistry` implementation with provider selection rules; ✅ `StellaOps.Cryptography.DependencyInjection` extensions; ✅ Tests covering fallback ordering. |
|
||||
|
||||
> Remark (2025-10-13, SEC2.B): Coordinated with Authority Core — audit sinks now receive `/token` success/failure events; awaiting host test suite once signing fixture lands.
|
||||
>
|
||||
> Remark (2025-10-13, SEC3.B): Pinged Docs & Plugin guilds — rate limit guidance published in `docs/security/rate-limits.md` and flagged for PLG6.DOC copy lift.
|
||||
>
|
||||
> Remark (2025-10-13, SEC5.B): Split follow-up into SEC5.B1 (libsodium provider) and SEC5.B2 (CLI verification) after scoping registry integration; work not yet started.
|
||||
|
||||
## Notes
|
||||
- Target Argon2 parameters follow OWASP Cheat Sheet (memory ≈ 19 MiB, iterations 2, parallelism 1). Allow overrides via configuration.
|
||||
- When CORE8 lands, pair with Team 2 to expose request context information required by the rate limiter (client_id enrichment).
|
||||
|
||||
@@ -32,7 +32,7 @@ Until these blocks land, connectors should stage changes behind a feature flag o
|
||||
| Ru.Bdu | BE-Conn-BDU | All tasks TODO | Map product releases into normalized rules; add provenance notes referencing BDU advisory identifiers. | Verify we have UTF-8 safe handling in builder; share sample sanitized inputs. |
|
||||
| Ru.Nkcki | BE-Conn-Nkcki | All tasks TODO | Similar to BDU; capture vendor firmware/build numbers and map into normalized rules. | Coordinate with Localization WG for Cyrillic transliteration strategy. |
|
||||
| Vndr.Apple | BE-Conn-Apple | Mapper/tests/telemetry marked DOING | Continue extending vendor range primitives (`apple.version`, `apple.build`) and adopt normalized rule arrays for OS build spans. | Request builder integration review on 2025-10-16; ensure fixtures cover multi-range tables and include provenance notes. |
|
||||
| Vndr.Cisco | BE-Conn-Cisco | All tasks TODO | When parser lands, normalise IOS/ASA version strings into SemVer-style or vendor-specific ranges and supply normalized arrays. | Identify whether ranges require custom comparer (maybe `ios.semver` style); escalate to Models if new scheme required. |
|
||||
| Vndr.Cisco | BE-Conn-Cisco | ✅ Emits SemVer primitives with vendor notes | Parser maps versions into SemVer primitives with `cisco.productId` vendor extensions; sample fixtures landing in `StellaOps.Feedser.Source.Vndr.Cisco.Tests`. | No custom comparer required; SemVer + vendor metadata suffices. |
|
||||
| Vndr.Msrc | BE-Conn-MSRC | All tasks TODO | Canonical mapper must output product/build coverage as normalized rules (likely `msrc.patch` scheme) with provenance referencing KB IDs. | Sync with Models on adding scheme identifiers for MSRC packages; plan fixture coverage for monthly rollups. |
|
||||
|
||||
## Storage alignment quick reference (2025-10-11)
|
||||
|
||||
163
src/StellaOps.Feedser.Source.Cccs.Tests/CccsConnectorTests.cs
Normal file
163
src/StellaOps.Feedser.Source.Cccs.Tests/CccsConnectorTests.cs
Normal file
@@ -0,0 +1,163 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Feedser.Source.Cccs;
|
||||
using StellaOps.Feedser.Source.Cccs.Configuration;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Source.Common.Http;
|
||||
using StellaOps.Feedser.Source.Common.Testing;
|
||||
using StellaOps.Feedser.Storage.Mongo;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using StellaOps.Feedser.Testing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Tests;
|
||||
|
||||
[Collection("mongo-fixture")]
|
||||
public sealed class CccsConnectorTests : IAsyncLifetime
|
||||
{
|
||||
private static readonly Uri FeedUri = new("https://test.local/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat");
|
||||
private static readonly Uri TaxonomyUri = new("https://test.local/api/cccs/taxonomy/v1/get?lang=en&vocabulary=cccs_alert_type");
|
||||
|
||||
private readonly MongoIntegrationFixture _fixture;
|
||||
private readonly CannedHttpMessageHandler _handler;
|
||||
|
||||
public CccsConnectorTests(MongoIntegrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
_handler = new CannedHttpMessageHandler();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchParseMap_ProducesCanonicalAdvisory()
|
||||
{
|
||||
await using var provider = await BuildServiceProviderAsync();
|
||||
SeedFeedResponses();
|
||||
|
||||
var connector = provider.GetRequiredService<CccsConnector>();
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
|
||||
var advisories = await advisoryStore.GetRecentAsync(10, CancellationToken.None);
|
||||
advisories.Should().HaveCount(1);
|
||||
|
||||
var advisory = advisories[0];
|
||||
advisory.AdvisoryKey.Should().Be("TEST-001");
|
||||
advisory.Title.Should().Be("Test Advisory Title");
|
||||
advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" });
|
||||
advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details");
|
||||
advisory.References.Should().Contain(reference => reference.Url == "https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en");
|
||||
advisory.AffectedPackages.Should().ContainSingle(pkg => pkg.Identifier == "Vendor Widget 1.0");
|
||||
advisory.AffectedPackages.Should().Contain(pkg => pkg.Identifier == "Vendor Widget 2.0");
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CccsConnectorPlugin.SourceName, CancellationToken.None);
|
||||
state.Should().NotBeNull();
|
||||
state!.Cursor.Should().NotBeNull();
|
||||
state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue();
|
||||
pendingDocs!.AsBsonArray.Should().BeEmpty();
|
||||
state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue();
|
||||
pendingMappings!.AsBsonArray.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Fetch_PersistsRawDocumentWithMetadata()
|
||||
{
|
||||
await using var provider = await BuildServiceProviderAsync();
|
||||
SeedFeedResponses();
|
||||
|
||||
var connector = provider.GetRequiredService<CccsConnector>();
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var document = await documentStore.FindBySourceAndUriAsync(CccsConnectorPlugin.SourceName, "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory", CancellationToken.None);
|
||||
document.Should().NotBeNull();
|
||||
document!.Status.Should().Be(DocumentStatuses.PendingParse);
|
||||
document.Metadata.Should().ContainKey("cccs.language").WhoseValue.Should().Be("en");
|
||||
document.Metadata.Should().ContainKey("cccs.serialNumber").WhoseValue.Should().Be("TEST-001");
|
||||
document.ContentType.Should().Be("application/json");
|
||||
}
|
||||
|
||||
private async Task<ServiceProvider> BuildServiceProviderAsync()
|
||||
{
|
||||
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
|
||||
_handler.Clear();
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
|
||||
services.AddSingleton(_handler);
|
||||
|
||||
services.AddMongoStorage(options =>
|
||||
{
|
||||
options.ConnectionString = _fixture.Runner.ConnectionString;
|
||||
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
});
|
||||
|
||||
services.AddSourceCommon();
|
||||
services.AddCccsConnector(options =>
|
||||
{
|
||||
options.Feeds.Clear();
|
||||
options.Feeds.Add(new CccsFeedEndpoint("en", FeedUri));
|
||||
options.RequestDelay = TimeSpan.Zero;
|
||||
options.MaxEntriesPerFetch = 10;
|
||||
options.MaxKnownEntries = 32;
|
||||
});
|
||||
|
||||
services.Configure<HttpClientFactoryOptions>(CccsOptions.HttpClientName, builderOptions =>
|
||||
{
|
||||
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
|
||||
{
|
||||
builder.PrimaryHandler = _handler;
|
||||
});
|
||||
});
|
||||
|
||||
var provider = services.BuildServiceProvider();
|
||||
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
|
||||
await bootstrapper.InitializeAsync(CancellationToken.None);
|
||||
return provider;
|
||||
}
|
||||
|
||||
private void SeedFeedResponses()
|
||||
{
|
||||
AddJsonResponse(FeedUri, ReadFixture("cccs-feed-en.json"));
|
||||
AddJsonResponse(TaxonomyUri, ReadFixture("cccs-taxonomy-en.json"));
|
||||
}
|
||||
|
||||
private void AddJsonResponse(Uri uri, string json, string? etag = null)
|
||||
{
|
||||
_handler.AddResponse(uri, () =>
|
||||
{
|
||||
var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
if (!string.IsNullOrWhiteSpace(etag))
|
||||
{
|
||||
response.Headers.ETag = new EntityTagHeaderValue(etag);
|
||||
}
|
||||
|
||||
return response;
|
||||
});
|
||||
}
|
||||
|
||||
private static string ReadFixture(string fileName)
|
||||
=> System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName));
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"ERROR": false,
|
||||
"response": [
|
||||
{
|
||||
"nid": 1001,
|
||||
"title": "Test Advisory Title",
|
||||
"uuid": "uuid-test-001",
|
||||
"banner": null,
|
||||
"lang": "en",
|
||||
"date_modified": "2025-08-11",
|
||||
"date_modified_ts": "2025-08-11T12:00:00Z",
|
||||
"date_created": "2025-08-10T15:30:00Z",
|
||||
"summary": "Summary of advisory.",
|
||||
"body": [
|
||||
"<article><p><strong>Number: TEST-001<br/>Date: 14 April 2018</strong></p><h2>Affected Products</h2><ul><li>Vendor Widget 1.0</li><li>Vendor Widget 2.0</li></ul><p>See <a href=\"https://example.com/details?utm_source=rss&utm_medium=email\">Details Link</a>.</p><p>Internal link <a href=\"/en/contact-cyber-centre?utm_campaign=newsletter\">Contact</a>.</p><p>Mitigation for CVE-2020-1234 and CVE-2021-9999.</p></article>"
|
||||
],
|
||||
"url": "/en/alerts-advisories/test-advisory",
|
||||
"alert_type": 397,
|
||||
"serial_number": "TEST-001",
|
||||
"subject": "Infrastructure",
|
||||
"moderation_state": "published",
|
||||
"external_url": "https://example.com/external/advisory"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"sourceId": "TEST-002-FR",
|
||||
"serialNumber": "TEST-002-FR",
|
||||
"uuid": "uuid-test-002",
|
||||
"language": "fr",
|
||||
"title": "Avis de sécurité – Mise à jour urgente",
|
||||
"summary": "Résumé de l'avis en français.",
|
||||
"canonicalUrl": "https://www.cyber.gc.ca/fr/alertes-avis/test-avis",
|
||||
"externalUrl": "https://exemple.ca/avis",
|
||||
"bodyHtml": "<article><p><strong>Numéro : TEST-002-FR<br/>Date : 15 août 2025</strong></p><h2>Produits touchés</h2><div class=\"product-list\"><ul><li>Produit Exemple 3.1</li><li>Produit Exemple 3.2<ul><li>Variante 3.2.1</li></ul></li></ul></div><p>Voir <a href=\"https://exemple.ca/details?utm_campaign=mailing\">Lien de détails</a>.</p><p>Lien interne <a href=\"/fr/contact-centre-cyber\">Contactez-nous</a>.</p><p>Correctifs pour CVE-2024-1111.</p></article>",
|
||||
"bodySegments": [
|
||||
"<article><p><strong>Numéro : TEST-002-FR<br/>Date : 15 août 2025</strong></p><h2>Produits touchés</h2><div class=\"product-list\"><ul><li>Produit Exemple 3.1</li><li>Produit Exemple 3.2<ul><li>Variante 3.2.1</li></ul></li></ul></div><p>Voir <a href=\"https://exemple.ca/details?utm_campaign=mailing\">Lien de détails</a>.</p><p>Lien interne <a href=\"/fr/contact-centre-cyber\">Contactez-nous</a>.</p><p>Correctifs pour CVE-2024-1111.</p></article>"
|
||||
],
|
||||
"alertType": "Alerte",
|
||||
"subject": "Infrastructure critique",
|
||||
"banner": null,
|
||||
"published": "2025-08-15T13:45:00Z",
|
||||
"modified": "2025-08-16T09:15:00Z",
|
||||
"rawCreated": "15 août 2025",
|
||||
"rawModified": "2025-08-16T09:15:00Z"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"sourceId": "TEST-001",
|
||||
"serialNumber": "TEST-001",
|
||||
"uuid": "uuid-test-001",
|
||||
"language": "en",
|
||||
"title": "Test Advisory Title",
|
||||
"summary": "Summary of advisory.",
|
||||
"canonicalUrl": "https://www.cyber.gc.ca/en/alerts-advisories/test-advisory",
|
||||
"externalUrl": "https://example.com/external/advisory",
|
||||
"bodyHtml": "<article><p><strong>Number: TEST-001<br/>Date: 14 April 2018</strong></p><h2>Affected Products</h2><ul><li>Vendor Widget 1.0</li><li>Vendor Widget 2.0</li></ul><p>See <a href=\"https://example.com/details?utm_source=rss&utm_medium=email\">Details Link</a>.</p><p>Internal link <a href=\"/en/contact-cyber-centre?utm_campaign=newsletter\">Contact</a>.</p><p>Mitigation for CVE-2020-1234 and CVE-2021-9999.</p></article>",
|
||||
"bodySegments": [
|
||||
"<article><p><strong>Number: TEST-001<br/>Date: 14 April 2018</strong></p><h2>Affected Products</h2><ul><li>Vendor Widget 1.0</li><li>Vendor Widget 2.0</li></ul><p>See <a href=\"https://example.com/details?utm_source=rss&utm_medium=email\">Details Link</a>.</p><p>Internal link <a href=\"/en/contact-cyber-centre?utm_campaign=newsletter\">Contact</a>.</p><p>Mitigation for CVE-2020-1234 and CVE-2021-9999.</p></article>"
|
||||
],
|
||||
"alertType": "Advisory",
|
||||
"subject": "Infrastructure",
|
||||
"banner": null,
|
||||
"published": "2025-08-10T15:30:00Z",
|
||||
"modified": "2025-08-11T12:00:00Z",
|
||||
"rawCreated": "August 10, 2025",
|
||||
"rawModified": "2025-08-11T12:00:00Z"
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
{
|
||||
"ERROR": false,
|
||||
"response": [
|
||||
{
|
||||
"id": 396,
|
||||
"title": "Advisory"
|
||||
},
|
||||
{
|
||||
"id": 397,
|
||||
"title": "Alert"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Feedser.Source.Cccs.Internal;
|
||||
using StellaOps.Feedser.Source.Common.Html;
|
||||
using Xunit;
|
||||
using Xunit.Abstractions;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Tests.Internal;
|
||||
|
||||
public sealed class CccsHtmlParserTests
|
||||
{
|
||||
private readonly ITestOutputHelper _output;
|
||||
private static readonly HtmlContentSanitizer Sanitizer = new();
|
||||
private static readonly CccsHtmlParser Parser = new(Sanitizer);
|
||||
|
||||
public CccsHtmlParserTests(ITestOutputHelper output)
|
||||
{
|
||||
_output = output ?? throw new ArgumentNullException(nameof(output));
|
||||
}
|
||||
|
||||
public static IEnumerable<object[]> ParserCases()
|
||||
{
|
||||
yield return new object[]
|
||||
{
|
||||
"cccs-raw-advisory.json",
|
||||
"TEST-001",
|
||||
"en",
|
||||
new[] { "Vendor Widget 1.0", "Vendor Widget 2.0" },
|
||||
new[]
|
||||
{
|
||||
"https://example.com/details",
|
||||
"https://www.cyber.gc.ca/en/contact-cyber-centre?lang=en"
|
||||
},
|
||||
new[] { "CVE-2020-1234", "CVE-2021-9999" }
|
||||
};
|
||||
|
||||
yield return new object[]
|
||||
{
|
||||
"cccs-raw-advisory-fr.json",
|
||||
"TEST-002-FR",
|
||||
"fr",
|
||||
new[] { "Produit Exemple 3.1", "Produit Exemple 3.2", "Variante 3.2.1" },
|
||||
new[]
|
||||
{
|
||||
"https://exemple.ca/details",
|
||||
"https://www.cyber.gc.ca/fr/contact-centre-cyber"
|
||||
},
|
||||
new[] { "CVE-2024-1111" }
|
||||
};
|
||||
}
|
||||
|
||||
[Theory]
|
||||
[MemberData(nameof(ParserCases))]
|
||||
public void Parse_ExtractsExpectedFields(
|
||||
string fixtureName,
|
||||
string expectedSerial,
|
||||
string expectedLanguage,
|
||||
string[] expectedProducts,
|
||||
string[] expectedReferenceUrls,
|
||||
string[] expectedCves)
|
||||
{
|
||||
var raw = LoadFixture<CccsRawAdvisoryDocument>(fixtureName);
|
||||
|
||||
var dto = Parser.Parse(raw);
|
||||
|
||||
_output.WriteLine("Products: {0}", string.Join("|", dto.Products));
|
||||
_output.WriteLine("References: {0}", string.Join("|", dto.References.Select(r => $"{r.Url} ({r.Label})")));
|
||||
_output.WriteLine("CVEs: {0}", string.Join("|", dto.CveIds));
|
||||
|
||||
dto.SerialNumber.Should().Be(expectedSerial);
|
||||
dto.Language.Should().Be(expectedLanguage);
|
||||
dto.Products.Should().BeEquivalentTo(expectedProducts);
|
||||
foreach (var url in expectedReferenceUrls)
|
||||
{
|
||||
dto.References.Should().Contain(reference => reference.Url == url);
|
||||
}
|
||||
|
||||
dto.CveIds.Should().BeEquivalentTo(expectedCves);
|
||||
dto.ContentHtml.Should().Contain("<ul>").And.Contain("<li>");
|
||||
dto.ContentHtml.Should().Contain("<h2", because: "heading structure must survive sanitisation for UI rendering");
|
||||
}
|
||||
|
||||
internal static T LoadFixture<T>(string fileName)
|
||||
{
|
||||
var path = Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName);
|
||||
var json = File.ReadAllText(path);
|
||||
return JsonSerializer.Deserialize<T>(json, new JsonSerializerOptions(JsonSerializerDefaults.Web))!;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,43 @@
|
||||
using System;
|
||||
using FluentAssertions;
|
||||
using StellaOps.Feedser.Source.Cccs.Internal;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Source.Common.Html;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Tests.Internal;
|
||||
|
||||
public sealed class CccsMapperTests
|
||||
{
|
||||
[Fact]
|
||||
public void Map_CreatesCanonicalAdvisory()
|
||||
{
|
||||
var raw = CccsHtmlParserTests.LoadFixture<CccsRawAdvisoryDocument>("cccs-raw-advisory.json");
|
||||
var dto = new CccsHtmlParser(new HtmlContentSanitizer()).Parse(raw);
|
||||
var document = new DocumentRecord(
|
||||
Guid.NewGuid(),
|
||||
CccsConnectorPlugin.SourceName,
|
||||
dto.CanonicalUrl,
|
||||
DateTimeOffset.UtcNow,
|
||||
"sha-test",
|
||||
DocumentStatuses.PendingMap,
|
||||
"application/json",
|
||||
Headers: null,
|
||||
Metadata: null,
|
||||
Etag: null,
|
||||
LastModified: dto.Modified,
|
||||
GridFsId: null);
|
||||
|
||||
var recordedAt = DateTimeOffset.Parse("2025-08-12T00:00:00Z");
|
||||
var advisory = CccsMapper.Map(dto, document, recordedAt);
|
||||
|
||||
advisory.AdvisoryKey.Should().Be("TEST-001");
|
||||
advisory.Title.Should().Be(dto.Title);
|
||||
advisory.Aliases.Should().Contain(new[] { "TEST-001", "CVE-2020-1234", "CVE-2021-9999" });
|
||||
advisory.References.Should().Contain(reference => reference.Url == dto.CanonicalUrl && reference.Kind == "details");
|
||||
advisory.References.Should().Contain(reference => reference.Url == "https://example.com/details");
|
||||
advisory.AffectedPackages.Should().HaveCount(2);
|
||||
advisory.Provenance.Should().ContainSingle(p => p.Source == CccsConnectorPlugin.SourceName && p.Kind == "advisory");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Cccs/StellaOps.Feedser.Source.Cccs.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="Fixtures\*.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
606
src/StellaOps.Feedser.Source.Cccs/CccsConnector.cs
Normal file
606
src/StellaOps.Feedser.Source.Cccs/CccsConnector.cs
Normal file
@@ -0,0 +1,606 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Globalization;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Feedser.Source.Cccs.Configuration;
|
||||
using StellaOps.Feedser.Source.Cccs.Internal;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Source.Common.Fetch;
|
||||
using StellaOps.Feedser.Storage.Mongo;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using StellaOps.Feedser.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs;
|
||||
|
||||
public sealed class CccsConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions RawSerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private static readonly JsonSerializerOptions DtoSerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private const string DtoSchemaVersion = "cccs.dto.v1";
|
||||
|
||||
private readonly CccsFeedClient _feedClient;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly CccsHtmlParser _htmlParser;
|
||||
private readonly CccsDiagnostics _diagnostics;
|
||||
private readonly CccsOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly ILogger<CccsConnector> _logger;
|
||||
|
||||
public CccsConnector(
|
||||
CccsFeedClient feedClient,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
CccsHtmlParser htmlParser,
|
||||
CccsDiagnostics diagnostics,
|
||||
IOptions<CccsOptions> options,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<CccsConnector> logger)
|
||||
{
|
||||
_feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_htmlParser = htmlParser ?? throw new ArgumentNullException(nameof(htmlParser));
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => CccsConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var pendingDocuments = new HashSet<Guid>(cursor.PendingDocuments);
|
||||
var pendingMappings = new HashSet<Guid>(cursor.PendingMappings);
|
||||
var knownHashes = new Dictionary<string, string>(cursor.KnownEntryHashes, StringComparer.Ordinal);
|
||||
var feedsProcessed = 0;
|
||||
var totalItems = 0;
|
||||
var added = 0;
|
||||
var unchanged = 0;
|
||||
|
||||
try
|
||||
{
|
||||
foreach (var feed in _options.Feeds)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
_diagnostics.FetchAttempt();
|
||||
var result = await _feedClient.FetchAsync(feed, _options.RequestTimeout, cancellationToken).ConfigureAwait(false);
|
||||
feedsProcessed++;
|
||||
totalItems += result.Items.Count;
|
||||
|
||||
if (result.Items.Count == 0)
|
||||
{
|
||||
_diagnostics.FetchSuccess();
|
||||
await DelayBetweenRequestsAsync(cancellationToken).ConfigureAwait(false);
|
||||
continue;
|
||||
}
|
||||
|
||||
var items = result.Items
|
||||
.Where(static item => !string.IsNullOrWhiteSpace(item.Title))
|
||||
.OrderByDescending(item => ParseDate(item.DateModifiedTimestamp) ?? ParseDate(item.DateModified) ?? DateTimeOffset.MinValue)
|
||||
.ThenByDescending(item => ParseDate(item.DateCreated) ?? DateTimeOffset.MinValue)
|
||||
.ToList();
|
||||
|
||||
foreach (var item in items)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var documentUri = BuildDocumentUri(item, feed);
|
||||
var rawDocument = CreateRawDocument(item, feed, result.AlertTypes);
|
||||
var payload = JsonSerializer.SerializeToUtf8Bytes(rawDocument, RawSerializerOptions);
|
||||
var sha = ComputeSha256(payload);
|
||||
|
||||
if (knownHashes.TryGetValue(documentUri, out var existingHash)
|
||||
&& string.Equals(existingHash, sha, StringComparison.Ordinal))
|
||||
{
|
||||
unchanged++;
|
||||
_diagnostics.FetchUnchanged();
|
||||
continue;
|
||||
}
|
||||
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, documentUri, cancellationToken).ConfigureAwait(false);
|
||||
if (existing is not null
|
||||
&& string.Equals(existing.Sha256, sha, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(existing.Status, DocumentStatuses.Mapped, StringComparison.Ordinal))
|
||||
{
|
||||
knownHashes[documentUri] = sha;
|
||||
unchanged++;
|
||||
_diagnostics.FetchUnchanged();
|
||||
continue;
|
||||
}
|
||||
|
||||
var gridFsId = await _rawDocumentStorage.UploadAsync(
|
||||
SourceName,
|
||||
documentUri,
|
||||
payload,
|
||||
"application/json",
|
||||
expiresAt: null,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["cccs.language"] = rawDocument.Language,
|
||||
["cccs.sourceId"] = rawDocument.SourceId,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(rawDocument.SerialNumber))
|
||||
{
|
||||
metadata["cccs.serialNumber"] = rawDocument.SerialNumber!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(rawDocument.AlertType))
|
||||
{
|
||||
metadata["cccs.alertType"] = rawDocument.AlertType!;
|
||||
}
|
||||
|
||||
var recordId = existing?.Id ?? Guid.NewGuid();
|
||||
var record = new DocumentRecord(
|
||||
recordId,
|
||||
SourceName,
|
||||
documentUri,
|
||||
now,
|
||||
sha,
|
||||
DocumentStatuses.PendingParse,
|
||||
"application/json",
|
||||
Headers: null,
|
||||
Metadata: metadata,
|
||||
Etag: null,
|
||||
LastModified: rawDocument.Modified ?? rawDocument.Published ?? result.LastModifiedUtc,
|
||||
GridFsId: gridFsId,
|
||||
ExpiresAt: null);
|
||||
|
||||
var upserted = await _documentStore.UpsertAsync(record, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Add(upserted.Id);
|
||||
pendingMappings.Remove(upserted.Id);
|
||||
knownHashes[documentUri] = sha;
|
||||
added++;
|
||||
_diagnostics.FetchDocument();
|
||||
|
||||
if (added >= _options.MaxEntriesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
_diagnostics.FetchSuccess();
|
||||
await DelayBetweenRequestsAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (added >= _options.MaxEntriesPerFetch)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException or JsonException or InvalidOperationException)
|
||||
{
|
||||
_diagnostics.FetchFailure();
|
||||
_logger.LogError(ex, "CCCS fetch failed");
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
var trimmedHashes = TrimKnownHashes(knownHashes, _options.MaxKnownEntries);
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownEntryHashes(trimmedHashes)
|
||||
.WithLastFetch(now);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
_logger.LogInformation(
|
||||
"CCCS fetch completed feeds={Feeds} items={Items} newDocuments={Added} unchanged={Unchanged} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}",
|
||||
feedsProcessed,
|
||||
totalItems,
|
||||
added,
|
||||
unchanged,
|
||||
pendingDocuments.Count,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToList();
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var parsed = 0;
|
||||
var parseFailures = 0;
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.ParseFailure();
|
||||
parseFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
_logger.LogWarning("CCCS document {DocumentId} missing GridFS payload", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
parseFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] payload;
|
||||
try
|
||||
{
|
||||
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
_logger.LogError(ex, "CCCS unable to download raw document {DocumentId}", documentId);
|
||||
throw;
|
||||
}
|
||||
|
||||
CccsRawAdvisoryDocument? raw;
|
||||
try
|
||||
{
|
||||
raw = JsonSerializer.Deserialize<CccsRawAdvisoryDocument>(payload, RawSerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
_logger.LogWarning(ex, "CCCS failed to deserialize raw document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
parseFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (raw is null)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
_logger.LogWarning("CCCS raw document {DocumentId} produced null payload", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
parseFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
CccsAdvisoryDto dto;
|
||||
try
|
||||
{
|
||||
dto = _htmlParser.Parse(raw);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.ParseFailure();
|
||||
_logger.LogWarning(ex, "CCCS failed to parse advisory DTO for {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
parseFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoJson = JsonSerializer.Serialize(dto, DtoSerializerOptions);
|
||||
var dtoBson = BsonDocument.Parse(dtoJson);
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, DtoSchemaVersion, dtoBson, now);
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
pendingDocuments.Remove(documentId);
|
||||
if (!pendingMappings.Contains(documentId))
|
||||
{
|
||||
pendingMappings.Add(documentId);
|
||||
}
|
||||
_diagnostics.ParseSuccess();
|
||||
parsed++;
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
if (parsed > 0 || parseFailures > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"CCCS parse completed parsed={Parsed} failures={Failures} pendingDocuments={PendingDocuments} pendingMappings={PendingMappings}",
|
||||
parsed,
|
||||
parseFailures,
|
||||
pendingDocuments.Count,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToList();
|
||||
var mapped = 0;
|
||||
var mappingFailures = 0;
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapFailure();
|
||||
mappingFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null)
|
||||
{
|
||||
_diagnostics.MapFailure();
|
||||
_logger.LogWarning("CCCS document {DocumentId} missing DTO payload", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
mappingFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
CccsAdvisoryDto? dto;
|
||||
try
|
||||
{
|
||||
var json = dtoRecord.Payload.ToJson();
|
||||
dto = JsonSerializer.Deserialize<CccsAdvisoryDto>(json, DtoSerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.MapFailure();
|
||||
_logger.LogWarning(ex, "CCCS failed to deserialize DTO for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
mappingFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dto is null)
|
||||
{
|
||||
_diagnostics.MapFailure();
|
||||
_logger.LogWarning("CCCS DTO for document {DocumentId} evaluated to null", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
mappingFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var advisory = CccsMapper.Map(dto, document, dtoRecord.ValidatedAt);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapSuccess();
|
||||
mapped++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_diagnostics.MapFailure();
|
||||
_logger.LogError(ex, "CCCS mapping failed for document {DocumentId}", documentId);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
mappingFailures++;
|
||||
}
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
if (mapped > 0 || mappingFailures > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"CCCS map completed mapped={Mapped} failures={Failures} pendingMappings={PendingMappings}",
|
||||
mapped,
|
||||
mappingFailures,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<CccsCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? CccsCursor.Empty : CccsCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private Task UpdateCursorAsync(CccsCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
|
||||
private async Task DelayBetweenRequestsAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
if (_options.RequestDelay <= TimeSpan.Zero)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (TaskCanceledException)
|
||||
{
|
||||
// Ignore cancellation during delay; caller handles.
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildDocumentUri(CccsFeedItem item, CccsFeedEndpoint feed)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(item.Url))
|
||||
{
|
||||
if (Uri.TryCreate(item.Url, UriKind.Absolute, out var absolute))
|
||||
{
|
||||
return absolute.ToString();
|
||||
}
|
||||
|
||||
var baseUri = new Uri("https://www.cyber.gc.ca", UriKind.Absolute);
|
||||
if (Uri.TryCreate(baseUri, item.Url, out var combined))
|
||||
{
|
||||
return combined.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
return $"https://www.cyber.gc.ca/api/cccs/threats/{feed.Language}/{item.Nid}";
|
||||
}
|
||||
|
||||
private static CccsRawAdvisoryDocument CreateRawDocument(CccsFeedItem item, CccsFeedEndpoint feed, IReadOnlyDictionary<int, string> taxonomy)
|
||||
{
|
||||
var language = string.IsNullOrWhiteSpace(item.Language) ? feed.Language : item.Language!.Trim();
|
||||
var identifier = !string.IsNullOrWhiteSpace(item.SerialNumber)
|
||||
? item.SerialNumber!.Trim()
|
||||
: !string.IsNullOrWhiteSpace(item.Uuid)
|
||||
? item.Uuid!.Trim()
|
||||
: $"nid-{item.Nid}";
|
||||
|
||||
var canonicalUrl = BuildDocumentUri(item, feed);
|
||||
var bodySegments = item.Body ?? Array.Empty<string>();
|
||||
var bodyHtml = string.Join(Environment.NewLine, bodySegments);
|
||||
var published = ParseDate(item.DateCreated);
|
||||
var modified = ParseDate(item.DateModifiedTimestamp) ?? ParseDate(item.DateModified);
|
||||
var alertType = ResolveAlertType(item, taxonomy);
|
||||
|
||||
return new CccsRawAdvisoryDocument
|
||||
{
|
||||
SourceId = identifier,
|
||||
SerialNumber = item.SerialNumber?.Trim(),
|
||||
Uuid = item.Uuid,
|
||||
Language = language.ToLowerInvariant(),
|
||||
Title = item.Title?.Trim() ?? identifier,
|
||||
Summary = item.Summary?.Trim(),
|
||||
CanonicalUrl = canonicalUrl,
|
||||
ExternalUrl = item.ExternalUrl,
|
||||
BodyHtml = bodyHtml,
|
||||
BodySegments = bodySegments,
|
||||
AlertType = alertType,
|
||||
Subject = item.Subject,
|
||||
Banner = item.Banner,
|
||||
Published = published,
|
||||
Modified = modified,
|
||||
RawDateCreated = item.DateCreated,
|
||||
RawDateModified = item.DateModifiedTimestamp ?? item.DateModified,
|
||||
};
|
||||
}
|
||||
|
||||
private static string? ResolveAlertType(CccsFeedItem item, IReadOnlyDictionary<int, string> taxonomy)
|
||||
{
|
||||
if (item.AlertType.ValueKind == JsonValueKind.Number)
|
||||
{
|
||||
var id = item.AlertType.GetInt32();
|
||||
return taxonomy.TryGetValue(id, out var label) ? label : id.ToString(CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
if (item.AlertType.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
return item.AlertType.GetString();
|
||||
}
|
||||
|
||||
if (item.AlertType.ValueKind == JsonValueKind.Array)
|
||||
{
|
||||
foreach (var element in item.AlertType.EnumerateArray())
|
||||
{
|
||||
if (element.ValueKind == JsonValueKind.Number)
|
||||
{
|
||||
var id = element.GetInt32();
|
||||
if (taxonomy.TryGetValue(id, out var label))
|
||||
{
|
||||
return label;
|
||||
}
|
||||
}
|
||||
else if (element.ValueKind == JsonValueKind.String)
|
||||
{
|
||||
var label = element.GetString();
|
||||
if (!string.IsNullOrWhiteSpace(label))
|
||||
{
|
||||
return label;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Dictionary<string, string> TrimKnownHashes(Dictionary<string, string> hashes, int maxEntries)
|
||||
{
|
||||
if (hashes.Count <= maxEntries)
|
||||
{
|
||||
return hashes;
|
||||
}
|
||||
|
||||
var overflow = hashes.Count - maxEntries;
|
||||
foreach (var key in hashes.Keys.Take(overflow).ToList())
|
||||
{
|
||||
hashes.Remove(key);
|
||||
}
|
||||
|
||||
return hashes;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(string? value)
|
||||
=> string.IsNullOrWhiteSpace(value)
|
||||
? null
|
||||
: DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
|
||||
? parsed
|
||||
: null;
|
||||
|
||||
private static string ComputeSha256(byte[] payload)
|
||||
=> Convert.ToHexString(SHA256.HashData(payload)).ToLowerInvariant();
|
||||
}
|
||||
21
src/StellaOps.Feedser.Source.Cccs/CccsConnectorPlugin.cs
Normal file
21
src/StellaOps.Feedser.Source.Cccs/CccsConnectorPlugin.cs
Normal file
@@ -0,0 +1,21 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs;
|
||||
|
||||
public sealed class CccsConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cccs";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<CccsConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<CccsConnector>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
using StellaOps.Feedser.Source.Cccs.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs;
|
||||
|
||||
public sealed class CccsDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "feedser:sources:cccs";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCccsConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CccsFetchJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CccsJobKinds.Fetch, typeof(CccsFetchJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Feedser.Source.Cccs.Configuration;
|
||||
using StellaOps.Feedser.Source.Cccs.Internal;
|
||||
using StellaOps.Feedser.Source.Common.Http;
|
||||
using StellaOps.Feedser.Source.Common.Html;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs;
|
||||
|
||||
public static class CccsServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCccsConnector(this IServiceCollection services, Action<CccsOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CccsOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CccsOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CccsOptions>>().Value;
|
||||
clientOptions.UserAgent = "StellaOps.Feedser.Cccs/1.0";
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
|
||||
foreach (var feed in options.Feeds.Where(static feed => feed.Uri is not null))
|
||||
{
|
||||
clientOptions.AllowedHosts.Add(feed.Uri!.Host);
|
||||
}
|
||||
|
||||
clientOptions.AllowedHosts.Add("www.cyber.gc.ca");
|
||||
clientOptions.AllowedHosts.Add("cyber.gc.ca");
|
||||
});
|
||||
|
||||
services.TryAddSingleton<HtmlContentSanitizer>();
|
||||
services.TryAddSingleton<CccsDiagnostics>();
|
||||
services.TryAddSingleton<CccsHtmlParser>();
|
||||
services.TryAddSingleton<CccsFeedClient>();
|
||||
services.AddTransient<CccsConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs;
|
||||
|
||||
public sealed class CccsConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public string Name => "cccs";
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => true;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name);
|
||||
|
||||
private sealed class StubConnector : IFeedConnector
|
||||
{
|
||||
public StubConnector(string sourceName) => SourceName = sourceName;
|
||||
|
||||
public string SourceName { get; }
|
||||
|
||||
public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
175
src/StellaOps.Feedser.Source.Cccs/Configuration/CccsOptions.cs
Normal file
175
src/StellaOps.Feedser.Source.Cccs/Configuration/CccsOptions.cs
Normal file
@@ -0,0 +1,175 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Configuration;
|
||||
|
||||
public sealed class CccsOptions
|
||||
{
|
||||
public const string HttpClientName = "feedser.source.cccs";
|
||||
|
||||
private readonly List<CccsFeedEndpoint> _feeds = new();
|
||||
|
||||
public CccsOptions()
|
||||
{
|
||||
_feeds.Add(new CccsFeedEndpoint("en", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=en&content_type=cccs_threat")));
|
||||
_feeds.Add(new CccsFeedEndpoint("fr", new Uri("https://www.cyber.gc.ca/api/cccs/threats/v1/get?lang=fr&content_type=cccs_threat")));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Feed endpoints to poll; configure per language or content category.
|
||||
/// </summary>
|
||||
public IList<CccsFeedEndpoint> Feeds => _feeds;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of entries to enqueue per fetch cycle.
|
||||
/// </summary>
|
||||
public int MaxEntriesPerFetch { get; set; } = 80;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum remembered entries (URI+hash) for deduplication.
|
||||
/// </summary>
|
||||
public int MaxKnownEntries { get; set; } = 512;
|
||||
|
||||
/// <summary>
|
||||
/// Timeout applied to feed and taxonomy requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Delay between successive feed requests to respect upstream throttling.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Backoff recorded in source state when fetch fails.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(1);
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (_feeds.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("At least one CCCS feed endpoint must be configured.");
|
||||
}
|
||||
|
||||
var seenLanguages = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var feed in _feeds)
|
||||
{
|
||||
feed.Validate();
|
||||
if (!seenLanguages.Add(feed.Language))
|
||||
{
|
||||
throw new InvalidOperationException($"Duplicate CCCS feed language configured: '{feed.Language}'. Each language should be unique to avoid duplicate ingestion.");
|
||||
}
|
||||
}
|
||||
|
||||
if (MaxEntriesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(MaxEntriesPerFetch)} must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxKnownEntries <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(MaxKnownEntries)} must be greater than zero.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
|
||||
}
|
||||
|
||||
if (FailureBackoff <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class CccsFeedEndpoint
|
||||
{
|
||||
public CccsFeedEndpoint()
|
||||
{
|
||||
}
|
||||
|
||||
public CccsFeedEndpoint(string language, Uri uri)
|
||||
{
|
||||
Language = language;
|
||||
Uri = uri;
|
||||
}
|
||||
|
||||
public string Language { get; set; } = "en";
|
||||
|
||||
public Uri? Uri { get; set; }
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(Language))
|
||||
{
|
||||
throw new InvalidOperationException("CCCS feed language must be specified.");
|
||||
}
|
||||
|
||||
if (Uri is null || !Uri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException($"CCCS feed endpoint URI must be an absolute URI (language='{Language}').");
|
||||
}
|
||||
}
|
||||
|
||||
public Uri BuildTaxonomyUri()
|
||||
{
|
||||
if (Uri is null)
|
||||
{
|
||||
throw new InvalidOperationException("Feed endpoint URI must be configured before building taxonomy URI.");
|
||||
}
|
||||
|
||||
var language = Uri.GetQueryParameterValueOrDefault("lang", Language);
|
||||
var builder = $"https://www.cyber.gc.ca/api/cccs/taxonomy/v1/get?lang={language}&vocabulary=cccs_alert_type";
|
||||
return new Uri(builder, UriKind.Absolute);
|
||||
}
|
||||
}
|
||||
|
||||
internal static class CccsUriExtensions
|
||||
{
|
||||
public static string GetQueryParameterValueOrDefault(this Uri uri, string key, string fallback)
|
||||
{
|
||||
if (uri is null)
|
||||
{
|
||||
return fallback;
|
||||
}
|
||||
|
||||
var query = uri.Query;
|
||||
if (string.IsNullOrEmpty(query))
|
||||
{
|
||||
return fallback;
|
||||
}
|
||||
|
||||
var trimmed = query.StartsWith("?", StringComparison.Ordinal) ? query[1..] : query;
|
||||
foreach (var pair in trimmed.Split(new[] { '&' }, StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var separatorIndex = pair.IndexOf('=');
|
||||
if (separatorIndex < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var left = pair[..separatorIndex].Trim();
|
||||
if (!left.Equals(key, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var right = pair[(separatorIndex + 1)..].Trim();
|
||||
if (right.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
return Uri.UnescapeDataString(right);
|
||||
}
|
||||
|
||||
return fallback;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Internal;
|
||||
|
||||
internal sealed record CccsAdvisoryDto
|
||||
{
|
||||
[JsonPropertyName("sourceId")]
|
||||
public string SourceId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("serialNumber")]
|
||||
public string SerialNumber { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; init; } = "en";
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("canonicalUrl")]
|
||||
public string CanonicalUrl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("contentHtml")]
|
||||
public string ContentHtml { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("alertType")]
|
||||
public string? AlertType { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public string? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("products")]
|
||||
public IReadOnlyList<string> Products { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public IReadOnlyList<CccsReferenceDto> References { get; init; } = Array.Empty<CccsReferenceDto>();
|
||||
|
||||
[JsonPropertyName("cveIds")]
|
||||
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
|
||||
}
|
||||
|
||||
internal sealed record CccsReferenceDto(
|
||||
[property: JsonPropertyName("url")] string Url,
|
||||
[property: JsonPropertyName("label")] string? Label);
|
||||
145
src/StellaOps.Feedser.Source.Cccs/Internal/CccsCursor.cs
Normal file
145
src/StellaOps.Feedser.Source.Cccs/Internal/CccsCursor.cs
Normal file
@@ -0,0 +1,145 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Internal;
|
||||
|
||||
internal sealed record CccsCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyDictionary<string, string> KnownEntryHashes,
|
||||
DateTimeOffset? LastFetchAt)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuidCollection = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyDictionary<string, string> EmptyHashes = new Dictionary<string, string>(StringComparer.Ordinal);
|
||||
|
||||
public static CccsCursor Empty { get; } = new(EmptyGuidCollection, EmptyGuidCollection, EmptyHashes, null);
|
||||
|
||||
public CccsCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
{
|
||||
var distinct = (documents ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
|
||||
return this with { PendingDocuments = distinct };
|
||||
}
|
||||
|
||||
public CccsCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
{
|
||||
var distinct = (mappings ?? Enumerable.Empty<Guid>()).Distinct().ToArray();
|
||||
return this with { PendingMappings = distinct };
|
||||
}
|
||||
|
||||
public CccsCursor WithKnownEntryHashes(IReadOnlyDictionary<string, string> hashes)
|
||||
{
|
||||
var map = hashes is null || hashes.Count == 0
|
||||
? EmptyHashes
|
||||
: new Dictionary<string, string>(hashes, StringComparer.Ordinal);
|
||||
return this with { KnownEntryHashes = map };
|
||||
}
|
||||
|
||||
public CccsCursor WithLastFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastFetchAt = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var doc = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
};
|
||||
|
||||
if (KnownEntryHashes.Count > 0)
|
||||
{
|
||||
var hashes = new BsonArray();
|
||||
foreach (var kvp in KnownEntryHashes)
|
||||
{
|
||||
hashes.Add(new BsonDocument
|
||||
{
|
||||
["uri"] = kvp.Key,
|
||||
["hash"] = kvp.Value,
|
||||
});
|
||||
}
|
||||
|
||||
doc["knownEntryHashes"] = hashes;
|
||||
}
|
||||
|
||||
if (LastFetchAt.HasValue)
|
||||
{
|
||||
doc["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return doc;
|
||||
}
|
||||
|
||||
public static CccsCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var hashes = ReadHashMap(document);
|
||||
var lastFetch = document.TryGetValue("lastFetchAt", out var value)
|
||||
? ParseDateTime(value)
|
||||
: null;
|
||||
|
||||
return new CccsCursor(pendingDocuments, pendingMappings, hashes, lastFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuidCollection;
|
||||
}
|
||||
|
||||
var items = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var guid))
|
||||
{
|
||||
items.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, string> ReadHashMap(BsonDocument document)
|
||||
{
|
||||
if (!document.TryGetValue("knownEntryHashes", out var value) || value is not BsonArray array || array.Count == 0)
|
||||
{
|
||||
return EmptyHashes;
|
||||
}
|
||||
|
||||
var map = new Dictionary<string, string>(array.Count, StringComparer.Ordinal);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (element is not BsonDocument entry)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!entry.TryGetValue("uri", out var uriValue) || uriValue.IsBsonNull || string.IsNullOrWhiteSpace(uriValue.AsString))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var hash = entry.TryGetValue("hash", out var hashValue) && !hashValue.IsBsonNull
|
||||
? hashValue.AsString
|
||||
: string.Empty;
|
||||
map[uriValue.AsString] = hash;
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDateTime(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Internal;
|
||||
|
||||
public sealed class CccsDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Feedser.Source.Cccs";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _fetchAttempts;
|
||||
private readonly Counter<long> _fetchSuccess;
|
||||
private readonly Counter<long> _fetchDocuments;
|
||||
private readonly Counter<long> _fetchUnchanged;
|
||||
private readonly Counter<long> _fetchFailures;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Counter<long> _parseQuarantine;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
|
||||
public CccsDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_fetchAttempts = _meter.CreateCounter<long>("cccs.fetch.attempts", unit: "operations");
|
||||
_fetchSuccess = _meter.CreateCounter<long>("cccs.fetch.success", unit: "operations");
|
||||
_fetchDocuments = _meter.CreateCounter<long>("cccs.fetch.documents", unit: "documents");
|
||||
_fetchUnchanged = _meter.CreateCounter<long>("cccs.fetch.unchanged", unit: "documents");
|
||||
_fetchFailures = _meter.CreateCounter<long>("cccs.fetch.failures", unit: "operations");
|
||||
_parseSuccess = _meter.CreateCounter<long>("cccs.parse.success", unit: "documents");
|
||||
_parseFailures = _meter.CreateCounter<long>("cccs.parse.failures", unit: "documents");
|
||||
_parseQuarantine = _meter.CreateCounter<long>("cccs.parse.quarantine", unit: "documents");
|
||||
_mapSuccess = _meter.CreateCounter<long>("cccs.map.success", unit: "advisories");
|
||||
_mapFailures = _meter.CreateCounter<long>("cccs.map.failures", unit: "advisories");
|
||||
}
|
||||
|
||||
public void FetchAttempt() => _fetchAttempts.Add(1);
|
||||
|
||||
public void FetchSuccess() => _fetchSuccess.Add(1);
|
||||
|
||||
public void FetchDocument() => _fetchDocuments.Add(1);
|
||||
|
||||
public void FetchUnchanged() => _fetchUnchanged.Add(1);
|
||||
|
||||
public void FetchFailure() => _fetchFailures.Add(1);
|
||||
|
||||
public void ParseSuccess() => _parseSuccess.Add(1);
|
||||
|
||||
public void ParseFailure() => _parseFailures.Add(1);
|
||||
|
||||
public void ParseQuarantine() => _parseQuarantine.Add(1);
|
||||
|
||||
public void MapSuccess() => _mapSuccess.Add(1);
|
||||
|
||||
public void MapFailure() => _mapFailures.Add(1);
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
146
src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedClient.cs
Normal file
146
src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedClient.cs
Normal file
@@ -0,0 +1,146 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Net.Http;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using StellaOps.Feedser.Source.Cccs.Configuration;
|
||||
using StellaOps.Feedser.Source.Common.Fetch;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Internal;
|
||||
|
||||
public sealed class CccsFeedClient
|
||||
{
|
||||
private static readonly string[] AcceptHeaders =
|
||||
{
|
||||
"application/json",
|
||||
"application/vnd.api+json;q=0.9",
|
||||
"text/json;q=0.8",
|
||||
"application/*+json;q=0.7",
|
||||
};
|
||||
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly ILogger<CccsFeedClient> _logger;
|
||||
|
||||
public CccsFeedClient(SourceFetchService fetchService, ILogger<CccsFeedClient> logger)
|
||||
{
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
internal async Task<CccsFeedResult> FetchAsync(CccsFeedEndpoint endpoint, TimeSpan requestTimeout, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(endpoint);
|
||||
if (endpoint.Uri is null)
|
||||
{
|
||||
throw new InvalidOperationException("Feed endpoint URI must be configured.");
|
||||
}
|
||||
|
||||
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, endpoint.Uri)
|
||||
{
|
||||
AcceptHeaders = AcceptHeaders,
|
||||
TimeoutOverride = requestTimeout,
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["cccs.language"] = endpoint.Language,
|
||||
["cccs.feedUri"] = endpoint.Uri.ToString(),
|
||||
},
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (!result.IsSuccess || result.Content is null)
|
||||
{
|
||||
_logger.LogWarning("CCCS feed fetch returned no content for {Uri} (status={Status})", endpoint.Uri, result.StatusCode);
|
||||
return CccsFeedResult.Empty;
|
||||
}
|
||||
|
||||
var feedResponse = Deserialize<CccsFeedResponse>(result.Content);
|
||||
if (feedResponse is null || feedResponse.Error)
|
||||
{
|
||||
_logger.LogWarning("CCCS feed response flagged an error for {Uri}", endpoint.Uri);
|
||||
return CccsFeedResult.Empty;
|
||||
}
|
||||
|
||||
var taxonomy = await FetchTaxonomyAsync(endpoint, requestTimeout, cancellationToken).ConfigureAwait(false);
|
||||
var items = (IReadOnlyList<CccsFeedItem>)feedResponse.Response ?? Array.Empty<CccsFeedItem>();
|
||||
return new CccsFeedResult(items, taxonomy, result.LastModified);
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
|
||||
{
|
||||
_logger.LogError(ex, "CCCS feed deserialization failed for {Uri}", endpoint.Uri);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "CCCS feed fetch failed for {Uri}", endpoint.Uri);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyDictionary<int, string>> FetchTaxonomyAsync(CccsFeedEndpoint endpoint, TimeSpan timeout, CancellationToken cancellationToken)
|
||||
{
|
||||
var taxonomyUri = endpoint.BuildTaxonomyUri();
|
||||
var request = new SourceFetchRequest(CccsOptions.HttpClientName, CccsConnectorPlugin.SourceName, taxonomyUri)
|
||||
{
|
||||
AcceptHeaders = AcceptHeaders,
|
||||
TimeoutOverride = timeout,
|
||||
Metadata = new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["cccs.language"] = endpoint.Language,
|
||||
["cccs.taxonomyUri"] = taxonomyUri.ToString(),
|
||||
},
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
var result = await _fetchService.FetchContentAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
if (!result.IsSuccess || result.Content is null)
|
||||
{
|
||||
_logger.LogDebug("CCCS taxonomy fetch returned no content for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
|
||||
var taxonomyResponse = Deserialize<CccsTaxonomyResponse>(result.Content);
|
||||
if (taxonomyResponse is null || taxonomyResponse.Error)
|
||||
{
|
||||
_logger.LogDebug("CCCS taxonomy response indicated error for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
|
||||
var map = new Dictionary<int, string>(taxonomyResponse.Response.Count);
|
||||
foreach (var item in taxonomyResponse.Response)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(item.Title))
|
||||
{
|
||||
map[item.Id] = item.Title!;
|
||||
}
|
||||
}
|
||||
|
||||
return map;
|
||||
}
|
||||
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to deserialize CCCS taxonomy for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
catch (Exception ex) when (ex is HttpRequestException or TaskCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "CCCS taxonomy fetch failed for {Uri}", taxonomyUri);
|
||||
return new Dictionary<int, string>(0);
|
||||
}
|
||||
}
|
||||
|
||||
private static T? Deserialize<T>(byte[] content)
|
||||
=> JsonSerializer.Deserialize<T>(content, SerializerOptions);
|
||||
}
|
||||
101
src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedModels.cs
Normal file
101
src/StellaOps.Feedser.Source.Cccs/Internal/CccsFeedModels.cs
Normal file
@@ -0,0 +1,101 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Internal;
|
||||
|
||||
internal sealed class CccsFeedResponse
|
||||
{
|
||||
[JsonPropertyName("ERROR")]
|
||||
public bool Error { get; init; }
|
||||
|
||||
[JsonPropertyName("response")]
|
||||
public List<CccsFeedItem> Response { get; init; } = new();
|
||||
}
|
||||
|
||||
internal sealed class CccsFeedItem
|
||||
{
|
||||
[JsonPropertyName("nid")]
|
||||
public int Nid { get; init; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
|
||||
[JsonPropertyName("uuid")]
|
||||
public string? Uuid { get; init; }
|
||||
|
||||
[JsonPropertyName("banner")]
|
||||
public string? Banner { get; init; }
|
||||
|
||||
[JsonPropertyName("lang")]
|
||||
public string? Language { get; init; }
|
||||
|
||||
[JsonPropertyName("date_modified")]
|
||||
public string? DateModified { get; init; }
|
||||
|
||||
[JsonPropertyName("date_modified_ts")]
|
||||
public string? DateModifiedTimestamp { get; init; }
|
||||
|
||||
[JsonPropertyName("date_created")]
|
||||
public string? DateCreated { get; init; }
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("body")]
|
||||
public string[] Body { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; init; }
|
||||
|
||||
[JsonPropertyName("alert_type")]
|
||||
public JsonElement AlertType { get; init; }
|
||||
|
||||
[JsonPropertyName("serial_number")]
|
||||
public string? SerialNumber { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public string? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("moderation_state")]
|
||||
public string? ModerationState { get; init; }
|
||||
|
||||
[JsonPropertyName("external_url")]
|
||||
public string? ExternalUrl { get; init; }
|
||||
}
|
||||
|
||||
internal sealed class CccsTaxonomyResponse
|
||||
{
|
||||
[JsonPropertyName("ERROR")]
|
||||
public bool Error { get; init; }
|
||||
|
||||
[JsonPropertyName("response")]
|
||||
public List<CccsTaxonomyItem> Response { get; init; } = new();
|
||||
}
|
||||
|
||||
internal sealed class CccsTaxonomyItem
|
||||
{
|
||||
[JsonPropertyName("id")]
|
||||
public int Id { get; init; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record CccsFeedResult(
|
||||
IReadOnlyList<CccsFeedItem> Items,
|
||||
IReadOnlyDictionary<int, string> AlertTypes,
|
||||
DateTimeOffset? LastModifiedUtc)
|
||||
{
|
||||
public static CccsFeedResult Empty { get; } = new(
|
||||
Array.Empty<CccsFeedItem>(),
|
||||
new Dictionary<int, string>(0),
|
||||
null);
|
||||
}
|
||||
|
||||
internal static class CccsFeedResultExtensions
|
||||
{
|
||||
public static CccsFeedResult ToResult(this IReadOnlyList<CccsFeedItem> items, DateTimeOffset? lastModified, IReadOnlyDictionary<int, string> alertTypes)
|
||||
=> new(items, alertTypes, lastModified);
|
||||
}
|
||||
449
src/StellaOps.Feedser.Source.Cccs/Internal/CccsHtmlParser.cs
Normal file
449
src/StellaOps.Feedser.Source.Cccs/Internal/CccsHtmlParser.cs
Normal file
@@ -0,0 +1,449 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Text.RegularExpressions;
|
||||
using AngleSharp.Dom;
|
||||
using AngleSharp.Html.Dom;
|
||||
using AngleSharp.Html.Parser;
|
||||
using StellaOps.Feedser.Source.Common.Html;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Internal;
|
||||
|
||||
public sealed class CccsHtmlParser
|
||||
{
|
||||
private static readonly Regex SerialRegex = new(@"(?:(Number|Num[eé]ro)\s*[::]\s*)(?<id>[A-Z0-9\-\/]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex DateRegex = new(@"(?:(Date|Date de publication)\s*[::]\s*)(?<date>[A-Za-zÀ-ÿ0-9,\.\s\-]+)", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex CveRegex = new(@"CVE-\d{4}-\d{4,}", RegexOptions.IgnoreCase | RegexOptions.Compiled);
|
||||
private static readonly Regex CollapseWhitespaceRegex = new(@"\s+", RegexOptions.Compiled);
|
||||
|
||||
private static readonly CultureInfo[] EnglishCultures =
|
||||
{
|
||||
CultureInfo.GetCultureInfo("en-CA"),
|
||||
CultureInfo.GetCultureInfo("en-US"),
|
||||
CultureInfo.InvariantCulture,
|
||||
};
|
||||
|
||||
private static readonly CultureInfo[] FrenchCultures =
|
||||
{
|
||||
CultureInfo.GetCultureInfo("fr-CA"),
|
||||
CultureInfo.GetCultureInfo("fr-FR"),
|
||||
CultureInfo.InvariantCulture,
|
||||
};
|
||||
|
||||
private static readonly string[] ProductHeadingKeywords =
|
||||
{
|
||||
"affected",
|
||||
"produit",
|
||||
"produits",
|
||||
"produits touch",
|
||||
"produits concern",
|
||||
"mesures recommand",
|
||||
};
|
||||
|
||||
private static readonly string[] TrackingParameterPrefixes =
|
||||
{
|
||||
"utm_",
|
||||
"mc_",
|
||||
"mkt_",
|
||||
"elq",
|
||||
};
|
||||
|
||||
private readonly HtmlContentSanitizer _sanitizer;
|
||||
private readonly HtmlParser _parser;
|
||||
|
||||
public CccsHtmlParser(HtmlContentSanitizer sanitizer)
|
||||
{
|
||||
_sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
|
||||
_parser = new HtmlParser(new HtmlParserOptions
|
||||
{
|
||||
IsScripting = false,
|
||||
IsKeepingSourceReferences = false,
|
||||
});
|
||||
}
|
||||
|
||||
internal CccsAdvisoryDto Parse(CccsRawAdvisoryDocument raw)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(raw);
|
||||
|
||||
var baseUri = TryCreateUri(raw.CanonicalUrl);
|
||||
var document = _parser.ParseDocument(raw.BodyHtml ?? string.Empty);
|
||||
var body = document.Body ?? document.DocumentElement;
|
||||
var sanitized = _sanitizer.Sanitize(body?.InnerHtml ?? raw.BodyHtml ?? string.Empty, baseUri);
|
||||
var contentRoot = body ?? document.DocumentElement;
|
||||
|
||||
var serialNumber = !string.IsNullOrWhiteSpace(raw.SerialNumber)
|
||||
? raw.SerialNumber!.Trim()
|
||||
: ExtractSerialNumber(document) ?? raw.SourceId;
|
||||
|
||||
var published = raw.Published ?? ExtractDate(document, raw.Language) ?? raw.Modified;
|
||||
var references = ExtractReferences(contentRoot, baseUri, raw.Language);
|
||||
var products = ExtractProducts(contentRoot);
|
||||
var cveIds = ExtractCveIds(document);
|
||||
|
||||
return new CccsAdvisoryDto
|
||||
{
|
||||
SourceId = raw.SourceId,
|
||||
SerialNumber = serialNumber,
|
||||
Language = raw.Language,
|
||||
Title = raw.Title,
|
||||
Summary = CollapseWhitespace(raw.Summary),
|
||||
CanonicalUrl = raw.CanonicalUrl,
|
||||
ContentHtml = sanitized,
|
||||
Published = published,
|
||||
Modified = raw.Modified ?? published,
|
||||
AlertType = raw.AlertType,
|
||||
Subject = raw.Subject,
|
||||
Products = products,
|
||||
References = references,
|
||||
CveIds = cveIds,
|
||||
};
|
||||
}
|
||||
|
||||
private static Uri? TryCreateUri(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return Uri.TryCreate(value, UriKind.Absolute, out var absolute) ? absolute : null;
|
||||
}
|
||||
|
||||
private static string? ExtractSerialNumber(IDocument document)
|
||||
{
|
||||
if (document.Body is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var element in document.QuerySelectorAll("strong, p, div"))
|
||||
{
|
||||
var text = element.TextContent;
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = SerialRegex.Match(text);
|
||||
if (match.Success && match.Groups["id"].Success)
|
||||
{
|
||||
var value = match.Groups["id"].Value.Trim();
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
var bodyText = document.Body.TextContent;
|
||||
var fallback = SerialRegex.Match(bodyText ?? string.Empty);
|
||||
return fallback.Success && fallback.Groups["id"].Success
|
||||
? fallback.Groups["id"].Value.Trim()
|
||||
: null;
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ExtractDate(IDocument document, string language)
|
||||
{
|
||||
if (document.Body is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var textSegments = new List<string>();
|
||||
foreach (var element in document.QuerySelectorAll("strong, p, div"))
|
||||
{
|
||||
var text = element.TextContent;
|
||||
if (string.IsNullOrWhiteSpace(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = DateRegex.Match(text);
|
||||
if (match.Success && match.Groups["date"].Success)
|
||||
{
|
||||
textSegments.Add(match.Groups["date"].Value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
if (textSegments.Count == 0 && !string.IsNullOrWhiteSpace(document.Body.TextContent))
|
||||
{
|
||||
textSegments.Add(document.Body.TextContent);
|
||||
}
|
||||
|
||||
var cultures = language.StartsWith("fr", StringComparison.OrdinalIgnoreCase) ? FrenchCultures : EnglishCultures;
|
||||
|
||||
foreach (var segment in textSegments)
|
||||
{
|
||||
foreach (var culture in cultures)
|
||||
{
|
||||
if (DateTime.TryParse(segment, culture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed))
|
||||
{
|
||||
return new DateTimeOffset(parsed.ToUniversalTime());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractProducts(IElement? root)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var results = new List<string>();
|
||||
|
||||
foreach (var heading in root.QuerySelectorAll("h1,h2,h3,h4,h5,h6"))
|
||||
{
|
||||
var text = heading.TextContent?.Trim();
|
||||
if (!IsProductHeading(text))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var sibling = heading.NextElementSibling;
|
||||
while (sibling is not null)
|
||||
{
|
||||
if (IsHeading(sibling))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (IsListElement(sibling))
|
||||
{
|
||||
AppendListItems(sibling, results);
|
||||
if (results.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
else if (IsContentContainer(sibling))
|
||||
{
|
||||
foreach (var list in sibling.QuerySelectorAll("ul,ol"))
|
||||
{
|
||||
AppendListItems(list, results);
|
||||
}
|
||||
|
||||
if (results.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
sibling = sibling.NextElementSibling;
|
||||
}
|
||||
|
||||
if (results.Count > 0)
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (results.Count == 0)
|
||||
{
|
||||
foreach (var li in root.QuerySelectorAll("ul li,ol li"))
|
||||
{
|
||||
var itemText = CollapseWhitespace(li.TextContent);
|
||||
if (!string.IsNullOrWhiteSpace(itemText))
|
||||
{
|
||||
results.Add(itemText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: results
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static bool IsProductHeading(string? heading)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(heading))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var lowered = heading.ToLowerInvariant();
|
||||
return ProductHeadingKeywords.Any(keyword => lowered.Contains(keyword, StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static bool IsHeading(IElement element)
|
||||
=> element.LocalName.Length == 2
|
||||
&& element.LocalName[0] == 'h'
|
||||
&& char.IsDigit(element.LocalName[1]);
|
||||
|
||||
private static bool IsListElement(IElement element)
|
||||
=> string.Equals(element.LocalName, "ul", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(element.LocalName, "ol", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static bool IsContentContainer(IElement element)
|
||||
=> string.Equals(element.LocalName, "div", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(element.LocalName, "section", StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(element.LocalName, "article", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private static void AppendListItems(IElement listElement, ICollection<string> buffer)
|
||||
{
|
||||
foreach (var li in listElement.QuerySelectorAll("li"))
|
||||
{
|
||||
if (li is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var clone = li.Clone(true) as IElement;
|
||||
if (clone is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var nested in clone.QuerySelectorAll("ul,ol"))
|
||||
{
|
||||
nested.Remove();
|
||||
}
|
||||
|
||||
var itemText = CollapseWhitespace(clone.TextContent);
|
||||
if (!string.IsNullOrWhiteSpace(itemText))
|
||||
{
|
||||
buffer.Add(itemText);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CccsReferenceDto> ExtractReferences(IElement? root, Uri? baseUri, string language)
|
||||
{
|
||||
if (root is null)
|
||||
{
|
||||
return Array.Empty<CccsReferenceDto>();
|
||||
}
|
||||
|
||||
var references = new List<CccsReferenceDto>();
|
||||
foreach (var anchor in root.QuerySelectorAll("a[href]"))
|
||||
{
|
||||
var href = anchor.GetAttribute("href");
|
||||
var normalized = NormalizeReferenceUrl(href, baseUri, language);
|
||||
if (normalized is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var label = CollapseWhitespace(anchor.TextContent);
|
||||
references.Add(new CccsReferenceDto(normalized, string.IsNullOrWhiteSpace(label) ? null : label));
|
||||
}
|
||||
|
||||
return references.Count == 0
|
||||
? Array.Empty<CccsReferenceDto>()
|
||||
: references
|
||||
.GroupBy(reference => reference.Url, StringComparer.Ordinal)
|
||||
.Select(group => group.First())
|
||||
.OrderBy(reference => reference.Url, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? NormalizeReferenceUrl(string? href, Uri? baseUri, string language)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(href))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!Uri.TryCreate(href, UriKind.Absolute, out var absolute))
|
||||
{
|
||||
if (baseUri is null || !Uri.TryCreate(baseUri, href, out absolute))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
var builder = new UriBuilder(absolute)
|
||||
{
|
||||
Fragment = string.Empty,
|
||||
};
|
||||
|
||||
var filteredQuery = FilterTrackingParameters(builder.Query, builder.Uri, language);
|
||||
builder.Query = filteredQuery;
|
||||
|
||||
return builder.Uri.ToString();
|
||||
}
|
||||
|
||||
private static string FilterTrackingParameters(string query, Uri uri, string language)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(query))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var trimmed = query.TrimStart('?');
|
||||
if (string.IsNullOrWhiteSpace(trimmed))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var parameters = trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries);
|
||||
var kept = new List<string>();
|
||||
|
||||
foreach (var parameter in parameters)
|
||||
{
|
||||
var separatorIndex = parameter.IndexOf('=');
|
||||
var key = separatorIndex >= 0 ? parameter[..separatorIndex] : parameter;
|
||||
if (TrackingParameterPrefixes.Any(prefix => key.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
|
||||
&& key.Equals("lang", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
kept.Add($"lang={language}");
|
||||
continue;
|
||||
}
|
||||
|
||||
kept.Add(parameter);
|
||||
}
|
||||
|
||||
if (uri.Host.Contains("cyber.gc.ca", StringComparison.OrdinalIgnoreCase)
|
||||
&& kept.All(parameter => !parameter.StartsWith("lang=", StringComparison.OrdinalIgnoreCase)))
|
||||
{
|
||||
kept.Add($"lang={language}");
|
||||
}
|
||||
|
||||
return kept.Count == 0 ? string.Empty : string.Join("&", kept);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> ExtractCveIds(IDocument document)
|
||||
{
|
||||
if (document.Body is null)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var matches = CveRegex.Matches(document.Body.TextContent ?? string.Empty);
|
||||
if (matches.Count == 0)
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
return matches
|
||||
.Select(match => match.Value.ToUpperInvariant())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.OrderBy(value => value, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? CollapseWhitespace(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var collapsed = CollapseWhitespaceRegex.Replace(value, " ").Trim();
|
||||
return collapsed.Length == 0 ? null : collapsed;
|
||||
}
|
||||
}
|
||||
151
src/StellaOps.Feedser.Source.Cccs/Internal/CccsMapper.cs
Normal file
151
src/StellaOps.Feedser.Source.Cccs/Internal/CccsMapper.cs
Normal file
@@ -0,0 +1,151 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Internal;
|
||||
|
||||
internal static class CccsMapper
|
||||
{
|
||||
public static Advisory Map(CccsAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var packages = BuildPackages(dto, recordedAt);
|
||||
var provenance = new[]
|
||||
{
|
||||
new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"advisory",
|
||||
dto.AlertType ?? dto.SerialNumber,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory })
|
||||
};
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.SerialNumber,
|
||||
title: dto.Title,
|
||||
summary: dto.Summary,
|
||||
language: dto.Language,
|
||||
published: dto.Published ?? dto.Modified,
|
||||
modified: dto.Modified ?? dto.Published,
|
||||
severity: null,
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: packages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: provenance);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(CccsAdvisoryDto dto)
|
||||
{
|
||||
var aliases = new List<string>(capacity: 4)
|
||||
{
|
||||
dto.SerialNumber,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(dto.SourceId)
|
||||
&& !string.Equals(dto.SourceId, dto.SerialNumber, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
aliases.Add(dto.SourceId);
|
||||
}
|
||||
|
||||
foreach (var cve in dto.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliases.Add(cve);
|
||||
}
|
||||
}
|
||||
|
||||
return aliases
|
||||
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>
|
||||
{
|
||||
new(dto.CanonicalUrl, "details", "cccs", null, new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
dto.CanonicalUrl,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References }))
|
||||
};
|
||||
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
reference.Url,
|
||||
"reference",
|
||||
"cccs",
|
||||
reference.Label,
|
||||
new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
reference.Url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References })));
|
||||
}
|
||||
|
||||
return references
|
||||
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(CccsAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Products.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Products.Count);
|
||||
foreach (var product in dto.Products)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(product))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var identifier = product.Trim();
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CccsConnectorPlugin.SourceName,
|
||||
"package",
|
||||
identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.AffectedPackages });
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform: null,
|
||||
versionRanges: Array.Empty<AffectedVersionRange>(),
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { provenance },
|
||||
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
|
||||
}
|
||||
|
||||
return packages.Count == 0
|
||||
? Array.Empty<AffectedPackage>()
|
||||
: packages
|
||||
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,58 @@
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs.Internal;
|
||||
|
||||
internal sealed record CccsRawAdvisoryDocument
|
||||
{
|
||||
[JsonPropertyName("sourceId")]
|
||||
public string SourceId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("serialNumber")]
|
||||
public string? SerialNumber { get; init; }
|
||||
|
||||
[JsonPropertyName("uuid")]
|
||||
public string? Uuid { get; init; }
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; init; } = "en";
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("canonicalUrl")]
|
||||
public string CanonicalUrl { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("externalUrl")]
|
||||
public string? ExternalUrl { get; init; }
|
||||
|
||||
[JsonPropertyName("bodyHtml")]
|
||||
public string BodyHtml { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("bodySegments")]
|
||||
public string[] BodySegments { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("alertType")]
|
||||
public string? AlertType { get; init; }
|
||||
|
||||
[JsonPropertyName("subject")]
|
||||
public string? Subject { get; init; }
|
||||
|
||||
[JsonPropertyName("banner")]
|
||||
public string? Banner { get; init; }
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("rawCreated")]
|
||||
public string? RawDateCreated { get; init; }
|
||||
|
||||
[JsonPropertyName("rawModified")]
|
||||
public string? RawDateModified { get; init; }
|
||||
}
|
||||
22
src/StellaOps.Feedser.Source.Cccs/Jobs.cs
Normal file
22
src/StellaOps.Feedser.Source.Cccs/Jobs.cs
Normal file
@@ -0,0 +1,22 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Feedser.Source.Cccs;
|
||||
|
||||
internal static class CccsJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cccs:fetch";
|
||||
}
|
||||
|
||||
internal sealed class CccsFetchJob : IJob
|
||||
{
|
||||
private readonly CccsConnector _connector;
|
||||
|
||||
public CccsFetchJob(CccsConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Feedser.Source.Cccs.Tests")]
|
||||
@@ -6,11 +6,11 @@
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Storage.Mongo/StellaOps.Feedser.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -2,9 +2,10 @@
|
||||
| Task | Owner(s) | Depends on | Notes |
|
||||
|---|---|---|---|
|
||||
|FEEDCONN-CCCS-02-001 Catalogue official CCCS advisory feeds|BE-Conn-CCCS|Research|**DONE (2025-10-11)** – Resolved RSS→Atom redirects (`/api/cccs/rss/v1/get?...` → `/api/cccs/atom/v1/get?...`), confirmed feed caps at 50 entries with inline HTML bodies, no `Last-Modified`/`ETag`, and `updated` timestamps in UTC. Findings and packet captures parked in `docs/feedser-connector-research-20251011.md`; retention sweep follow-up tracked in 02-007.|
|
||||
|FEEDCONN-CCCS-02-002 Implement fetch & source state handling|BE-Conn-CCCS|Source.Common, Storage.Mongo|**TODO** – Register HTTP client with redirect allowance + `User-Agent` override, persist Atom payload plus derived SHA256 to `document` store, and use feed-level `<updated>` / entry `<updated>` for cursoring. Capture absence of cache headers by stamping synthetic `fetchedAt` TTL and throttling retries to 1/min (Azure App Gateway fronted).|
|
||||
|FEEDCONN-CCCS-02-003 DTO/parser implementation|BE-Conn-CCCS|Source.Common|**TODO** – Build DTO that keeps original HTML in `ContentHtml` and extracts structured fields: `Serial number`, `Date`, product bullet lists, and reference hyperlinks. Strip tracking query params, collapse whitespace, and normalise French vs English feeds via `lang=` query.|
|
||||
|FEEDCONN-CCCS-02-004 Canonical mapping & range primitives|BE-Conn-CCCS|Models|**TODO** – Map advisories into canonical records with aliases, references, vendor/package range primitives, and provenance. Align normalized SemVer rules per `../StellaOps.Feedser.Merge/RANGE_PRIMITIVES_COORDINATION.md`.<br>2025-10-11 research trail: emit `NormalizedVersions` like `[{"scheme":"semver","type":"range","min":"<min>","minInclusive":true,"max":"<max>","maxInclusive":false,"notes":"cccs:bulletin-id"}]`; include provenance notes to keep storage decision reasons.|
|
||||
|FEEDCONN-CCCS-02-005 Deterministic fixtures & tests|QA|Testing|**TODO** – Add regression tests with canned fixtures; support `UPDATE_CCCS_FIXTURES=1` to refresh snapshots.|
|
||||
|FEEDCONN-CCCS-02-006 Observability & documentation|DevEx|Docs|**TODO** – Document connector configuration, add logging/metrics, and update backlog once feature-complete.|
|
||||
|FEEDCONN-CCCS-02-007 Historical advisory harvesting plan|BE-Conn-CCCS|Research|**TODO** – HTML index exposes `?page=<n>` pagination (`https://www.cyber.gc.ca/en/alerts-advisories?page=4` tested OK). Need to measure depth, record earliest advisory date, and confirm whether separate feeds exist for bulletins/vulnerabilities. Produce backfill plan (HTML scrape → Atom transformation) and language split guidance.|
|
||||
|FEEDCONN-CCCS-02-002 Implement fetch & source state handling|BE-Conn-CCCS|Source.Common, Storage.Mongo|**DONE (2025-10-14)** – `CccsConnector.FetchAsync` now hydrates feeds via `CccsFeedClient`, persists per-entry JSON payloads with SHA256 dedupe and cursor state, throttles requests, and records taxonomy + language metadata in document state.|
|
||||
|FEEDCONN-CCCS-02-003 DTO/parser implementation|BE-Conn-CCCS|Source.Common|**DONE (2025-10-14)** – Added `CccsHtmlParser` to sanitize Atom body HTML, extract serial/date/product bullets, collapse whitespace, and emit normalized reference URLs; `ParseAsync` now persists DTO records under schema `cccs.dto.v1`.|
|
||||
|FEEDCONN-CCCS-02-004 Canonical mapping & range primitives|BE-Conn-CCCS|Models|**DONE (2025-10-14)** – `CccsMapper` now materializes canonical advisories (aliases from serial/source/CVEs, references incl. canonical URL, vendor package records) with provenance masks; `MapAsync` stores results in `AdvisoryStore`.|
|
||||
|FEEDCONN-CCCS-02-005 Deterministic fixtures & tests|QA|Testing|**DONE (2025-10-14)** – Added English/French fixtures plus parser + connector end-to-end tests (`StellaOps.Feedser.Source.Cccs.Tests`). Canned HTTP handler + Mongo fixture enables fetch→parse→map regression; fixtures refresh via `UPDATE_CCCS_FIXTURES=1`.|
|
||||
|FEEDCONN-CCCS-02-006 Observability & documentation|DevEx|Docs|**DONE (2025-10-15)** – Added `CccsDiagnostics` meter (fetch/parse/map counters), enriched connector logs with document counts, and published `docs/ops/feedser-cccs-operations.md` covering config, telemetry, and sanitiser guidance.|
|
||||
|FEEDCONN-CCCS-02-007 Historical advisory harvesting plan|BE-Conn-CCCS|Research|**DONE (2025-10-15)** – Measured `/api/cccs/threats/v1/get` inventory (~5.1k rows/lang; earliest 2018-06-08), documented backfill workflow + language split strategy, and linked the runbook for Offline Kit execution.|
|
||||
|FEEDCONN-CCCS-02-008 Raw DOM parsing refinement|BE-Conn-CCCS|Source.Common|**DONE (2025-10-15)** – Parser now walks unsanitised DOM (heading + nested list coverage), sanitizer keeps `<h#>`/`section` nodes, and regression fixtures/tests assert EN/FR list handling + preserved HTML structure.|
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
using System;
|
||||
using System.Net.Http;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using FluentAssertions;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Http;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Feedser.Source.CertBund.Configuration;
|
||||
using StellaOps.Feedser.Source.Common.Http;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Source.Common.Fetch;
|
||||
using StellaOps.Feedser.Source.Common.Testing;
|
||||
using StellaOps.Feedser.Storage.Mongo;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using StellaOps.Feedser.Storage.Mongo.Dtos;
|
||||
using StellaOps.Feedser.Testing;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Tests;
|
||||
|
||||
[Collection("mongo-fixture")]
|
||||
public sealed class CertBundConnectorTests : IAsyncLifetime
|
||||
{
|
||||
private static readonly Uri FeedUri = new("https://test.local/content/public/securityAdvisory/rss");
|
||||
private static readonly Uri PortalUri = new("https://test.local/portal/");
|
||||
private static readonly Uri DetailUri = new("https://test.local/portal/api/securityadvisory?name=WID-SEC-2025-2264");
|
||||
|
||||
private readonly MongoIntegrationFixture _fixture;
|
||||
private readonly CannedHttpMessageHandler _handler;
|
||||
|
||||
public CertBundConnectorTests(MongoIntegrationFixture fixture)
|
||||
{
|
||||
_fixture = fixture;
|
||||
_handler = new CannedHttpMessageHandler();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task FetchParseMap_ProducesCanonicalAdvisory()
|
||||
{
|
||||
await using var provider = await BuildServiceProviderAsync();
|
||||
SeedResponses();
|
||||
|
||||
var connector = provider.GetRequiredService<CertBundConnector>();
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
await connector.ParseAsync(provider, CancellationToken.None);
|
||||
await connector.MapAsync(provider, CancellationToken.None);
|
||||
|
||||
var advisoryStore = provider.GetRequiredService<IAdvisoryStore>();
|
||||
var advisories = await advisoryStore.GetRecentAsync(5, CancellationToken.None);
|
||||
advisories.Should().HaveCount(1);
|
||||
|
||||
var advisory = advisories[0];
|
||||
advisory.AdvisoryKey.Should().Be("WID-SEC-2025-2264");
|
||||
advisory.Aliases.Should().Contain("CVE-2025-1234");
|
||||
advisory.AffectedPackages.Should().Contain(package => package.Identifier.Contains("Ivanti"));
|
||||
advisory.References.Should().Contain(reference => reference.Url == DetailUri.ToString());
|
||||
advisory.Language.Should().Be("de");
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertBundConnectorPlugin.SourceName, CancellationToken.None);
|
||||
state.Should().NotBeNull();
|
||||
state!.Cursor.Should().NotBeNull();
|
||||
state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue();
|
||||
pendingDocs!.AsBsonArray.Should().BeEmpty();
|
||||
state.Cursor.TryGetValue("pendingMappings", out var pendingMappings).Should().BeTrue();
|
||||
pendingMappings!.AsBsonArray.Should().BeEmpty();
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Fetch_PersistsDocumentWithMetadata()
|
||||
{
|
||||
await using var provider = await BuildServiceProviderAsync();
|
||||
SeedResponses();
|
||||
|
||||
var connector = provider.GetRequiredService<CertBundConnector>();
|
||||
await connector.FetchAsync(provider, CancellationToken.None);
|
||||
|
||||
var documentStore = provider.GetRequiredService<IDocumentStore>();
|
||||
var document = await documentStore.FindBySourceAndUriAsync(CertBundConnectorPlugin.SourceName, DetailUri.ToString(), CancellationToken.None);
|
||||
document.Should().NotBeNull();
|
||||
document!.Metadata.Should().ContainKey("certbund.advisoryId").WhoseValue.Should().Be("WID-SEC-2025-2264");
|
||||
document.Metadata.Should().ContainKey("certbund.category");
|
||||
document.Metadata.Should().ContainKey("certbund.published");
|
||||
document.Status.Should().Be(DocumentStatuses.PendingParse);
|
||||
|
||||
var stateRepository = provider.GetRequiredService<ISourceStateRepository>();
|
||||
var state = await stateRepository.TryGetAsync(CertBundConnectorPlugin.SourceName, CancellationToken.None);
|
||||
state.Should().NotBeNull();
|
||||
state!.Cursor.Should().NotBeNull();
|
||||
state.Cursor.TryGetValue("pendingDocuments", out var pendingDocs).Should().BeTrue();
|
||||
pendingDocs!.AsBsonArray.Should().HaveCount(1);
|
||||
}
|
||||
|
||||
private async Task<ServiceProvider> BuildServiceProviderAsync()
|
||||
{
|
||||
await _fixture.Client.DropDatabaseAsync(_fixture.Database.DatabaseNamespace.DatabaseName);
|
||||
_handler.Clear();
|
||||
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging(builder => builder.AddProvider(NullLoggerProvider.Instance));
|
||||
services.AddSingleton(_handler);
|
||||
|
||||
services.AddMongoStorage(options =>
|
||||
{
|
||||
options.ConnectionString = _fixture.Runner.ConnectionString;
|
||||
options.DatabaseName = _fixture.Database.DatabaseNamespace.DatabaseName;
|
||||
options.CommandTimeout = TimeSpan.FromSeconds(5);
|
||||
});
|
||||
|
||||
services.AddSourceCommon();
|
||||
services.AddCertBundConnector(options =>
|
||||
{
|
||||
options.FeedUri = FeedUri;
|
||||
options.PortalBootstrapUri = PortalUri;
|
||||
options.DetailApiUri = new Uri("https://test.local/portal/api/securityadvisory");
|
||||
options.RequestDelay = TimeSpan.Zero;
|
||||
options.MaxAdvisoriesPerFetch = 10;
|
||||
options.MaxKnownAdvisories = 32;
|
||||
});
|
||||
|
||||
services.Configure<HttpClientFactoryOptions>(CertBundOptions.HttpClientName, builderOptions =>
|
||||
{
|
||||
builderOptions.HttpMessageHandlerBuilderActions.Add(builder =>
|
||||
{
|
||||
builder.PrimaryHandler = _handler;
|
||||
});
|
||||
});
|
||||
|
||||
var provider = services.BuildServiceProvider();
|
||||
var bootstrapper = provider.GetRequiredService<MongoBootstrapper>();
|
||||
await bootstrapper.InitializeAsync(CancellationToken.None);
|
||||
return provider;
|
||||
}
|
||||
|
||||
private void SeedResponses()
|
||||
{
|
||||
AddJsonResponse(DetailUri, ReadFixture("certbund-detail.json"));
|
||||
AddXmlResponse(FeedUri, ReadFixture("certbund-feed.xml"), "application/rss+xml");
|
||||
AddHtmlResponse(PortalUri, "<html><body>OK</body></html>");
|
||||
}
|
||||
|
||||
private void AddJsonResponse(Uri uri, string json, string? etag = null)
|
||||
{
|
||||
_handler.AddResponse(uri, () =>
|
||||
{
|
||||
var response = new HttpResponseMessage(System.Net.HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(json, Encoding.UTF8, "application/json"),
|
||||
};
|
||||
if (!string.IsNullOrWhiteSpace(etag))
|
||||
{
|
||||
response.Headers.ETag = new EntityTagHeaderValue(etag);
|
||||
}
|
||||
|
||||
return response;
|
||||
});
|
||||
}
|
||||
|
||||
private void AddXmlResponse(Uri uri, string xml, string contentType)
|
||||
{
|
||||
_handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(xml, Encoding.UTF8, contentType),
|
||||
});
|
||||
}
|
||||
|
||||
private void AddHtmlResponse(Uri uri, string html)
|
||||
{
|
||||
_handler.AddResponse(uri, () => new HttpResponseMessage(System.Net.HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(html, Encoding.UTF8, "text/html"),
|
||||
});
|
||||
}
|
||||
|
||||
private static string ReadFixture(string fileName)
|
||||
=> System.IO.File.ReadAllText(System.IO.Path.Combine(AppContext.BaseDirectory, "Fixtures", fileName));
|
||||
|
||||
public Task InitializeAsync() => Task.CompletedTask;
|
||||
|
||||
public Task DisposeAsync() => Task.CompletedTask;
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"name": "WID-SEC-2025-2264",
|
||||
"title": "Ivanti Endpoint Manager: Mehrere Schwachstellen ermöglichen Codeausführung",
|
||||
"summary": "Ein entfernter, anonymer Angreifer kann mehrere Schwachstellen in Ivanti Endpoint Manager ausnutzen.",
|
||||
"description": "<p>Ivanti Endpoint Manager weist mehrere Schwachstellen auf.</p><p>Ein Angreifer kann beliebigen Code ausführen.</p>",
|
||||
"severity": "hoch",
|
||||
"language": "de",
|
||||
"published": "2025-10-14T06:24:49Z",
|
||||
"updated": "2025-10-14T07:00:00Z",
|
||||
"cveIds": [
|
||||
"CVE-2025-1234",
|
||||
"CVE-2025-5678"
|
||||
],
|
||||
"references": [
|
||||
{
|
||||
"url": "https://example.com/vendor/advisory",
|
||||
"label": "Vendor Advisory"
|
||||
},
|
||||
{
|
||||
"url": "https://example.com/mitre",
|
||||
"label": "MITRE"
|
||||
}
|
||||
],
|
||||
"products": [
|
||||
{
|
||||
"vendor": "Ivanti",
|
||||
"name": "Endpoint Manager",
|
||||
"versions": "2023.1 bis 2024.2"
|
||||
},
|
||||
{
|
||||
"vendor": "Ivanti",
|
||||
"name": "Endpoint Manager Cloud",
|
||||
"versions": "alle"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0">
|
||||
<channel>
|
||||
<title>BSI Warn- und Informationsdienst</title>
|
||||
<link>https://wid.cert-bund.de/portal/wid/securityadvisory</link>
|
||||
<description>Test feed</description>
|
||||
<pubDate>Tue, 14 Oct 2025 07:06:21 GMT</pubDate>
|
||||
<item>
|
||||
<title>[hoch] Ivanti Endpoint Manager: Mehrere Schwachstellen ermöglichen Codeausführung</title>
|
||||
<link>https://wid.cert-bund.de/portal/wid/securityadvisory?name=WID-SEC-2025-2264</link>
|
||||
<category>hoch</category>
|
||||
<pubDate>Tue, 14 Oct 2025 06:24:49 GMT</pubDate>
|
||||
</item>
|
||||
</channel>
|
||||
</rss>
|
||||
@@ -0,0 +1,22 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.CertBund/StellaOps.Feedser.Source.CertBund.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="FluentAssertions" Version="6.12.0" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<None Update="Fixtures\*.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
<None Update="Fixtures\*.xml">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
435
src/StellaOps.Feedser.Source.CertBund/CertBundConnector.cs
Normal file
435
src/StellaOps.Feedser.Source.CertBund/CertBundConnector.cs
Normal file
@@ -0,0 +1,435 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Feedser.Source.CertBund.Configuration;
|
||||
using StellaOps.Feedser.Source.CertBund.Internal;
|
||||
using StellaOps.Feedser.Source.Common;
|
||||
using StellaOps.Feedser.Source.Common.Fetch;
|
||||
using StellaOps.Feedser.Source.Common.Html;
|
||||
using StellaOps.Feedser.Storage.Mongo;
|
||||
using StellaOps.Feedser.Storage.Mongo.Advisories;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
using StellaOps.Feedser.Storage.Mongo.Dtos;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund;
|
||||
|
||||
public sealed class CertBundConnector : IFeedConnector
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly CertBundFeedClient _feedClient;
|
||||
private readonly CertBundDetailParser _detailParser;
|
||||
private readonly SourceFetchService _fetchService;
|
||||
private readonly RawDocumentStorage _rawDocumentStorage;
|
||||
private readonly IDocumentStore _documentStore;
|
||||
private readonly IDtoStore _dtoStore;
|
||||
private readonly IAdvisoryStore _advisoryStore;
|
||||
private readonly ISourceStateRepository _stateRepository;
|
||||
private readonly CertBundOptions _options;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly CertBundDiagnostics _diagnostics;
|
||||
private readonly ILogger<CertBundConnector> _logger;
|
||||
|
||||
public CertBundConnector(
|
||||
CertBundFeedClient feedClient,
|
||||
CertBundDetailParser detailParser,
|
||||
SourceFetchService fetchService,
|
||||
RawDocumentStorage rawDocumentStorage,
|
||||
IDocumentStore documentStore,
|
||||
IDtoStore dtoStore,
|
||||
IAdvisoryStore advisoryStore,
|
||||
ISourceStateRepository stateRepository,
|
||||
IOptions<CertBundOptions> options,
|
||||
CertBundDiagnostics diagnostics,
|
||||
TimeProvider? timeProvider,
|
||||
ILogger<CertBundConnector> logger)
|
||||
{
|
||||
_feedClient = feedClient ?? throw new ArgumentNullException(nameof(feedClient));
|
||||
_detailParser = detailParser ?? throw new ArgumentNullException(nameof(detailParser));
|
||||
_fetchService = fetchService ?? throw new ArgumentNullException(nameof(fetchService));
|
||||
_rawDocumentStorage = rawDocumentStorage ?? throw new ArgumentNullException(nameof(rawDocumentStorage));
|
||||
_documentStore = documentStore ?? throw new ArgumentNullException(nameof(documentStore));
|
||||
_dtoStore = dtoStore ?? throw new ArgumentNullException(nameof(dtoStore));
|
||||
_advisoryStore = advisoryStore ?? throw new ArgumentNullException(nameof(advisoryStore));
|
||||
_stateRepository = stateRepository ?? throw new ArgumentNullException(nameof(stateRepository));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_diagnostics = diagnostics ?? throw new ArgumentNullException(nameof(diagnostics));
|
||||
_timeProvider = timeProvider ?? TimeProvider.System;
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string SourceName => CertBundConnectorPlugin.SourceName;
|
||||
|
||||
public async Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
IReadOnlyList<CertBundFeedItem> feedItems;
|
||||
|
||||
_diagnostics.FeedFetchAttempt();
|
||||
try
|
||||
{
|
||||
feedItems = await _feedClient.LoadAsync(cancellationToken).ConfigureAwait(false);
|
||||
_diagnostics.FeedFetchSuccess(feedItems.Count);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund feed fetch failed");
|
||||
_diagnostics.FeedFetchFailure();
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
var coverageDays = CalculateCoverageDays(feedItems, now);
|
||||
_diagnostics.RecordFeedCoverage(coverageDays);
|
||||
|
||||
if (feedItems.Count == 0)
|
||||
{
|
||||
await UpdateCursorAsync(cursor.WithLastFetch(now), cancellationToken).ConfigureAwait(false);
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var knownAdvisories = new HashSet<string>(cursor.KnownAdvisories, StringComparer.OrdinalIgnoreCase);
|
||||
var processed = 0;
|
||||
var alreadyKnown = 0;
|
||||
var notModified = 0;
|
||||
var detailFailures = 0;
|
||||
var truncated = false;
|
||||
var latestPublished = cursor.LastPublished ?? DateTimeOffset.MinValue;
|
||||
|
||||
foreach (var item in feedItems.OrderByDescending(static i => i.Published))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (knownAdvisories.Contains(item.AdvisoryId))
|
||||
{
|
||||
alreadyKnown++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (processed >= _options.MaxAdvisoriesPerFetch)
|
||||
{
|
||||
truncated = true;
|
||||
break;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_diagnostics.DetailFetchAttempt();
|
||||
var existing = await _documentStore.FindBySourceAndUriAsync(SourceName, item.DetailUri.ToString(), cancellationToken).ConfigureAwait(false);
|
||||
var request = new SourceFetchRequest(CertBundOptions.HttpClientName, SourceName, item.DetailUri)
|
||||
{
|
||||
AcceptHeaders = new[] { "application/json", "text/json" },
|
||||
Metadata = CertBundDocumentMetadata.CreateMetadata(item),
|
||||
ETag = existing?.Etag,
|
||||
LastModified = existing?.LastModified,
|
||||
TimeoutOverride = _options.RequestTimeout,
|
||||
};
|
||||
|
||||
var result = await _fetchService.FetchAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
if (result.IsNotModified)
|
||||
{
|
||||
_diagnostics.DetailFetchNotModified();
|
||||
notModified++;
|
||||
knownAdvisories.Add(item.AdvisoryId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!result.IsSuccess || result.Document is null)
|
||||
{
|
||||
_diagnostics.DetailFetchFailure("skipped");
|
||||
detailFailures++;
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.DetailFetchSuccess();
|
||||
pendingDocuments.Add(result.Document.Id);
|
||||
pendingMappings.Remove(result.Document.Id);
|
||||
knownAdvisories.Add(item.AdvisoryId);
|
||||
processed++;
|
||||
|
||||
if (_options.RequestDelay > TimeSpan.Zero)
|
||||
{
|
||||
await Task.Delay(_options.RequestDelay, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund detail fetch failed for {AdvisoryId}", item.AdvisoryId);
|
||||
_diagnostics.DetailFetchFailure("exception");
|
||||
detailFailures++;
|
||||
await _stateRepository.MarkFailureAsync(SourceName, now, _options.FailureBackoff, ex.Message, cancellationToken).ConfigureAwait(false);
|
||||
throw;
|
||||
}
|
||||
|
||||
if (item.Published > latestPublished)
|
||||
{
|
||||
latestPublished = item.Published;
|
||||
}
|
||||
}
|
||||
|
||||
_diagnostics.DetailFetchEnqueued(processed);
|
||||
|
||||
if (feedItems.Count > 0 || processed > 0 || detailFailures > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"CERT-Bund fetch cycle: feed items {FeedItems}, enqueued {Enqueued}, already known {Known}, not modified {NotModified}, detail failures {DetailFailures}, pending documents {PendingDocuments}, pending mappings {PendingMappings}, truncated {Truncated}, coverageDays={CoverageDays}",
|
||||
feedItems.Count,
|
||||
processed,
|
||||
alreadyKnown,
|
||||
notModified,
|
||||
detailFailures,
|
||||
pendingDocuments.Count,
|
||||
pendingMappings.Count,
|
||||
truncated,
|
||||
coverageDays ?? double.NaN);
|
||||
}
|
||||
|
||||
var trimmedKnown = knownAdvisories.Count > _options.MaxKnownAdvisories
|
||||
? knownAdvisories.OrderByDescending(id => id, StringComparer.OrdinalIgnoreCase)
|
||||
.Take(_options.MaxKnownAdvisories)
|
||||
.ToArray()
|
||||
: knownAdvisories.ToArray();
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(pendingDocuments)
|
||||
.WithPendingMappings(pendingMappings)
|
||||
.WithKnownAdvisories(trimmedKnown)
|
||||
.WithLastPublished(latestPublished == DateTimeOffset.MinValue ? cursor.LastPublished : latestPublished)
|
||||
.WithLastFetch(now);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingDocuments.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var remainingDocuments = cursor.PendingDocuments.ToHashSet();
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var now = _timeProvider.GetUtcNow();
|
||||
var parsedCount = 0;
|
||||
var failedCount = 0;
|
||||
|
||||
foreach (var documentId in cursor.PendingDocuments)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!document.GridFsId.HasValue)
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.ParseFailure("missing_payload");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
byte[] payload;
|
||||
try
|
||||
{
|
||||
payload = await _rawDocumentStorage.DownloadAsync(document.GridFsId.Value, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund unable to download document {DocumentId}", document.Id);
|
||||
_diagnostics.ParseFailure("download_failed");
|
||||
throw;
|
||||
}
|
||||
|
||||
CertBundAdvisoryDto dto;
|
||||
try
|
||||
{
|
||||
dto = _detailParser.Parse(new Uri(document.Uri), new Uri(document.Metadata?["certbund.portalUri"] ?? document.Uri), payload);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund failed to parse advisory detail {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.ParseFailure("parse_error");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
_diagnostics.ParseSuccess(dto.Products.Count, dto.CveIds.Count);
|
||||
parsedCount++;
|
||||
|
||||
var bson = BsonDocument.Parse(JsonSerializer.Serialize(dto, SerializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, SourceName, "cert-bund.detail.v1", bson, now);
|
||||
await _dtoStore.UpsertAsync(dtoRecord, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.PendingMap, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
remainingDocuments.Remove(documentId);
|
||||
pendingMappings.Add(document.Id);
|
||||
}
|
||||
|
||||
if (cursor.PendingDocuments.Count > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"CERT-Bund parse cycle: parsed {Parsed}, failures {Failures}, remaining documents {RemainingDocuments}, pending mappings {PendingMappings}",
|
||||
parsedCount,
|
||||
failedCount,
|
||||
remainingDocuments.Count,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor
|
||||
.WithPendingDocuments(remainingDocuments)
|
||||
.WithPendingMappings(pendingMappings);
|
||||
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
public async Task MapAsync(IServiceProvider services, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
|
||||
var cursor = await GetCursorAsync(cancellationToken).ConfigureAwait(false);
|
||||
if (cursor.PendingMappings.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var pendingMappings = cursor.PendingMappings.ToHashSet();
|
||||
var mappedCount = 0;
|
||||
var failedCount = 0;
|
||||
|
||||
foreach (var documentId in cursor.PendingMappings)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var document = await _documentStore.FindAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (document is null)
|
||||
{
|
||||
pendingMappings.Remove(documentId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var dtoRecord = await _dtoStore.FindByDocumentIdAsync(documentId, cancellationToken).ConfigureAwait(false);
|
||||
if (dtoRecord is null)
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapFailure("missing_dto");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
CertBundAdvisoryDto? dto;
|
||||
try
|
||||
{
|
||||
dto = JsonSerializer.Deserialize<CertBundAdvisoryDto>(dtoRecord.Payload.ToJson(), SerializerOptions);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund failed to deserialize DTO for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapFailure("deserialize_failed");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (dto is null)
|
||||
{
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapFailure("null_dto");
|
||||
failedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var advisory = CertBundMapper.Map(dto, document, dtoRecord.ValidatedAt);
|
||||
await _advisoryStore.UpsertAsync(advisory, cancellationToken).ConfigureAwait(false);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Mapped, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapSuccess(advisory.AffectedPackages.Length, advisory.Aliases.Length);
|
||||
mappedCount++;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "CERT-Bund mapping failed for document {DocumentId}", document.Id);
|
||||
await _documentStore.UpdateStatusAsync(document.Id, DocumentStatuses.Failed, cancellationToken).ConfigureAwait(false);
|
||||
pendingMappings.Remove(documentId);
|
||||
_diagnostics.MapFailure("exception");
|
||||
failedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
if (cursor.PendingMappings.Count > 0)
|
||||
{
|
||||
_logger.LogInformation(
|
||||
"CERT-Bund map cycle: mapped {Mapped}, failures {Failures}, remaining pending mappings {PendingMappings}",
|
||||
mappedCount,
|
||||
failedCount,
|
||||
pendingMappings.Count);
|
||||
}
|
||||
|
||||
var updatedCursor = cursor.WithPendingMappings(pendingMappings);
|
||||
await UpdateCursorAsync(updatedCursor, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static double? CalculateCoverageDays(IReadOnlyList<CertBundFeedItem> items, DateTimeOffset fetchedAt)
|
||||
{
|
||||
if (items is null || items.Count == 0)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var oldest = items.Min(static item => item.Published);
|
||||
if (oldest == DateTimeOffset.MinValue)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var span = fetchedAt - oldest;
|
||||
return span >= TimeSpan.Zero ? span.TotalDays : null;
|
||||
}
|
||||
|
||||
private async Task<CertBundCursor> GetCursorAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var state = await _stateRepository.TryGetAsync(SourceName, cancellationToken).ConfigureAwait(false);
|
||||
return state is null ? CertBundCursor.Empty : CertBundCursor.FromBson(state.Cursor);
|
||||
}
|
||||
|
||||
private Task UpdateCursorAsync(CertBundCursor cursor, CancellationToken cancellationToken)
|
||||
{
|
||||
var document = cursor.ToBsonDocument();
|
||||
var completedAt = cursor.LastFetchAt ?? _timeProvider.GetUtcNow();
|
||||
return _stateRepository.UpdateCursorAsync(SourceName, document, completedAt, cancellationToken);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund;
|
||||
|
||||
public sealed class CertBundConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public const string SourceName = "cert-bund";
|
||||
|
||||
public string Name => SourceName;
|
||||
|
||||
public bool IsAvailable(IServiceProvider services)
|
||||
=> services.GetService<CertBundConnector>() is not null;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
return services.GetRequiredService<CertBundConnector>();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
using System;
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.DependencyInjection;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
using StellaOps.Feedser.Source.CertBund.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund;
|
||||
|
||||
public sealed class CertBundDependencyInjectionRoutine : IDependencyInjectionRoutine
|
||||
{
|
||||
private const string ConfigurationSection = "feedser:sources:cert-bund";
|
||||
|
||||
public IServiceCollection Register(IServiceCollection services, IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
services.AddCertBundConnector(options =>
|
||||
{
|
||||
configuration.GetSection(ConfigurationSection).Bind(options);
|
||||
options.Validate();
|
||||
});
|
||||
|
||||
services.AddTransient<CertBundFetchJob>();
|
||||
|
||||
services.PostConfigure<JobSchedulerOptions>(options =>
|
||||
{
|
||||
EnsureJob(options, CertBundJobKinds.Fetch, typeof(CertBundFetchJob));
|
||||
});
|
||||
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void EnsureJob(JobSchedulerOptions options, string kind, Type jobType)
|
||||
{
|
||||
if (options.Definitions.ContainsKey(kind))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
options.Definitions[kind] = new JobDefinition(
|
||||
kind,
|
||||
jobType,
|
||||
options.DefaultTimeout,
|
||||
options.DefaultLeaseDuration,
|
||||
CronExpression: null,
|
||||
Enabled: true);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,48 @@
|
||||
using System;
|
||||
using System.Net;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Feedser.Source.CertBund.Configuration;
|
||||
using StellaOps.Feedser.Source.CertBund.Internal;
|
||||
using StellaOps.Feedser.Source.Common.Html;
|
||||
using StellaOps.Feedser.Source.Common.Http;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund;
|
||||
|
||||
public static class CertBundServiceCollectionExtensions
|
||||
{
|
||||
public static IServiceCollection AddCertBundConnector(this IServiceCollection services, Action<CertBundOptions> configure)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configure);
|
||||
|
||||
services.AddOptions<CertBundOptions>()
|
||||
.Configure(configure)
|
||||
.PostConfigure(static options => options.Validate());
|
||||
|
||||
services.AddSourceHttpClient(CertBundOptions.HttpClientName, static (sp, clientOptions) =>
|
||||
{
|
||||
var options = sp.GetRequiredService<IOptions<CertBundOptions>>().Value;
|
||||
clientOptions.Timeout = options.RequestTimeout;
|
||||
clientOptions.UserAgent = "StellaOps.Feedser.CertBund/1.0";
|
||||
clientOptions.AllowedHosts.Clear();
|
||||
clientOptions.AllowedHosts.Add(options.FeedUri.Host);
|
||||
clientOptions.AllowedHosts.Add(options.DetailApiUri.Host);
|
||||
clientOptions.AllowedHosts.Add(options.PortalBootstrapUri.Host);
|
||||
clientOptions.ConfigureHandler = handler =>
|
||||
{
|
||||
handler.AutomaticDecompression = DecompressionMethods.All;
|
||||
handler.UseCookies = true;
|
||||
handler.CookieContainer = new System.Net.CookieContainer();
|
||||
};
|
||||
});
|
||||
|
||||
services.TryAddSingleton<HtmlContentSanitizer>();
|
||||
services.TryAddSingleton<CertBundDiagnostics>();
|
||||
services.TryAddSingleton<CertBundFeedClient>();
|
||||
services.TryAddSingleton<CertBundDetailParser>();
|
||||
services.AddTransient<CertBundConnector>();
|
||||
return services;
|
||||
}
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Plugin;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund;
|
||||
|
||||
public sealed class CertBundConnectorPlugin : IConnectorPlugin
|
||||
{
|
||||
public string Name => "certbund";
|
||||
|
||||
public bool IsAvailable(IServiceProvider services) => true;
|
||||
|
||||
public IFeedConnector Create(IServiceProvider services) => new StubConnector(Name);
|
||||
|
||||
private sealed class StubConnector : IFeedConnector
|
||||
{
|
||||
public StubConnector(string sourceName) => SourceName = sourceName;
|
||||
|
||||
public string SourceName { get; }
|
||||
|
||||
public Task FetchAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task ParseAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
||||
public Task MapAsync(IServiceProvider services, CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,104 @@
|
||||
using System.Net;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Configuration;
|
||||
|
||||
public sealed class CertBundOptions
|
||||
{
|
||||
public const string HttpClientName = "feedser.source.certbund";
|
||||
|
||||
/// <summary>
|
||||
/// RSS feed providing the latest CERT-Bund advisories.
|
||||
/// </summary>
|
||||
public Uri FeedUri { get; set; } = new("https://wid.cert-bund.de/content/public/securityAdvisory/rss");
|
||||
|
||||
/// <summary>
|
||||
/// Portal endpoint used to bootstrap session cookies (required for the SPA JSON API).
|
||||
/// </summary>
|
||||
public Uri PortalBootstrapUri { get; set; } = new("https://wid.cert-bund.de/portal/");
|
||||
|
||||
/// <summary>
|
||||
/// Detail API endpoint template; advisory identifier is appended as the <c>name</c> query parameter.
|
||||
/// </summary>
|
||||
public Uri DetailApiUri { get; set; } = new("https://wid.cert-bund.de/portal/api/securityadvisory");
|
||||
|
||||
/// <summary>
|
||||
/// Optional timeout override for feed/detail requests.
|
||||
/// </summary>
|
||||
public TimeSpan RequestTimeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
/// <summary>
|
||||
/// Delay applied between successive detail fetches to respect upstream politeness.
|
||||
/// </summary>
|
||||
public TimeSpan RequestDelay { get; set; } = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
/// <summary>
|
||||
/// Backoff recorded in source state when a fetch attempt fails.
|
||||
/// </summary>
|
||||
public TimeSpan FailureBackoff { get; set; } = TimeSpan.FromMinutes(5);
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of advisories to enqueue per fetch iteration.
|
||||
/// </summary>
|
||||
public int MaxAdvisoriesPerFetch { get; set; } = 50;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of advisory identifiers remembered to prevent re-processing.
|
||||
/// </summary>
|
||||
public int MaxKnownAdvisories { get; set; } = 512;
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (FeedUri is null || !FeedUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("CERT-Bund feed URI must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (PortalBootstrapUri is null || !PortalBootstrapUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("CERT-Bund portal bootstrap URI must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (DetailApiUri is null || !DetailApiUri.IsAbsoluteUri)
|
||||
{
|
||||
throw new InvalidOperationException("CERT-Bund detail API URI must be an absolute URI.");
|
||||
}
|
||||
|
||||
if (RequestTimeout <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(RequestTimeout)} must be positive.");
|
||||
}
|
||||
|
||||
if (RequestDelay < TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(RequestDelay)} cannot be negative.");
|
||||
}
|
||||
|
||||
if (FailureBackoff <= TimeSpan.Zero)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(FailureBackoff)} must be positive.");
|
||||
}
|
||||
|
||||
if (MaxAdvisoriesPerFetch <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(MaxAdvisoriesPerFetch)} must be greater than zero.");
|
||||
}
|
||||
|
||||
if (MaxKnownAdvisories <= 0)
|
||||
{
|
||||
throw new InvalidOperationException($"{nameof(MaxKnownAdvisories)} must be greater than zero.");
|
||||
}
|
||||
}
|
||||
|
||||
public Uri BuildDetailUri(string advisoryId)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(advisoryId))
|
||||
{
|
||||
throw new ArgumentException("Advisory identifier must be provided.", nameof(advisoryId));
|
||||
}
|
||||
|
||||
var builder = new UriBuilder(DetailApiUri);
|
||||
var queryPrefix = string.IsNullOrEmpty(builder.Query) ? string.Empty : builder.Query.TrimStart('?') + "&";
|
||||
builder.Query = $"{queryPrefix}name={Uri.EscapeDataString(advisoryId)}";
|
||||
return builder.Uri;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
public sealed record CertBundAdvisoryDto
|
||||
{
|
||||
[JsonPropertyName("advisoryId")]
|
||||
public string AdvisoryId { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string Title { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("contentHtml")]
|
||||
public string ContentHtml { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("severity")]
|
||||
public string? Severity { get; init; }
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; init; } = "de";
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("modified")]
|
||||
public DateTimeOffset? Modified { get; init; }
|
||||
|
||||
[JsonPropertyName("portalUri")]
|
||||
public Uri PortalUri { get; init; } = new("https://wid.cert-bund.de/");
|
||||
|
||||
[JsonPropertyName("detailUri")]
|
||||
public Uri DetailUri { get; init; } = new("https://wid.cert-bund.de/");
|
||||
|
||||
[JsonPropertyName("cveIds")]
|
||||
public IReadOnlyList<string> CveIds { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("products")]
|
||||
public IReadOnlyList<CertBundProductDto> Products { get; init; } = Array.Empty<CertBundProductDto>();
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public IReadOnlyList<CertBundReferenceDto> References { get; init; } = Array.Empty<CertBundReferenceDto>();
|
||||
}
|
||||
|
||||
public sealed record CertBundProductDto
|
||||
{
|
||||
[JsonPropertyName("vendor")]
|
||||
public string? Vendor { get; init; }
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; init; }
|
||||
|
||||
[JsonPropertyName("versions")]
|
||||
public string? Versions { get; init; }
|
||||
}
|
||||
|
||||
public sealed record CertBundReferenceDto
|
||||
{
|
||||
[JsonPropertyName("url")]
|
||||
public string Url { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("label")]
|
||||
public string? Label { get; init; }
|
||||
}
|
||||
118
src/StellaOps.Feedser.Source.CertBund/Internal/CertBundCursor.cs
Normal file
118
src/StellaOps.Feedser.Source.CertBund/Internal/CertBundCursor.cs
Normal file
@@ -0,0 +1,118 @@
|
||||
using System;
|
||||
using System.Linq;
|
||||
using MongoDB.Bson;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
internal sealed record CertBundCursor(
|
||||
IReadOnlyCollection<Guid> PendingDocuments,
|
||||
IReadOnlyCollection<Guid> PendingMappings,
|
||||
IReadOnlyCollection<string> KnownAdvisories,
|
||||
DateTimeOffset? LastPublished,
|
||||
DateTimeOffset? LastFetchAt)
|
||||
{
|
||||
private static readonly IReadOnlyCollection<Guid> EmptyGuids = Array.Empty<Guid>();
|
||||
private static readonly IReadOnlyCollection<string> EmptyStrings = Array.Empty<string>();
|
||||
|
||||
public static CertBundCursor Empty { get; } = new(EmptyGuids, EmptyGuids, EmptyStrings, null, null);
|
||||
|
||||
public CertBundCursor WithPendingDocuments(IEnumerable<Guid> documents)
|
||||
=> this with { PendingDocuments = Distinct(documents) };
|
||||
|
||||
public CertBundCursor WithPendingMappings(IEnumerable<Guid> mappings)
|
||||
=> this with { PendingMappings = Distinct(mappings) };
|
||||
|
||||
public CertBundCursor WithKnownAdvisories(IEnumerable<string> advisories)
|
||||
=> this with { KnownAdvisories = advisories?.Distinct(StringComparer.OrdinalIgnoreCase).ToArray() ?? EmptyStrings };
|
||||
|
||||
public CertBundCursor WithLastPublished(DateTimeOffset? published)
|
||||
=> this with { LastPublished = published };
|
||||
|
||||
public CertBundCursor WithLastFetch(DateTimeOffset? timestamp)
|
||||
=> this with { LastFetchAt = timestamp };
|
||||
|
||||
public BsonDocument ToBsonDocument()
|
||||
{
|
||||
var document = new BsonDocument
|
||||
{
|
||||
["pendingDocuments"] = new BsonArray(PendingDocuments.Select(id => id.ToString())),
|
||||
["pendingMappings"] = new BsonArray(PendingMappings.Select(id => id.ToString())),
|
||||
["knownAdvisories"] = new BsonArray(KnownAdvisories),
|
||||
};
|
||||
|
||||
if (LastPublished.HasValue)
|
||||
{
|
||||
document["lastPublished"] = LastPublished.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
if (LastFetchAt.HasValue)
|
||||
{
|
||||
document["lastFetchAt"] = LastFetchAt.Value.UtcDateTime;
|
||||
}
|
||||
|
||||
return document;
|
||||
}
|
||||
|
||||
public static CertBundCursor FromBson(BsonDocument? document)
|
||||
{
|
||||
if (document is null || document.ElementCount == 0)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var pendingDocuments = ReadGuidArray(document, "pendingDocuments");
|
||||
var pendingMappings = ReadGuidArray(document, "pendingMappings");
|
||||
var knownAdvisories = ReadStringArray(document, "knownAdvisories");
|
||||
var lastPublished = document.TryGetValue("lastPublished", out var publishedValue)
|
||||
? ParseDate(publishedValue)
|
||||
: null;
|
||||
var lastFetch = document.TryGetValue("lastFetchAt", out var fetchValue)
|
||||
? ParseDate(fetchValue)
|
||||
: null;
|
||||
|
||||
return new CertBundCursor(pendingDocuments, pendingMappings, knownAdvisories, lastPublished, lastFetch);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid> Distinct(IEnumerable<Guid>? values)
|
||||
=> values?.Distinct().ToArray() ?? EmptyGuids;
|
||||
|
||||
private static IReadOnlyCollection<Guid> ReadGuidArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyGuids;
|
||||
}
|
||||
|
||||
var items = new List<Guid>(array.Count);
|
||||
foreach (var element in array)
|
||||
{
|
||||
if (Guid.TryParse(element?.ToString(), out var id))
|
||||
{
|
||||
items.Add(id);
|
||||
}
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ReadStringArray(BsonDocument document, string field)
|
||||
{
|
||||
if (!document.TryGetValue(field, out var value) || value is not BsonArray array)
|
||||
{
|
||||
return EmptyStrings;
|
||||
}
|
||||
|
||||
return array.Select(element => element?.ToString() ?? string.Empty)
|
||||
.Where(static s => !string.IsNullOrWhiteSpace(s))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseDate(BsonValue value)
|
||||
=> value.BsonType switch
|
||||
{
|
||||
BsonType.DateTime => DateTime.SpecifyKind(value.ToUniversalTime(), DateTimeKind.Utc),
|
||||
BsonType.String when DateTimeOffset.TryParse(value.AsString, out var parsed) => parsed.ToUniversalTime(),
|
||||
_ => null,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using StellaOps.Feedser.Source.Common.Html;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
public sealed class CertBundDetailParser
|
||||
{
|
||||
private static readonly JsonSerializerOptions SerializerOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
private readonly HtmlContentSanitizer _sanitizer;
|
||||
|
||||
public CertBundDetailParser(HtmlContentSanitizer sanitizer)
|
||||
=> _sanitizer = sanitizer ?? throw new ArgumentNullException(nameof(sanitizer));
|
||||
|
||||
public CertBundAdvisoryDto Parse(Uri detailUri, Uri portalUri, byte[] payload)
|
||||
{
|
||||
var detail = JsonSerializer.Deserialize<CertBundDetailResponse>(payload, SerializerOptions)
|
||||
?? throw new InvalidOperationException("CERT-Bund detail payload deserialized to null.");
|
||||
|
||||
var advisoryId = detail.Name ?? throw new InvalidOperationException("CERT-Bund detail missing advisory name.");
|
||||
var contentHtml = _sanitizer.Sanitize(detail.Description ?? string.Empty, portalUri);
|
||||
|
||||
return new CertBundAdvisoryDto
|
||||
{
|
||||
AdvisoryId = advisoryId,
|
||||
Title = detail.Title ?? advisoryId,
|
||||
Summary = detail.Summary,
|
||||
ContentHtml = contentHtml,
|
||||
Severity = detail.Severity,
|
||||
Language = string.IsNullOrWhiteSpace(detail.Language) ? "de" : detail.Language!,
|
||||
Published = detail.Published,
|
||||
Modified = detail.Updated ?? detail.Published,
|
||||
PortalUri = portalUri,
|
||||
DetailUri = detailUri,
|
||||
CveIds = detail.CveIds?.Where(static id => !string.IsNullOrWhiteSpace(id))
|
||||
.Select(static id => id!.Trim())
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray() ?? Array.Empty<string>(),
|
||||
References = MapReferences(detail.References),
|
||||
Products = MapProducts(detail.Products),
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CertBundReferenceDto> MapReferences(CertBundDetailReference[]? references)
|
||||
{
|
||||
if (references is null || references.Length == 0)
|
||||
{
|
||||
return Array.Empty<CertBundReferenceDto>();
|
||||
}
|
||||
|
||||
return references
|
||||
.Where(static reference => !string.IsNullOrWhiteSpace(reference.Url))
|
||||
.Select(reference => new CertBundReferenceDto
|
||||
{
|
||||
Url = reference.Url!,
|
||||
Label = reference.Label,
|
||||
})
|
||||
.DistinctBy(static reference => reference.Url, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<CertBundProductDto> MapProducts(CertBundDetailProduct[]? products)
|
||||
{
|
||||
if (products is null || products.Length == 0)
|
||||
{
|
||||
return Array.Empty<CertBundProductDto>();
|
||||
}
|
||||
|
||||
return products
|
||||
.Where(static product => !string.IsNullOrWhiteSpace(product.Vendor) || !string.IsNullOrWhiteSpace(product.Name))
|
||||
.Select(product => new CertBundProductDto
|
||||
{
|
||||
Vendor = product.Vendor,
|
||||
Name = product.Name,
|
||||
Versions = product.Versions,
|
||||
})
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
internal sealed record CertBundDetailResponse
|
||||
{
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; init; }
|
||||
|
||||
[JsonPropertyName("title")]
|
||||
public string? Title { get; init; }
|
||||
|
||||
[JsonPropertyName("summary")]
|
||||
public string? Summary { get; init; }
|
||||
|
||||
[JsonPropertyName("description")]
|
||||
public string? Description { get; init; }
|
||||
|
||||
[JsonPropertyName("severity")]
|
||||
public string? Severity { get; init; }
|
||||
|
||||
[JsonPropertyName("language")]
|
||||
public string? Language { get; init; }
|
||||
|
||||
[JsonPropertyName("published")]
|
||||
public DateTimeOffset? Published { get; init; }
|
||||
|
||||
[JsonPropertyName("updated")]
|
||||
public DateTimeOffset? Updated { get; init; }
|
||||
|
||||
[JsonPropertyName("cveIds")]
|
||||
public string[]? CveIds { get; init; }
|
||||
|
||||
[JsonPropertyName("references")]
|
||||
public CertBundDetailReference[]? References { get; init; }
|
||||
|
||||
[JsonPropertyName("products")]
|
||||
public CertBundDetailProduct[]? Products { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record CertBundDetailReference
|
||||
{
|
||||
[JsonPropertyName("url")]
|
||||
public string? Url { get; init; }
|
||||
|
||||
[JsonPropertyName("label")]
|
||||
public string? Label { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record CertBundDetailProduct
|
||||
{
|
||||
[JsonPropertyName("vendor")]
|
||||
public string? Vendor { get; init; }
|
||||
|
||||
[JsonPropertyName("name")]
|
||||
public string? Name { get; init; }
|
||||
|
||||
[JsonPropertyName("versions")]
|
||||
public string? Versions { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Metrics;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
/// <summary>
|
||||
/// Emits OpenTelemetry counters and histograms for the CERT-Bund connector.
|
||||
/// </summary>
|
||||
public sealed class CertBundDiagnostics : IDisposable
|
||||
{
|
||||
private const string MeterName = "StellaOps.Feedser.Source.CertBund";
|
||||
private const string MeterVersion = "1.0.0";
|
||||
|
||||
private readonly Meter _meter;
|
||||
private readonly Counter<long> _feedFetchAttempts;
|
||||
private readonly Counter<long> _feedFetchSuccess;
|
||||
private readonly Counter<long> _feedFetchFailures;
|
||||
private readonly Histogram<long> _feedItemCount;
|
||||
private readonly Histogram<long> _feedEnqueuedCount;
|
||||
private readonly Histogram<double> _feedCoverageDays;
|
||||
private readonly Counter<long> _detailFetchAttempts;
|
||||
private readonly Counter<long> _detailFetchSuccess;
|
||||
private readonly Counter<long> _detailFetchNotModified;
|
||||
private readonly Counter<long> _detailFetchFailures;
|
||||
private readonly Counter<long> _parseSuccess;
|
||||
private readonly Counter<long> _parseFailures;
|
||||
private readonly Histogram<long> _parseProductCount;
|
||||
private readonly Histogram<long> _parseCveCount;
|
||||
private readonly Counter<long> _mapSuccess;
|
||||
private readonly Counter<long> _mapFailures;
|
||||
private readonly Histogram<long> _mapPackageCount;
|
||||
private readonly Histogram<long> _mapAliasCount;
|
||||
|
||||
public CertBundDiagnostics()
|
||||
{
|
||||
_meter = new Meter(MeterName, MeterVersion);
|
||||
_feedFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certbund.feed.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of RSS feed load attempts.");
|
||||
_feedFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certbund.feed.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of successful RSS feed loads.");
|
||||
_feedFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certbund.feed.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of RSS feed load failures.");
|
||||
_feedItemCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.feed.items.count",
|
||||
unit: "items",
|
||||
description: "Distribution of RSS item counts per fetch.");
|
||||
_feedEnqueuedCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.feed.enqueued.count",
|
||||
unit: "documents",
|
||||
description: "Distribution of advisory documents enqueued per fetch.");
|
||||
_feedCoverageDays = _meter.CreateHistogram<double>(
|
||||
name: "certbund.feed.coverage.days",
|
||||
unit: "days",
|
||||
description: "Coverage window in days between fetch time and the oldest published advisory in the feed.");
|
||||
_detailFetchAttempts = _meter.CreateCounter<long>(
|
||||
name: "certbund.detail.fetch.attempts",
|
||||
unit: "operations",
|
||||
description: "Number of detail fetch attempts.");
|
||||
_detailFetchSuccess = _meter.CreateCounter<long>(
|
||||
name: "certbund.detail.fetch.success",
|
||||
unit: "operations",
|
||||
description: "Number of detail fetches that persisted a document.");
|
||||
_detailFetchNotModified = _meter.CreateCounter<long>(
|
||||
name: "certbund.detail.fetch.not_modified",
|
||||
unit: "operations",
|
||||
description: "Number of detail fetches returning HTTP 304.");
|
||||
_detailFetchFailures = _meter.CreateCounter<long>(
|
||||
name: "certbund.detail.fetch.failures",
|
||||
unit: "operations",
|
||||
description: "Number of detail fetches that failed.");
|
||||
_parseSuccess = _meter.CreateCounter<long>(
|
||||
name: "certbund.parse.success",
|
||||
unit: "documents",
|
||||
description: "Number of documents parsed into CERT-Bund DTOs.");
|
||||
_parseFailures = _meter.CreateCounter<long>(
|
||||
name: "certbund.parse.failures",
|
||||
unit: "documents",
|
||||
description: "Number of documents that failed to parse.");
|
||||
_parseProductCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.parse.products.count",
|
||||
unit: "products",
|
||||
description: "Distribution of product entries captured per advisory.");
|
||||
_parseCveCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.parse.cve.count",
|
||||
unit: "aliases",
|
||||
description: "Distribution of CVE identifiers captured per advisory.");
|
||||
_mapSuccess = _meter.CreateCounter<long>(
|
||||
name: "certbund.map.success",
|
||||
unit: "advisories",
|
||||
description: "Number of canonical advisories emitted by the mapper.");
|
||||
_mapFailures = _meter.CreateCounter<long>(
|
||||
name: "certbund.map.failures",
|
||||
unit: "advisories",
|
||||
description: "Number of mapping failures.");
|
||||
_mapPackageCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.map.affected.count",
|
||||
unit: "packages",
|
||||
description: "Distribution of affected packages emitted per advisory.");
|
||||
_mapAliasCount = _meter.CreateHistogram<long>(
|
||||
name: "certbund.map.aliases.count",
|
||||
unit: "aliases",
|
||||
description: "Distribution of alias counts per advisory.");
|
||||
}
|
||||
|
||||
public void FeedFetchAttempt() => _feedFetchAttempts.Add(1);
|
||||
|
||||
public void FeedFetchSuccess(int itemCount)
|
||||
{
|
||||
_feedFetchSuccess.Add(1);
|
||||
if (itemCount >= 0)
|
||||
{
|
||||
_feedItemCount.Record(itemCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void FeedFetchFailure(string reason = "error")
|
||||
=> _feedFetchFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void RecordFeedCoverage(double? coverageDays)
|
||||
{
|
||||
if (coverageDays is { } days && days >= 0)
|
||||
{
|
||||
_feedCoverageDays.Record(days);
|
||||
}
|
||||
}
|
||||
|
||||
public void DetailFetchAttempt() => _detailFetchAttempts.Add(1);
|
||||
|
||||
public void DetailFetchSuccess() => _detailFetchSuccess.Add(1);
|
||||
|
||||
public void DetailFetchNotModified() => _detailFetchNotModified.Add(1);
|
||||
|
||||
public void DetailFetchFailure(string reason = "error")
|
||||
=> _detailFetchFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void DetailFetchEnqueued(int count)
|
||||
{
|
||||
if (count >= 0)
|
||||
{
|
||||
_feedEnqueuedCount.Record(count);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseSuccess(int productCount, int cveCount)
|
||||
{
|
||||
_parseSuccess.Add(1);
|
||||
|
||||
if (productCount >= 0)
|
||||
{
|
||||
_parseProductCount.Record(productCount);
|
||||
}
|
||||
|
||||
if (cveCount >= 0)
|
||||
{
|
||||
_parseCveCount.Record(cveCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void ParseFailure(string reason = "error")
|
||||
=> _parseFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
public void MapSuccess(int affectedPackages, int aliasCount)
|
||||
{
|
||||
_mapSuccess.Add(1);
|
||||
|
||||
if (affectedPackages >= 0)
|
||||
{
|
||||
_mapPackageCount.Record(affectedPackages);
|
||||
}
|
||||
|
||||
if (aliasCount >= 0)
|
||||
{
|
||||
_mapAliasCount.Record(aliasCount);
|
||||
}
|
||||
}
|
||||
|
||||
public void MapFailure(string reason = "error")
|
||||
=> _mapFailures.Add(1, ReasonTag(reason));
|
||||
|
||||
private static KeyValuePair<string, object?> ReasonTag(string reason)
|
||||
=> new("reason", string.IsNullOrWhiteSpace(reason) ? "unknown" : reason.ToLowerInvariant());
|
||||
|
||||
public void Dispose() => _meter.Dispose();
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
internal static class CertBundDocumentMetadata
|
||||
{
|
||||
public static Dictionary<string, string> CreateMetadata(CertBundFeedItem item)
|
||||
{
|
||||
var metadata = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["certbund.advisoryId"] = item.AdvisoryId,
|
||||
["certbund.portalUri"] = item.PortalUri.ToString(),
|
||||
["certbund.published"] = item.Published.ToString("O"),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(item.Category))
|
||||
{
|
||||
metadata["certbund.category"] = item.Category!;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(item.Title))
|
||||
{
|
||||
metadata["certbund.title"] = item.Title!;
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Globalization;
|
||||
using System.Linq;
|
||||
using System.Net.Http;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Xml.Linq;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Feedser.Source.CertBund.Configuration;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
public sealed class CertBundFeedClient
|
||||
{
|
||||
private readonly IHttpClientFactory _httpClientFactory;
|
||||
private readonly CertBundOptions _options;
|
||||
private readonly ILogger<CertBundFeedClient> _logger;
|
||||
private readonly SemaphoreSlim _bootstrapSemaphore = new(1, 1);
|
||||
private volatile bool _bootstrapped;
|
||||
|
||||
public CertBundFeedClient(
|
||||
IHttpClientFactory httpClientFactory,
|
||||
IOptions<CertBundOptions> options,
|
||||
ILogger<CertBundFeedClient> logger)
|
||||
{
|
||||
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
|
||||
_options = (options ?? throw new ArgumentNullException(nameof(options))).Value ?? throw new ArgumentNullException(nameof(options));
|
||||
_options.Validate();
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CertBundFeedItem>> LoadAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var client = _httpClientFactory.CreateClient(CertBundOptions.HttpClientName);
|
||||
await EnsureSessionAsync(client, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, _options.FeedUri);
|
||||
request.Headers.TryAddWithoutValidation("Accept", "application/rss+xml, application/xml;q=0.9, text/xml;q=0.8");
|
||||
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
await using var stream = await response.Content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false);
|
||||
var document = XDocument.Load(stream);
|
||||
|
||||
var items = new List<CertBundFeedItem>();
|
||||
foreach (var element in document.Descendants("item"))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var linkValue = element.Element("link")?.Value?.Trim();
|
||||
if (string.IsNullOrWhiteSpace(linkValue) || !Uri.TryCreate(linkValue, UriKind.Absolute, out var portalUri))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var advisoryId = TryExtractNameParameter(portalUri);
|
||||
if (string.IsNullOrWhiteSpace(advisoryId))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var detailUri = _options.BuildDetailUri(advisoryId);
|
||||
var pubDateText = element.Element("pubDate")?.Value;
|
||||
var published = ParseDate(pubDateText);
|
||||
var title = element.Element("title")?.Value?.Trim();
|
||||
var category = element.Element("category")?.Value?.Trim();
|
||||
|
||||
items.Add(new CertBundFeedItem(advisoryId, detailUri, portalUri, published, title, category));
|
||||
}
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
private async Task EnsureSessionAsync(HttpClient client, CancellationToken cancellationToken)
|
||||
{
|
||||
if (_bootstrapped)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
await _bootstrapSemaphore.WaitAsync(cancellationToken).ConfigureAwait(false);
|
||||
try
|
||||
{
|
||||
if (_bootstrapped)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Get, _options.PortalBootstrapUri);
|
||||
request.Headers.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
|
||||
using var response = await client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false);
|
||||
response.EnsureSuccessStatusCode();
|
||||
|
||||
_bootstrapped = true;
|
||||
}
|
||||
finally
|
||||
{
|
||||
_bootstrapSemaphore.Release();
|
||||
}
|
||||
}
|
||||
|
||||
private static string? TryExtractNameParameter(Uri portalUri)
|
||||
{
|
||||
if (portalUri is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var query = portalUri.Query;
|
||||
if (string.IsNullOrEmpty(query))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = query.TrimStart('?');
|
||||
foreach (var pair in trimmed.Split('&', StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var separatorIndex = pair.IndexOf('=');
|
||||
if (separatorIndex <= 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var key = pair[..separatorIndex].Trim();
|
||||
if (!key.Equals("name", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var value = pair[(separatorIndex + 1)..];
|
||||
return Uri.UnescapeDataString(value);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static DateTimeOffset ParseDate(string? value)
|
||||
=> DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var parsed)
|
||||
? parsed
|
||||
: DateTimeOffset.UtcNow;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
using System;
|
||||
|
||||
public sealed record CertBundFeedItem(
|
||||
string AdvisoryId,
|
||||
Uri DetailUri,
|
||||
Uri PortalUri,
|
||||
DateTimeOffset Published,
|
||||
string? Title,
|
||||
string? Category);
|
||||
168
src/StellaOps.Feedser.Source.CertBund/Internal/CertBundMapper.cs
Normal file
168
src/StellaOps.Feedser.Source.CertBund/Internal/CertBundMapper.cs
Normal file
@@ -0,0 +1,168 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using StellaOps.Feedser.Models;
|
||||
using StellaOps.Feedser.Storage.Mongo.Documents;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund.Internal;
|
||||
|
||||
internal static class CertBundMapper
|
||||
{
|
||||
public static Advisory Map(CertBundAdvisoryDto dto, DocumentRecord document, DateTimeOffset recordedAt)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(dto);
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
var aliases = BuildAliases(dto);
|
||||
var references = BuildReferences(dto, recordedAt);
|
||||
var packages = BuildPackages(dto, recordedAt);
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"advisory",
|
||||
dto.AdvisoryId,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory });
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey: dto.AdvisoryId,
|
||||
title: dto.Title,
|
||||
summary: dto.Summary,
|
||||
language: dto.Language?.ToLowerInvariant() ?? "de",
|
||||
published: dto.Published,
|
||||
modified: dto.Modified,
|
||||
severity: MapSeverity(dto.Severity),
|
||||
exploitKnown: false,
|
||||
aliases: aliases,
|
||||
references: references,
|
||||
affectedPackages: packages,
|
||||
cvssMetrics: Array.Empty<CvssMetric>(),
|
||||
provenance: new[] { provenance });
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> BuildAliases(CertBundAdvisoryDto dto)
|
||||
{
|
||||
var aliases = new List<string>(capacity: 4) { dto.AdvisoryId };
|
||||
foreach (var cve in dto.CveIds)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(cve))
|
||||
{
|
||||
aliases.Add(cve);
|
||||
}
|
||||
}
|
||||
|
||||
return aliases
|
||||
.Where(static alias => !string.IsNullOrWhiteSpace(alias))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static alias => alias, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AdvisoryReference> BuildReferences(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
var references = new List<AdvisoryReference>
|
||||
{
|
||||
new(dto.DetailUri.ToString(), "details", "cert-bund", null, new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
dto.DetailUri.ToString(),
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References }))
|
||||
};
|
||||
|
||||
foreach (var reference in dto.References)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(reference.Url))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
reference.Url,
|
||||
kind: "reference",
|
||||
sourceTag: "cert-bund",
|
||||
summary: reference.Label,
|
||||
provenance: new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"reference",
|
||||
reference.Url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References })));
|
||||
}
|
||||
|
||||
return references
|
||||
.DistinctBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.OrderBy(static reference => reference.Url, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IReadOnlyList<AffectedPackage> BuildPackages(CertBundAdvisoryDto dto, DateTimeOffset recordedAt)
|
||||
{
|
||||
if (dto.Products.Count == 0)
|
||||
{
|
||||
return Array.Empty<AffectedPackage>();
|
||||
}
|
||||
|
||||
var packages = new List<AffectedPackage>(dto.Products.Count);
|
||||
foreach (var product in dto.Products)
|
||||
{
|
||||
var vendor = Validation.TrimToNull(product.Vendor) ?? "Unspecified";
|
||||
var name = Validation.TrimToNull(product.Name);
|
||||
var identifier = name is null ? vendor : $"{vendor} {name}";
|
||||
|
||||
var provenance = new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"package",
|
||||
identifier,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.AffectedPackages });
|
||||
|
||||
var ranges = string.IsNullOrWhiteSpace(product.Versions)
|
||||
? Array.Empty<AffectedVersionRange>()
|
||||
: new[]
|
||||
{
|
||||
new AffectedVersionRange(
|
||||
rangeKind: "string",
|
||||
introducedVersion: null,
|
||||
fixedVersion: null,
|
||||
lastAffectedVersion: null,
|
||||
rangeExpression: product.Versions,
|
||||
provenance: new AdvisoryProvenance(
|
||||
CertBundConnectorPlugin.SourceName,
|
||||
"package-range",
|
||||
product.Versions,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.VersionRanges }))
|
||||
};
|
||||
|
||||
packages.Add(new AffectedPackage(
|
||||
AffectedPackageTypes.Vendor,
|
||||
identifier,
|
||||
platform: null,
|
||||
versionRanges: ranges,
|
||||
statuses: Array.Empty<AffectedPackageStatus>(),
|
||||
provenance: new[] { provenance },
|
||||
normalizedVersions: Array.Empty<NormalizedVersionRule>()));
|
||||
}
|
||||
|
||||
return packages
|
||||
.DistinctBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static package => package.Identifier, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static string? MapSeverity(string? severity)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(severity))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return severity.ToLowerInvariant() switch
|
||||
{
|
||||
"hoch" or "high" => "high",
|
||||
"mittel" or "medium" => "medium",
|
||||
"gering" or "low" => "low",
|
||||
_ => severity.ToLowerInvariant(),
|
||||
};
|
||||
}
|
||||
}
|
||||
22
src/StellaOps.Feedser.Source.CertBund/Jobs.cs
Normal file
22
src/StellaOps.Feedser.Source.CertBund/Jobs.cs
Normal file
@@ -0,0 +1,22 @@
|
||||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Feedser.Core.Jobs;
|
||||
|
||||
namespace StellaOps.Feedser.Source.CertBund;
|
||||
|
||||
internal static class CertBundJobKinds
|
||||
{
|
||||
public const string Fetch = "source:cert-bund:fetch";
|
||||
}
|
||||
|
||||
internal sealed class CertBundFetchJob : IJob
|
||||
{
|
||||
private readonly CertBundConnector _connector;
|
||||
|
||||
public CertBundFetchJob(CertBundConnector connector)
|
||||
=> _connector = connector ?? throw new ArgumentNullException(nameof(connector));
|
||||
|
||||
public Task ExecuteAsync(JobExecutionContext context, CancellationToken cancellationToken)
|
||||
=> _connector.FetchAsync(context.Services, cancellationToken);
|
||||
}
|
||||
39
src/StellaOps.Feedser.Source.CertBund/README.md
Normal file
39
src/StellaOps.Feedser.Source.CertBund/README.md
Normal file
@@ -0,0 +1,39 @@
|
||||
# CERT-Bund Security Advisories – Connector Notes
|
||||
|
||||
## Publication endpoints
|
||||
- **RSS feed (latest 250 advisories)** – `https://wid.cert-bund.de/content/public/securityAdvisory/rss`. The feed refreshes quickly; the current window spans roughly 6 days of activity, so fetch jobs must run frequently to avoid churn.
|
||||
- **Portal bootstrap** – `https://wid.cert-bund.de/portal/` is hit once per process start to prime the session (`client_config` cookie) before any API calls.
|
||||
- **Detail API** – `https://wid.cert-bund.de/portal/api/securityadvisory?name=<ID>`. The connector reuses the bootstrapped `SocketsHttpHandler` so cookies and headers match the Angular SPA. Manual reproduction requires the same cookie container; otherwise the endpoint responds with the shell HTML document.
|
||||
|
||||
## Telemetry
|
||||
The OpenTelemetry meter is `StellaOps.Feedser.Source.CertBund`. Key instruments:
|
||||
|
||||
| Metric | Type | Notes |
|
||||
| --- | --- | --- |
|
||||
| `certbund.feed.fetch.attempts` / `.success` / `.failures` | counter | Feed poll lifecycle. |
|
||||
| `certbund.feed.items.count` | histogram | Items returned per RSS fetch. |
|
||||
| `certbund.feed.enqueued.count` | histogram | Detail documents queued per cycle (post-dedupe, before truncation). |
|
||||
| `certbund.feed.coverage.days` | histogram | Rolling window (fetch time − oldest published entry). Useful to alert when feed depth contracts. |
|
||||
| `certbund.detail.fetch.*` | counter | Attempts, successes, HTTP 304, and failure counts; failures are tagged by reason (`skipped`, `exception`). |
|
||||
| `certbund.parse.success` / `.failures` | counter | Parsing outcomes; histograms capture product and CVE counts. |
|
||||
| `certbund.map.success` / `.failures` | counter | Canonical mapping results; histograms capture affected-package and alias volume. |
|
||||
|
||||
Dashboards should chart coverage days and enqueued counts alongside fetch failures: sharp drops indicate the upstream window tightened or parsing stalled.
|
||||
|
||||
## Logging signals
|
||||
- `CERT-Bund fetch cycle: feed items …` summarises each RSS run (enqueued, already-known, HTTP 304, failures, coverage window).
|
||||
- Parse and map stages log corresponding counts when work remains in the cursor.
|
||||
- Errors include advisory/document identifiers to simplify replays.
|
||||
|
||||
## Historical coverage
|
||||
- RSS contains the newest **250** items (≈6 days at the current publication rate). The connector prunes the “known advisory” set to 512 IDs to avoid unbounded memory but retains enough headroom for short-term replay.
|
||||
- Older advisories remain accessible through the same detail API (`WID-SEC-<year>-<sequence>` identifiers). For deep backfills run a scripted sweep that queues historical IDs in descending order; the connector will persist any payloads that still resolve. Document these batches under source state comments so Merge/Docs can track provenance.
|
||||
|
||||
## Locale & translation stance
|
||||
- CERT-Bund publishes advisory titles and summaries **only in German** (language tag `de`). The connector preserves original casing/content and sets `Advisory.Language = "de"`.
|
||||
- Operator guidance:
|
||||
1. Front-line analysts consuming Feedser data should maintain German literacy or rely on approved machine-translation pipelines.
|
||||
2. When mirroring advisories into English dashboards, store translations outside the canonical advisory payload to keep determinism. Suggested approach: create an auxiliary collection keyed by advisory ID with timestamped translated snippets.
|
||||
3. Offline Kit bundles must document that CERT-Bund content is untranslated to avoid surprise during audits.
|
||||
|
||||
The Docs guild will surface the translation policy (retain German source, optionally layer operator-provided translations) in the broader i18n section; this README is the connector-level reference.
|
||||
@@ -6,11 +6,10 @@
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Plugin/StellaOps.Plugin.csproj" />
|
||||
|
||||
<ProjectReference Include="../StellaOps.Feedser.Source.Common/StellaOps.Feedser.Source.Common.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Feedser.Models/StellaOps.Feedser.Models.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user