stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

audit notes work completed, test fixes work (95% done), new sprints, new data sources setup and configuration

Browse Source
This commit is contained in:
master
2026-01-14 10:48:00 +02:00
parent d7be6ba34b
commit 95d5898650
379 changed files with 40695 additions and 19041 deletions
Download Patch File Download Diff File Expand all files Collapse all files

308
src/Concelier/__Libraries/StellaOps.Concelier.Core/Configuration/MirrorRateLimitConfig.cs Normal file
View File

@@ -0,0 +1,308 @@
// -----------------------------------------------------------------------------
// MirrorRateLimitConfig.cs
// Sprint: SPRINT_20260114_SOURCES_SETUP
// Task: 3.1 - Rate Limit Configuration Models
// Description: Rate limiting configuration for mirror server endpoints.
// Maps to Router library's RateLimitConfig structure.
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.Configuration;
/// <summary>
/// Rate limiting configuration for mirror server endpoints.
/// This maps to Router library's <c>RateLimitConfig</c> for Gateway execution.
/// </summary>
/// <remarks>
/// Configuration example:
/// <code>
/// mirrorServer:
/// rateLimits:
/// forInstance:
/// perSeconds: 60
/// maxRequests: 100
/// forEnvironment:
/// valkeyConnection: "localhost:6379"
/// perSeconds: 3600
/// maxRequests: 10000
/// routes:
/// index:
/// pattern: "/api/mirror/index"
/// perSeconds: 60
/// maxRequests: 60
/// bundle:
/// pattern: "/api/mirror/bundle/*"
/// perSeconds: 60
/// maxRequests: 600
/// </code>
/// </remarks>
public sealed record MirrorRateLimitConfig
{
/// <summary>
/// Instance-level rate limits (in-memory, per-process).
/// Maps to Router's <c>rate_limiting.for_instance</c>.
/// </summary>
public InstanceRateLimitConfig? ForInstance { get; init; }
/// <summary>
/// Environment-level rate limits (Valkey-backed, distributed).
/// Maps to Router's <c>rate_limiting.for_environment</c>.
/// </summary>
public EnvironmentRateLimitConfig? ForEnvironment { get; init; }
/// <summary>
/// Activation threshold: only check environment limits when traffic exceeds this per 5 min.
/// Set to 0 to always check. Default: 1000.
/// </summary>
public int ActivationThresholdPer5Min { get; init; } = 1000;
/// <summary>
/// Whether rate limiting is configured (at least one scope defined).
/// </summary>
public bool IsEnabled => ForInstance is not null || ForEnvironment is not null;
/// <summary>
/// Validate configuration values.
/// </summary>
public MirrorRateLimitConfig Validate()
{
if (ActivationThresholdPer5Min < 0)
throw new ArgumentException("Activation threshold must be >= 0", nameof(ActivationThresholdPer5Min));
ForInstance?.Validate();
ForEnvironment?.Validate();
return this;
}
}
/// <summary>
/// Instance-level rate limit configuration (in-memory, per-process).
/// </summary>
public sealed record InstanceRateLimitConfig
{
/// <summary>
/// Time window in seconds for the rate limit.
/// </summary>
public int PerSeconds { get; init; } = 60;
/// <summary>
/// Maximum requests allowed in the time window.
/// </summary>
public int MaxRequests { get; init; } = 100;
/// <summary>
/// Optional burst window in seconds.
/// </summary>
public int? AllowBurstForSeconds { get; init; }
/// <summary>
/// Maximum burst requests allowed.
/// </summary>
public int? AllowMaxBurstRequests { get; init; }
/// <summary>
/// Validate configuration values.
/// </summary>
public void Validate()
{
if (PerSeconds <= 0)
throw new ArgumentException("PerSeconds must be > 0", nameof(PerSeconds));
if (MaxRequests <= 0)
throw new ArgumentException("MaxRequests must be > 0", nameof(MaxRequests));
if (AllowBurstForSeconds is <= 0)
throw new ArgumentException("AllowBurstForSeconds must be > 0 if specified", nameof(AllowBurstForSeconds));
if (AllowMaxBurstRequests is <= 0)
throw new ArgumentException("AllowMaxBurstRequests must be > 0 if specified", nameof(AllowMaxBurstRequests));
// Both burst values must be set together or neither
if ((AllowBurstForSeconds.HasValue) != (AllowMaxBurstRequests.HasValue))
throw new ArgumentException("AllowBurstForSeconds and AllowMaxBurstRequests must both be set or neither");
}
}
/// <summary>
/// Environment-level rate limit configuration (Valkey-backed, distributed).
/// </summary>
public sealed record EnvironmentRateLimitConfig
{
/// <summary>
/// Valkey connection string.
/// </summary>
public string? ValkeyConnection { get; init; }
/// <summary>
/// Valkey bucket/prefix for rate limit keys.
/// </summary>
public string ValkeyBucket { get; init; } = "stella-mirror-rate-limit";
/// <summary>
/// Time window in seconds.
/// </summary>
public int PerSeconds { get; init; } = 3600;
/// <summary>
/// Maximum requests in the time window.
/// </summary>
public int MaxRequests { get; init; } = 10000;
/// <summary>
/// Optional burst window in seconds.
/// </summary>
public int? AllowBurstForSeconds { get; init; }
/// <summary>
/// Maximum burst requests allowed.
/// </summary>
public int? AllowMaxBurstRequests { get; init; }
/// <summary>
/// Per-route rate limit overrides.
/// Keys are route names (e.g., "index", "bundle"), values are route configs.
/// </summary>
public ImmutableDictionary<string, RouteRateLimitConfig> Routes { get; init; }
= ImmutableDictionary<string, RouteRateLimitConfig>.Empty;
/// <summary>
/// Circuit breaker configuration for Valkey resilience.
/// </summary>
public CircuitBreakerConfig? CircuitBreaker { get; init; }
/// <summary>
/// Validate configuration values.
/// </summary>
public void Validate()
{
if (string.IsNullOrWhiteSpace(ValkeyBucket))
throw new ArgumentException("ValkeyBucket is required", nameof(ValkeyBucket));
if (PerSeconds <= 0)
throw new ArgumentException("PerSeconds must be > 0", nameof(PerSeconds));
if (MaxRequests <= 0)
throw new ArgumentException("MaxRequests must be > 0", nameof(MaxRequests));
if (AllowBurstForSeconds is <= 0)
throw new ArgumentException("AllowBurstForSeconds must be > 0 if specified", nameof(AllowBurstForSeconds));
if (AllowMaxBurstRequests is <= 0)
throw new ArgumentException("AllowMaxBurstRequests must be > 0 if specified", nameof(AllowMaxBurstRequests));
// Both burst values must be set together or neither
if ((AllowBurstForSeconds.HasValue) != (AllowMaxBurstRequests.HasValue))
throw new ArgumentException("AllowBurstForSeconds and AllowMaxBurstRequests must both be set or neither");
foreach (var (name, config) in Routes)
{
config.Validate(name);
}
CircuitBreaker?.Validate();
}
}
/// <summary>
/// Per-route rate limit configuration.
/// </summary>
public sealed record RouteRateLimitConfig
{
/// <summary>
/// Route pattern: exact ("/api/mirror/index"), prefix ("/api/mirror/bundle/*"), or regex.
/// </summary>
public required string Pattern { get; init; }
/// <summary>
/// Pattern match type: Exact, Prefix, or Regex.
/// </summary>
public RouteMatchType MatchType { get; init; } = RouteMatchType.Prefix;
/// <summary>
/// Time window in seconds.
/// </summary>
public int PerSeconds { get; init; }
/// <summary>
/// Maximum requests in the time window.
/// </summary>
public int MaxRequests { get; init; }
/// <summary>
/// Optional burst window in seconds.
/// </summary>
public int? AllowBurstForSeconds { get; init; }
/// <summary>
/// Maximum burst requests allowed.
/// </summary>
public int? AllowMaxBurstRequests { get; init; }
/// <summary>
/// Validate configuration values.
/// </summary>
public void Validate(string routeName)
{
if (string.IsNullOrWhiteSpace(Pattern))
throw new ArgumentException($"Route '{routeName}': Pattern is required");
if (PerSeconds <= 0)
throw new ArgumentException($"Route '{routeName}': PerSeconds must be > 0");
if (MaxRequests <= 0)
throw new ArgumentException($"Route '{routeName}': MaxRequests must be > 0");
}
}
/// <summary>
/// Route pattern match type.
/// </summary>
public enum RouteMatchType
{
/// <summary>Exact path match.</summary>
Exact,
/// <summary>Prefix match (pattern ends with *).</summary>
Prefix,
/// <summary>Regular expression match.</summary>
Regex
}
/// <summary>
/// Circuit breaker configuration for Valkey resilience.
/// </summary>
public sealed record CircuitBreakerConfig
{
/// <summary>
/// Number of failures before opening the circuit.
/// </summary>
public int FailureThreshold { get; init; } = 5;
/// <summary>
/// Seconds to keep circuit open before attempting recovery.
/// </summary>
public int TimeoutSeconds { get; init; } = 30;
/// <summary>
/// Seconds in half-open state before full reset.
/// </summary>
public int HalfOpenTimeoutSeconds { get; init; } = 10;
/// <summary>
/// Validate configuration values.
/// </summary>
public void Validate()
{
if (FailureThreshold < 1)
throw new ArgumentException("FailureThreshold must be >= 1", nameof(FailureThreshold));
if (TimeoutSeconds < 1)
throw new ArgumentException("TimeoutSeconds must be >= 1", nameof(TimeoutSeconds));
if (HalfOpenTimeoutSeconds < 1)
throw new ArgumentException("HalfOpenTimeoutSeconds must be >= 1", nameof(HalfOpenTimeoutSeconds));
}
}

223
src/Concelier/__Libraries/StellaOps.Concelier.Core/Configuration/SourceConfiguration.cs Normal file
View File

@@ -0,0 +1,223 @@
// -----------------------------------------------------------------------------
// SourceConfiguration.cs
// Sprint: SPRINT_20260114_SOURCES_SETUP
// Task: 3.1 - Configuration Models
// Description: Root configuration for advisory data sources and mirror server
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
namespace StellaOps.Concelier.Core.Configuration;
/// <summary>
/// Root configuration for advisory data sources.
/// Supports direct upstream sources (NVD, OSV, GHSA, etc.) and StellaOps mirrors.
/// </summary>
public sealed record SourcesConfiguration
{
/// <summary>
/// Source mode: "direct" for upstream sources, "mirror" for StellaOps mirrors.
/// Default is "mirror" for simpler setup.
/// </summary>
public SourceMode Mode { get; init; } = SourceMode.Mirror;
/// <summary>
/// StellaOps mirror endpoint when using mirror mode.
/// </summary>
public string? MirrorEndpoint { get; init; }
/// <summary>
/// Mirror server configuration when exposing gathered data as a mirror.
/// </summary>
public MirrorServerConfig MirrorServer { get; init; } = new();
/// <summary>
/// Individual source configurations (NVD, OSV, GHSA, etc.).
/// Each source can be enabled/disabled and configured independently.
/// </summary>
public ImmutableDictionary<string, SourceConfig> Sources { get; init; }
= ImmutableDictionary<string, SourceConfig>.Empty;
/// <summary>
/// Auto-enable sources that pass connectivity checks.
/// When true, all sources are enabled by default during setup.
/// </summary>
public bool AutoEnableHealthySources { get; init; } = true;
/// <summary>
/// Timeout for connectivity checks in seconds.
/// </summary>
public int ConnectivityCheckTimeoutSeconds { get; init; } = 30;
}
/// <summary>
/// Source mode for advisory data ingestion.
/// </summary>
public enum SourceMode
{
/// <summary>
/// Direct connection to upstream sources (NVD, OSV, GHSA, etc.).
/// Requires individual source credentials and network access.
/// </summary>
Direct,
/// <summary>
/// Connection to StellaOps pre-aggregated mirror.
/// Simpler setup, single endpoint, pre-normalized data.
/// </summary>
Mirror,
/// <summary>
/// Hybrid mode: use mirror as primary, fall back to direct sources.
/// </summary>
Hybrid
}
/// <summary>
/// Configuration for an individual advisory source.
/// </summary>
public sealed record SourceConfig
{
/// <summary>
/// Whether this source is enabled.
/// </summary>
public bool Enabled { get; init; } = true;
/// <summary>
/// Priority for merge ordering (lower = higher priority).
/// </summary>
public int Priority { get; init; } = 100;
/// <summary>
/// API key or token for authenticated sources.
/// </summary>
public string? ApiKey { get; init; }
/// <summary>
/// Optional custom endpoint URL (overrides default).
/// </summary>
public string? Endpoint { get; init; }
/// <summary>
/// Request delay between API calls (rate limiting).
/// </summary>
public TimeSpan RequestDelay { get; init; } = TimeSpan.FromMilliseconds(200);
/// <summary>
/// Backoff duration after failures.
/// </summary>
public TimeSpan FailureBackoff { get; init; } = TimeSpan.FromMinutes(5);
/// <summary>
/// Maximum pages to fetch per sync cycle.
/// </summary>
public int MaxPagesPerFetch { get; init; } = 10;
/// <summary>
/// Source-specific metadata.
/// </summary>
public ImmutableDictionary<string, string> Metadata { get; init; }
= ImmutableDictionary<string, string>.Empty;
}
/// <summary>
/// Configuration for exposing gathered data as a mirror server.
/// </summary>
public sealed record MirrorServerConfig
{
/// <summary>
/// Whether the mirror server is enabled.
/// </summary>
public bool Enabled { get; init; }
/// <summary>
/// Root directory for mirror exports.
/// </summary>
public string ExportRoot { get; init; } = "./exports/mirror";
/// <summary>
/// Authentication mode for mirror clients.
/// </summary>
public MirrorAuthMode Authentication { get; init; } = MirrorAuthMode.Anonymous;
/// <summary>
/// OAuth configuration when using OAuth authentication.
/// </summary>
public OAuthMirrorConfig? OAuth { get; init; }
/// <summary>
/// Rate limiting configuration for mirror endpoints.
/// Maps to Router library rate limiting.
/// </summary>
public MirrorRateLimitConfig RateLimits { get; init; } = new();
/// <summary>
/// Signing key path for DSSE attestations on mirror bundles.
/// </summary>
public string? SigningKeyPath { get; init; }
/// <summary>
/// Whether to include attestations in mirror bundles.
/// </summary>
public bool IncludeAttestations { get; init; } = true;
}
/// <summary>
/// Authentication mode for mirror server.
/// </summary>
public enum MirrorAuthMode
{
/// <summary>
/// No authentication required (open access).
/// </summary>
Anonymous,
/// <summary>
/// OAuth 2.0 client credentials flow.
/// </summary>
OAuth,
/// <summary>
/// API key-based authentication.
/// </summary>
ApiKey,
/// <summary>
/// mTLS client certificate authentication.
/// </summary>
Mtls
}
/// <summary>
/// OAuth configuration for mirror server authentication.
/// </summary>
public sealed record OAuthMirrorConfig
{
/// <summary>
/// OAuth issuer URL (for discovery).
/// </summary>
[Required]
public string Issuer { get; init; } = "";
/// <summary>
/// Required audience in access tokens.
/// </summary>
public string? Audience { get; init; }
/// <summary>
/// Required scopes for access.
/// </summary>
public ImmutableArray<string> RequiredScopes { get; init; }
= ImmutableArray<string>.Empty;
/// <summary>
/// Whether to validate issuer HTTPS metadata.
/// </summary>
public bool RequireHttpsMetadata { get; init; } = true;
/// <summary>
/// Token clock skew tolerance.
/// </summary>
public TimeSpan ClockSkew { get; init; } = TimeSpan.FromMinutes(1);
}

98
src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/ISourceRegistry.cs Normal file
View File

@@ -0,0 +1,98 @@
// -----------------------------------------------------------------------------
// ISourceRegistry.cs
// Sprint: SPRINT_20260114_SOURCES_SETUP
// Task: 3.3 - Source Registry Interface
// Description: Interface for managing and checking advisory source connectivity
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.Sources;
/// <summary>
/// Registry for managing advisory data sources (NVD, OSV, GHSA, etc.).
/// Provides connectivity checking and auto-configuration capabilities.
/// </summary>
public interface ISourceRegistry
{
/// <summary>
/// Get all registered source definitions.
/// </summary>
IReadOnlyList<SourceDefinition> GetAllSources();
/// <summary>
/// Get a specific source definition by ID.
/// </summary>
SourceDefinition? GetSource(string sourceId);
/// <summary>
/// Get sources by category.
/// </summary>
IReadOnlyList<SourceDefinition> GetSourcesByCategory(SourceCategory category);
/// <summary>
/// Check connectivity for a specific source.
/// </summary>
/// <param name="sourceId">Source identifier.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Connectivity result with status and error details if failed.</returns>
Task<SourceConnectivityResult> CheckConnectivityAsync(
string sourceId,
CancellationToken cancellationToken = default);
/// <summary>
/// Check all sources and auto-configure based on availability.
/// Sources that pass connectivity checks are enabled by default.
/// </summary>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Aggregated check result with enabled/disabled sources.</returns>
Task<SourceCheckResult> CheckAllAndAutoConfigureAsync(
CancellationToken cancellationToken = default);
/// <summary>
/// Check connectivity for multiple sources in parallel.
/// </summary>
/// <param name="sourceIds">Source identifiers to check.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Individual results for each source.</returns>
Task<ImmutableArray<SourceConnectivityResult>> CheckMultipleAsync(
IEnumerable<string> sourceIds,
CancellationToken cancellationToken = default);
/// <summary>
/// Enable a source for data ingestion.
/// </summary>
Task<bool> EnableSourceAsync(
string sourceId,
CancellationToken cancellationToken = default);
/// <summary>
/// Disable a source.
/// </summary>
Task<bool> DisableSourceAsync(
string sourceId,
CancellationToken cancellationToken = default);
/// <summary>
/// Get currently enabled sources.
/// </summary>
Task<ImmutableArray<string>> GetEnabledSourcesAsync(
CancellationToken cancellationToken = default);
/// <summary>
/// Check if a specific source is enabled.
/// </summary>
bool IsEnabled(string sourceId);
/// <summary>
/// Retry connectivity check for a failed source.
/// </summary>
Task<SourceConnectivityResult> RetryCheckAsync(
string sourceId,
CancellationToken cancellationToken = default);
/// <summary>
/// Get the last connectivity check result for a source.
/// </summary>
SourceConnectivityResult? GetLastCheckResult(string sourceId);
}

160
src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceCheckResult.cs Normal file
View File

@@ -0,0 +1,160 @@
// -----------------------------------------------------------------------------
// SourceCheckResult.cs
// Sprint: SPRINT_20260114_SOURCES_SETUP
// Task: 3.3 - Source Check Aggregation
// Description: Aggregated result of checking multiple sources
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.Sources;
/// <summary>
/// Aggregated result of checking all sources for connectivity.
/// Used by the setup wizard to auto-enable healthy sources.
/// </summary>
public sealed record SourceCheckResult
{
/// <summary>
/// Individual results for each checked source.
/// </summary>
public ImmutableArray<SourceConnectivityResult> Results { get; init; }
= ImmutableArray<SourceConnectivityResult>.Empty;
/// <summary>
/// Source IDs that passed connectivity checks and are auto-enabled.
/// </summary>
public ImmutableArray<string> EnabledSources { get; init; }
= ImmutableArray<string>.Empty;
/// <summary>
/// Source IDs that failed connectivity checks.
/// </summary>
public ImmutableArray<string> DisabledSources { get; init; }
= ImmutableArray<string>.Empty;
/// <summary>
/// Source IDs that are degraded but still enabled.
/// </summary>
public ImmutableArray<string> DegradedSources { get; init; }
= ImmutableArray<string>.Empty;
/// <summary>
/// When the check was performed.
/// </summary>
public DateTimeOffset CheckedAt { get; init; }
/// <summary>
/// Total duration of all checks.
/// </summary>
public TimeSpan TotalDuration { get; init; }
/// <summary>
/// Whether all sources are healthy.
/// </summary>
public bool AllHealthy => DisabledSources.Length == 0;
/// <summary>
/// Whether any sources failed.
/// </summary>
public bool HasFailures => DisabledSources.Length > 0;
/// <summary>
/// Total number of sources checked.
/// </summary>
public int TotalChecked => Results.Length;
/// <summary>
/// Number of healthy sources.
/// </summary>
public int HealthyCount => EnabledSources.Length;
/// <summary>
/// Number of failed sources.
/// </summary>
public int FailedCount => DisabledSources.Length;
/// <summary>
/// Summary message for display.
/// </summary>
public string Summary => AllHealthy
? $"All {TotalChecked} sources are healthy"
: $"{HealthyCount}/{TotalChecked} sources healthy, {FailedCount} failed";
/// <summary>
/// Get results grouped by category.
/// </summary>
public ImmutableDictionary<SourceCategory, ImmutableArray<SourceConnectivityResult>> ByCategory(
ISourceRegistry registry)
{
var builder = ImmutableDictionary.CreateBuilder<SourceCategory, ImmutableArray<SourceConnectivityResult>>();
var groups = Results
.GroupBy(r => registry.GetSource(r.SourceId)?.Category ?? SourceCategory.Other)
.ToDictionary(g => g.Key, g => g.ToImmutableArray());
foreach (var (category, results) in groups)
{
builder[category] = results;
}
return builder.ToImmutable();
}
/// <summary>
/// Get failed results for display.
/// </summary>
public ImmutableArray<SourceConnectivityResult> GetFailedResults()
=> Results.Where(r => r.Status == SourceConnectivityStatus.Failed).ToImmutableArray();
/// <summary>
/// Create an empty result.
/// </summary>
public static SourceCheckResult Empty(DateTimeOffset? checkedAt = null)
=> new()
{
CheckedAt = checkedAt ?? DateTimeOffset.UtcNow,
TotalDuration = TimeSpan.Zero
};
/// <summary>
/// Create result from individual results.
/// </summary>
public static SourceCheckResult FromResults(
IEnumerable<SourceConnectivityResult> results,
DateTimeOffset checkedAt,
TimeSpan duration)
{
var resultArray = results.ToImmutableArray();
var enabled = new List<string>();
var disabled = new List<string>();
var degraded = new List<string>();
foreach (var result in resultArray)
{
switch (result.Status)
{
case SourceConnectivityStatus.Healthy:
enabled.Add(result.SourceId);
break;
case SourceConnectivityStatus.Degraded:
enabled.Add(result.SourceId);
degraded.Add(result.SourceId);
break;
case SourceConnectivityStatus.Failed:
disabled.Add(result.SourceId);
break;
}
}
return new SourceCheckResult
{
Results = resultArray,
EnabledSources = enabled.ToImmutableArray(),
DisabledSources = disabled.ToImmutableArray(),
DegradedSources = degraded.ToImmutableArray(),
CheckedAt = checkedAt,
TotalDuration = duration
};
}
}

231
src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceConnectivityResult.cs Normal file
View File

@@ -0,0 +1,231 @@
// -----------------------------------------------------------------------------
// SourceConnectivityResult.cs
// Sprint: SPRINT_20260114_SOURCES_SETUP
// Task: 3.3 - Source Connectivity Models
// Description: Result types for source connectivity checks
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.Sources;
/// <summary>
/// Result of a connectivity check for a single source.
/// </summary>
public sealed record SourceConnectivityResult
{
/// <summary>
/// Source identifier.
/// </summary>
public required string SourceId { get; init; }
/// <summary>
/// Connectivity status.
/// </summary>
public SourceConnectivityStatus Status { get; init; }
/// <summary>
/// When the check was performed.
/// </summary>
public DateTimeOffset CheckedAt { get; init; }
/// <summary>
/// Response latency if successful.
/// </summary>
public TimeSpan? Latency { get; init; }
/// <summary>
/// Error message if failed.
/// </summary>
public string? ErrorMessage { get; init; }
/// <summary>
/// Error code for categorization.
/// </summary>
public string? ErrorCode { get; init; }
/// <summary>
/// HTTP status code if applicable.
/// </summary>
public int? HttpStatusCode { get; init; }
/// <summary>
/// Possible reasons for the failure (for user display).
/// </summary>
public ImmutableArray<string> PossibleReasons { get; init; }
= ImmutableArray<string>.Empty;
/// <summary>
/// Steps to remediate the issue.
/// </summary>
public ImmutableArray<RemediationStep> RemediationSteps { get; init; }
= ImmutableArray<RemediationStep>.Empty;
/// <summary>
/// Documentation URL for more information.
/// </summary>
public string? DocumentationUrl { get; init; }
/// <summary>
/// Additional diagnostic data.
/// </summary>
public ImmutableDictionary<string, string> Diagnostics { get; init; }
= ImmutableDictionary<string, string>.Empty;
/// <summary>
/// Whether the source is healthy (can be enabled).
/// </summary>
public bool IsHealthy => Status is SourceConnectivityStatus.Healthy or SourceConnectivityStatus.Degraded;
/// <summary>
/// Create a healthy result.
/// </summary>
public static SourceConnectivityResult Healthy(
string sourceId,
TimeSpan latency,
DateTimeOffset? checkedAt = null)
=> new()
{
SourceId = sourceId,
Status = SourceConnectivityStatus.Healthy,
CheckedAt = checkedAt ?? DateTimeOffset.UtcNow,
Latency = latency
};
/// <summary>
/// Create a degraded result.
/// </summary>
public static SourceConnectivityResult Degraded(
string sourceId,
TimeSpan latency,
string message,
DateTimeOffset? checkedAt = null)
=> new()
{
SourceId = sourceId,
Status = SourceConnectivityStatus.Degraded,
CheckedAt = checkedAt ?? DateTimeOffset.UtcNow,
Latency = latency,
ErrorMessage = message
};
/// <summary>
/// Create a failed result.
/// </summary>
public static SourceConnectivityResult Failed(
string sourceId,
string errorCode,
string errorMessage,
ImmutableArray<string> possibleReasons,
ImmutableArray<RemediationStep> remediationSteps,
DateTimeOffset? checkedAt = null,
TimeSpan? latency = null,
int? httpStatusCode = null)
=> new()
{
SourceId = sourceId,
Status = SourceConnectivityStatus.Failed,
CheckedAt = checkedAt ?? DateTimeOffset.UtcNow,
Latency = latency,
ErrorCode = errorCode,
ErrorMessage = errorMessage,
HttpStatusCode = httpStatusCode,
PossibleReasons = possibleReasons,
RemediationSteps = remediationSteps
};
/// <summary>
/// Create a not-found result.
/// </summary>
public static SourceConnectivityResult NotFound(string sourceId)
=> new()
{
SourceId = sourceId,
Status = SourceConnectivityStatus.Failed,
CheckedAt = DateTimeOffset.UtcNow,
ErrorCode = "SOURCE_NOT_FOUND",
ErrorMessage = $"Source '{sourceId}' is not registered",
PossibleReasons = ImmutableArray.Create(
"The source ID may be misspelled",
"This source may not be available in your region"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep { Order = 1, Description = "Verify the source ID is correct" },
new RemediationStep { Order = 2, Description = "Run 'stella sources list' to see available sources" })
};
}
/// <summary>
/// Connectivity status for a source.
/// </summary>
public enum SourceConnectivityStatus
{
/// <summary>Source is unknown or not checked.</summary>
Unknown,
/// <summary>Source is fully available.</summary>
Healthy,
/// <summary>Source is available but with issues (slow, rate limited, etc.).</summary>
Degraded,
/// <summary>Source is not available.</summary>
Failed,
/// <summary>Connectivity check is in progress.</summary>
Checking,
/// <summary>Source is disabled by configuration.</summary>
Disabled
}
/// <summary>
/// A step to remediate a connectivity issue.
/// </summary>
public sealed record RemediationStep
{
/// <summary>
/// Step order (1-based).
/// </summary>
public int Order { get; init; }
/// <summary>
/// Human-readable description of the step.
/// </summary>
public required string Description { get; init; }
/// <summary>
/// Optional command to run.
/// </summary>
public string? Command { get; init; }
/// <summary>
/// Type of command (bash, powershell, url, etc.).
/// </summary>
public CommandType CommandType { get; init; } = CommandType.Bash;
/// <summary>
/// URL for more information.
/// </summary>
public string? DocumentationUrl { get; init; }
}
/// <summary>
/// Type of remediation command.
/// </summary>
public enum CommandType
{
/// <summary>Bash/shell command.</summary>
Bash,
/// <summary>PowerShell command.</summary>
PowerShell,
/// <summary>URL to open.</summary>
Url,
/// <summary>StellaOps CLI command.</summary>
StellaCli,
/// <summary>Environment variable to set.</summary>
EnvVar
}

970
src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceDefinitions.cs Normal file
View File

@@ -0,0 +1,970 @@
// -----------------------------------------------------------------------------
// SourceDefinitions.cs
// Sprint: SPRINT_20260114_SOURCES_SETUP
// Task: 3.3 - Source Catalog
// Description: Static catalog of all supported advisory data sources
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
namespace StellaOps.Concelier.Core.Sources;
/// <summary>
/// Definition of an advisory data source.
/// </summary>
public sealed record SourceDefinition
{
/// <summary>
/// Unique identifier for the source (e.g., "nvd", "ghsa", "osv").
/// </summary>
public required string Id { get; init; }
/// <summary>
/// Human-readable display name.
/// </summary>
public required string DisplayName { get; init; }
/// <summary>
/// Source category.
/// </summary>
public SourceCategory Category { get; init; }
/// <summary>
/// Source type (upstream, mirror, etc.).
/// </summary>
public SourceType Type { get; init; }
/// <summary>
/// Brief description of the source.
/// </summary>
public string Description { get; init; } = "";
/// <summary>
/// Base API endpoint URL.
/// </summary>
public required string BaseEndpoint { get; init; }
/// <summary>
/// Health check endpoint for connectivity verification.
/// </summary>
public required string HealthCheckEndpoint { get; init; }
/// <summary>
/// Named HTTP client for this source.
/// </summary>
public string HttpClientName { get; init; } = "";
/// <summary>
/// Whether authentication is required.
/// </summary>
public bool RequiresAuthentication { get; init; }
/// <summary>
/// Environment variable name for API key/token.
/// </summary>
public string? CredentialEnvVar { get; init; }
/// <summary>
/// URL for obtaining credentials.
/// </summary>
public string? CredentialUrl { get; init; }
/// <summary>
/// Status page URL for the source.
/// </summary>
public string? StatusPageUrl { get; init; }
/// <summary>
/// Documentation URL.
/// </summary>
public string? DocumentationUrl { get; init; }
/// <summary>
/// Default priority for merge ordering.
/// </summary>
public int DefaultPriority { get; init; } = 100;
/// <summary>
/// Geographic regions this source covers (if region-specific).
/// </summary>
public ImmutableArray<string> Regions { get; init; }
= ImmutableArray<string>.Empty;
/// <summary>
/// Whether the source is enabled by default.
/// </summary>
public bool EnabledByDefault { get; init; } = true;
/// <summary>
/// Tags for filtering/grouping sources.
/// </summary>
public ImmutableArray<string> Tags { get; init; }
= ImmutableArray<string>.Empty;
}
/// <summary>
/// Category of advisory source.
/// </summary>
public enum SourceCategory
{
/// <summary>Primary vulnerability databases (NVD, OSV).</summary>
Primary,
/// <summary>Vendor-specific advisories (Red Hat, Microsoft).</summary>
Vendor,
/// <summary>Linux distribution advisories (Debian, Ubuntu).</summary>
Distribution,
/// <summary>Language ecosystem advisories (npm, PyPI).</summary>
Ecosystem,
/// <summary>National CERTs and government sources.</summary>
Cert,
/// <summary>CSAF/VEX document sources.</summary>
Csaf,
/// <summary>Exploit and threat intelligence sources.</summary>
Threat,
/// <summary>StellaOps mirrors.</summary>
Mirror,
/// <summary>Other/uncategorized sources.</summary>
Other
}
/// <summary>
/// Type of source connection.
/// </summary>
public enum SourceType
{
/// <summary>Direct upstream API connection.</summary>
Upstream,
/// <summary>StellaOps pre-aggregated mirror.</summary>
StellaMirror,
/// <summary>Local file-based source.</summary>
LocalFile,
/// <summary>Custom/user-defined source.</summary>
Custom
}
/// <summary>
/// Static catalog of all supported sources.
/// </summary>
public static class SourceDefinitions
{
// ===== Primary Databases =====
public static readonly SourceDefinition Nvd = new()
{
Id = "nvd",
DisplayName = "NVD (NIST)",
Category = SourceCategory.Primary,
Type = SourceType.Upstream,
Description = "NIST National Vulnerability Database",
BaseEndpoint = "https://services.nvd.nist.gov/rest/json/cves/2.0",
HealthCheckEndpoint = "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=1",
HttpClientName = "NvdClient",
RequiresAuthentication = false, // Optional but recommended
CredentialEnvVar = "NVD_API_KEY",
CredentialUrl = "https://nvd.nist.gov/developers/request-an-api-key",
StatusPageUrl = "https://nvd.nist.gov/",
DocumentationUrl = "https://nvd.nist.gov/developers/vulnerabilities",
DefaultPriority = 10,
Tags = ImmutableArray.Create("cve", "primary", "global")
};
public static readonly SourceDefinition Osv = new()
{
Id = "osv",
DisplayName = "OSV (Google)",
Category = SourceCategory.Primary,
Type = SourceType.Upstream,
Description = "Open Source Vulnerabilities database",
BaseEndpoint = "https://api.osv.dev/v1",
HealthCheckEndpoint = "https://api.osv.dev/v1/query",
HttpClientName = "OsvClient",
RequiresAuthentication = false,
StatusPageUrl = "https://osv.dev",
DocumentationUrl = "https://osv.dev/docs/",
DefaultPriority = 15,
Tags = ImmutableArray.Create("osv", "primary", "ecosystem")
};
public static readonly SourceDefinition Ghsa = new()
{
Id = "ghsa",
DisplayName = "GitHub Security Advisories",
Category = SourceCategory.Primary,
Type = SourceType.Upstream,
Description = "GitHub Security Advisories database",
BaseEndpoint = "https://api.github.com/graphql",
HealthCheckEndpoint = "https://api.github.com/zen",
HttpClientName = "GhsaClient",
RequiresAuthentication = true,
CredentialEnvVar = "GITHUB_PAT",
CredentialUrl = "https://github.com/settings/tokens",
StatusPageUrl = "https://www.githubstatus.com/",
DocumentationUrl = "https://docs.github.com/en/graphql/reference/objects#securityadvisory",
DefaultPriority = 20,
Tags = ImmutableArray.Create("github", "primary", "ecosystem")
};
public static readonly SourceDefinition Cve = new()
{
Id = "cve",
DisplayName = "CVE.org (MITRE)",
Category = SourceCategory.Primary,
Type = SourceType.Upstream,
Description = "MITRE CVE Program",
BaseEndpoint = "https://cveawg.mitre.org/api/",
HealthCheckEndpoint = "https://cveawg.mitre.org/api/cve/CVE-2021-44228",
HttpClientName = "CveClient",
RequiresAuthentication = false,
StatusPageUrl = "https://cve.mitre.org/",
DocumentationUrl = "https://cveawg.mitre.org/api/",
DefaultPriority = 5,
Tags = ImmutableArray.Create("cve", "primary", "authoritative")
};
public static readonly SourceDefinition Epss = new()
{
Id = "epss",
DisplayName = "EPSS (FIRST)",
Category = SourceCategory.Threat,
Type = SourceType.Upstream,
Description = "Exploit Prediction Scoring System",
BaseEndpoint = "https://api.first.org/data/v1",
HealthCheckEndpoint = "https://api.first.org/data/v1/epss?cve=CVE-2021-44228",
HttpClientName = "EpssClient",
RequiresAuthentication = false,
StatusPageUrl = "https://www.first.org/epss/",
DocumentationUrl = "https://www.first.org/epss/api",
DefaultPriority = 50,
Tags = ImmutableArray.Create("epss", "threat", "scoring")
};
public static readonly SourceDefinition Kev = new()
{
Id = "kev",
DisplayName = "CISA KEV",
Category = SourceCategory.Threat,
Type = SourceType.Upstream,
Description = "Known Exploited Vulnerabilities Catalog",
BaseEndpoint = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
HealthCheckEndpoint = "https://www.cisa.gov/sites/default/files/feeds/known_exploited_vulnerabilities.json",
HttpClientName = "KevClient",
RequiresAuthentication = false,
StatusPageUrl = "https://www.cisa.gov/known-exploited-vulnerabilities-catalog",
DocumentationUrl = "https://www.cisa.gov/known-exploited-vulnerabilities-catalog",
DefaultPriority = 25,
Tags = ImmutableArray.Create("kev", "threat", "exploit")
};
// ===== Vendor Advisories =====
public static readonly SourceDefinition RedHat = new()
{
Id = "redhat",
DisplayName = "Red Hat Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Red Hat Security Data API",
BaseEndpoint = "https://access.redhat.com/hydra/rest/securitydata/",
HealthCheckEndpoint = "https://access.redhat.com/hydra/rest/securitydata/cve.json?per_page=1",
HttpClientName = "RedHatClient",
RequiresAuthentication = false,
StatusPageUrl = "https://status.redhat.com/",
DocumentationUrl = "https://access.redhat.com/documentation/en-us/red_hat_security_data_api/",
DefaultPriority = 30,
Tags = ImmutableArray.Create("redhat", "vendor", "linux")
};
public static readonly SourceDefinition Microsoft = new()
{
Id = "microsoft",
DisplayName = "Microsoft Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Microsoft Security Response Center",
BaseEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/",
HealthCheckEndpoint = "https://api.msrc.microsoft.com/sug/v2.0/en-US/affectedProduct",
HttpClientName = "MsrcClient",
RequiresAuthentication = false,
StatusPageUrl = "https://msrc.microsoft.com/",
DocumentationUrl = "https://msrc.microsoft.com/update-guide/",
DefaultPriority = 35,
Tags = ImmutableArray.Create("microsoft", "vendor", "windows")
};
public static readonly SourceDefinition Amazon = new()
{
Id = "amazon",
DisplayName = "Amazon Linux Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Amazon Linux Security Center",
BaseEndpoint = "https://alas.aws.amazon.com/",
HealthCheckEndpoint = "https://alas.aws.amazon.com/alas.rss",
HttpClientName = "AmazonClient",
RequiresAuthentication = false,
StatusPageUrl = "https://status.aws.amazon.com/",
DefaultPriority = 40,
Tags = ImmutableArray.Create("amazon", "vendor", "cloud", "linux")
};
public static readonly SourceDefinition Google = new()
{
Id = "google",
DisplayName = "Google Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Google Security Bulletins",
BaseEndpoint = "https://source.android.com/docs/security/bulletin/",
HealthCheckEndpoint = "https://source.android.com/docs/security/bulletin/",
HttpClientName = "GoogleClient",
RequiresAuthentication = false,
DefaultPriority = 45,
Tags = ImmutableArray.Create("google", "vendor", "android")
};
public static readonly SourceDefinition Oracle = new()
{
Id = "oracle",
DisplayName = "Oracle Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Oracle Security Alerts",
BaseEndpoint = "https://www.oracle.com/security-alerts/",
HealthCheckEndpoint = "https://www.oracle.com/security-alerts/",
HttpClientName = "OracleClient",
RequiresAuthentication = false,
DefaultPriority = 50,
Tags = ImmutableArray.Create("oracle", "vendor", "java")
};
public static readonly SourceDefinition Apple = new()
{
Id = "apple",
DisplayName = "Apple Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Apple Security Updates",
BaseEndpoint = "https://support.apple.com/en-us/HT201222",
HealthCheckEndpoint = "https://support.apple.com/en-us/HT201222",
HttpClientName = "AppleClient",
RequiresAuthentication = false,
DefaultPriority = 55,
Tags = ImmutableArray.Create("apple", "vendor", "macos", "ios")
};
public static readonly SourceDefinition Cisco = new()
{
Id = "cisco",
DisplayName = "Cisco Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Cisco Security Advisories",
BaseEndpoint = "https://tools.cisco.com/security/center/publicationService.x",
HealthCheckEndpoint = "https://tools.cisco.com/security/center/publicationListing.x",
HttpClientName = "CiscoClient",
RequiresAuthentication = false,
StatusPageUrl = "https://status.cisco.com/",
DefaultPriority = 60,
Tags = ImmutableArray.Create("cisco", "vendor", "network")
};
public static readonly SourceDefinition Fortinet = new()
{
Id = "fortinet",
DisplayName = "Fortinet PSIRT",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Fortinet Product Security Incident Response Team",
BaseEndpoint = "https://www.fortiguard.com/psirt",
HealthCheckEndpoint = "https://www.fortiguard.com/psirt",
HttpClientName = "FortinetClient",
RequiresAuthentication = false,
DefaultPriority = 65,
Tags = ImmutableArray.Create("fortinet", "vendor", "network", "security")
};
public static readonly SourceDefinition Juniper = new()
{
Id = "juniper",
DisplayName = "Juniper Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Juniper Security Advisories",
BaseEndpoint = "https://supportportal.juniper.net/s/",
HealthCheckEndpoint = "https://supportportal.juniper.net/s/",
HttpClientName = "JuniperClient",
RequiresAuthentication = false,
DefaultPriority = 70,
Tags = ImmutableArray.Create("juniper", "vendor", "network")
};
public static readonly SourceDefinition Palo = new()
{
Id = "paloalto",
DisplayName = "Palo Alto Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "Palo Alto Networks Security Advisories",
BaseEndpoint = "https://security.paloaltonetworks.com/",
HealthCheckEndpoint = "https://security.paloaltonetworks.com/",
HttpClientName = "PaloClient",
RequiresAuthentication = false,
DefaultPriority = 75,
Tags = ImmutableArray.Create("paloalto", "vendor", "network", "security")
};
public static readonly SourceDefinition Vmware = new()
{
Id = "vmware",
DisplayName = "VMware Security",
Category = SourceCategory.Vendor,
Type = SourceType.Upstream,
Description = "VMware Security Advisories",
BaseEndpoint = "https://www.vmware.com/security/advisories.html",
HealthCheckEndpoint = "https://www.vmware.com/security/advisories.html",
HttpClientName = "VmwareClient",
RequiresAuthentication = false,
DefaultPriority = 80,
Tags = ImmutableArray.Create("vmware", "vendor", "virtualization")
};
// ===== Linux Distributions =====
public static readonly SourceDefinition Debian = new()
{
Id = "debian",
DisplayName = "Debian Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "Debian Security Tracker",
BaseEndpoint = "https://security-tracker.debian.org/tracker/data/json",
HealthCheckEndpoint = "https://security-tracker.debian.org/tracker/",
HttpClientName = "DebianClient",
RequiresAuthentication = false,
StatusPageUrl = "https://security-tracker.debian.org/tracker/",
DocumentationUrl = "https://www.debian.org/security/",
DefaultPriority = 30,
Tags = ImmutableArray.Create("debian", "distro", "linux")
};
public static readonly SourceDefinition Ubuntu = new()
{
Id = "ubuntu",
DisplayName = "Ubuntu Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "Ubuntu Security Notices",
BaseEndpoint = "https://ubuntu.com/security/cves.json",
HealthCheckEndpoint = "https://ubuntu.com/security/cves.json?limit=1",
HttpClientName = "UbuntuClient",
RequiresAuthentication = false,
StatusPageUrl = "https://ubuntu.com/security",
DefaultPriority = 32,
Tags = ImmutableArray.Create("ubuntu", "distro", "linux")
};
public static readonly SourceDefinition Alpine = new()
{
Id = "alpine",
DisplayName = "Alpine Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "Alpine Linux Security Database",
BaseEndpoint = "https://secdb.alpinelinux.org/",
HealthCheckEndpoint = "https://secdb.alpinelinux.org/",
HttpClientName = "AlpineClient",
RequiresAuthentication = false,
DefaultPriority = 34,
Tags = ImmutableArray.Create("alpine", "distro", "linux", "container")
};
public static readonly SourceDefinition Suse = new()
{
Id = "suse",
DisplayName = "SUSE Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "SUSE Security Updates",
BaseEndpoint = "https://www.suse.com/support/update/",
HealthCheckEndpoint = "https://www.suse.com/support/update/",
HttpClientName = "SuseClient",
RequiresAuthentication = false,
DefaultPriority = 36,
Tags = ImmutableArray.Create("suse", "distro", "linux")
};
public static readonly SourceDefinition Rhel = new()
{
Id = "rhel",
DisplayName = "RHEL Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "Red Hat Enterprise Linux Security",
BaseEndpoint = "https://access.redhat.com/hydra/rest/securitydata/",
HealthCheckEndpoint = "https://access.redhat.com/hydra/rest/securitydata/cve.json?per_page=1",
HttpClientName = "RhelClient",
RequiresAuthentication = false,
DefaultPriority = 38,
Tags = ImmutableArray.Create("rhel", "distro", "linux", "enterprise")
};
public static readonly SourceDefinition Centos = new()
{
Id = "centos",
DisplayName = "CentOS Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "CentOS Security Updates",
BaseEndpoint = "https://lists.centos.org/pipermail/centos-announce/",
HealthCheckEndpoint = "https://lists.centos.org/pipermail/centos-announce/",
HttpClientName = "CentosClient",
RequiresAuthentication = false,
DefaultPriority = 40,
Tags = ImmutableArray.Create("centos", "distro", "linux")
};
public static readonly SourceDefinition Fedora = new()
{
Id = "fedora",
DisplayName = "Fedora Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "Fedora Security Updates",
BaseEndpoint = "https://bodhi.fedoraproject.org/updates/",
HealthCheckEndpoint = "https://bodhi.fedoraproject.org/updates/?status=stable&type=security&rows_per_page=1",
HttpClientName = "FedoraClient",
RequiresAuthentication = false,
DefaultPriority = 42,
Tags = ImmutableArray.Create("fedora", "distro", "linux")
};
public static readonly SourceDefinition Arch = new()
{
Id = "arch",
DisplayName = "Arch Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "Arch Linux Security Tracker",
BaseEndpoint = "https://security.archlinux.org/",
HealthCheckEndpoint = "https://security.archlinux.org/issues/all.json",
HttpClientName = "ArchClient",
RequiresAuthentication = false,
DefaultPriority = 44,
Tags = ImmutableArray.Create("arch", "distro", "linux")
};
public static readonly SourceDefinition Gentoo = new()
{
Id = "gentoo",
DisplayName = "Gentoo Security",
Category = SourceCategory.Distribution,
Type = SourceType.Upstream,
Description = "Gentoo Linux Security Advisories",
BaseEndpoint = "https://security.gentoo.org/glsa/feed.rss",
HealthCheckEndpoint = "https://security.gentoo.org/",
HttpClientName = "GentooClient",
RequiresAuthentication = false,
DefaultPriority = 46,
Tags = ImmutableArray.Create("gentoo", "distro", "linux")
};
// ===== Language Ecosystems =====
public static readonly SourceDefinition Npm = new()
{
Id = "npm",
DisplayName = "npm Advisories",
Category = SourceCategory.Ecosystem,
Type = SourceType.Upstream,
Description = "npm Security Advisories (via OSV)",
BaseEndpoint = "https://api.osv.dev/v1",
HealthCheckEndpoint = "https://api.osv.dev/v1/query",
HttpClientName = "NpmClient",
RequiresAuthentication = false,
DefaultPriority = 50,
Tags = ImmutableArray.Create("npm", "ecosystem", "javascript", "node")
};
public static readonly SourceDefinition PyPi = new()
{
Id = "pypi",
DisplayName = "PyPI Advisories",
Category = SourceCategory.Ecosystem,
Type = SourceType.Upstream,
Description = "Python Package Index Security Advisories (via OSV)",
BaseEndpoint = "https://api.osv.dev/v1",
HealthCheckEndpoint = "https://api.osv.dev/v1/query",
HttpClientName = "PyPiClient",
RequiresAuthentication = false,
DefaultPriority = 52,
Tags = ImmutableArray.Create("pypi", "ecosystem", "python")
};
public static readonly SourceDefinition Go = new()
{
Id = "go",
DisplayName = "Go Advisories",
Category = SourceCategory.Ecosystem,
Type = SourceType.Upstream,
Description = "Go Vulnerability Database",
BaseEndpoint = "https://vuln.go.dev/",
HealthCheckEndpoint = "https://vuln.go.dev/",
HttpClientName = "GoClient",
RequiresAuthentication = false,
DefaultPriority = 54,
Tags = ImmutableArray.Create("go", "ecosystem", "golang")
};
public static readonly SourceDefinition RubyGems = new()
{
Id = "rubygems",
DisplayName = "RubyGems Advisories",
Category = SourceCategory.Ecosystem,
Type = SourceType.Upstream,
Description = "RubyGems Security Advisories (via OSV)",
BaseEndpoint = "https://api.osv.dev/v1",
HealthCheckEndpoint = "https://api.osv.dev/v1/query",
HttpClientName = "RubyGemsClient",
RequiresAuthentication = false,
DefaultPriority = 56,
Tags = ImmutableArray.Create("rubygems", "ecosystem", "ruby")
};
public static readonly SourceDefinition Nuget = new()
{
Id = "nuget",
DisplayName = "NuGet Advisories",
Category = SourceCategory.Ecosystem,
Type = SourceType.Upstream,
Description = "NuGet Security Advisories (via GHSA)",
BaseEndpoint = "https://api.github.com/graphql",
HealthCheckEndpoint = "https://api.github.com/zen",
HttpClientName = "NugetClient",
RequiresAuthentication = true,
CredentialEnvVar = "GITHUB_PAT",
DefaultPriority = 58,
Tags = ImmutableArray.Create("nuget", "ecosystem", "dotnet", "csharp")
};
public static readonly SourceDefinition Maven = new()
{
Id = "maven",
DisplayName = "Maven Advisories",
Category = SourceCategory.Ecosystem,
Type = SourceType.Upstream,
Description = "Maven Central Security Advisories (via OSV)",
BaseEndpoint = "https://api.osv.dev/v1",
HealthCheckEndpoint = "https://api.osv.dev/v1/query",
HttpClientName = "MavenClient",
RequiresAuthentication = false,
DefaultPriority = 60,
Tags = ImmutableArray.Create("maven", "ecosystem", "java")
};
public static readonly SourceDefinition Crates = new()
{
Id = "crates",
DisplayName = "Crates.io Advisories",
Category = SourceCategory.Ecosystem,
Type = SourceType.Upstream,
Description = "Rust Crates.io Security Advisories",
BaseEndpoint = "https://rustsec.org/advisories/",
HealthCheckEndpoint = "https://rustsec.org/advisories/",
HttpClientName = "CratesClient",
RequiresAuthentication = false,
DefaultPriority = 62,
Tags = ImmutableArray.Create("crates", "ecosystem", "rust")
};
public static readonly SourceDefinition Packagist = new()
{
Id = "packagist",
DisplayName = "Packagist Advisories",
Category = SourceCategory.Ecosystem,
Type = SourceType.Upstream,
Description = "PHP Packagist Security Advisories (via OSV)",
BaseEndpoint = "https://api.osv.dev/v1",
HealthCheckEndpoint = "https://api.osv.dev/v1/query",
HttpClientName = "PackagistClient",
RequiresAuthentication = false,
DefaultPriority = 64,
Tags = ImmutableArray.Create("packagist", "ecosystem", "php")
};
public static readonly SourceDefinition Hex = new()
{
Id = "hex",
DisplayName = "Hex.pm Advisories",
Category = SourceCategory.Ecosystem,
Type = SourceType.Upstream,
Description = "Elixir Hex.pm Security Advisories (via OSV)",
BaseEndpoint = "https://api.osv.dev/v1",
HealthCheckEndpoint = "https://api.osv.dev/v1/query",
HttpClientName = "HexClient",
RequiresAuthentication = false,
DefaultPriority = 66,
Tags = ImmutableArray.Create("hex", "ecosystem", "elixir", "erlang")
};
// ===== CSAF/VEX Sources =====
public static readonly SourceDefinition Csaf = new()
{
Id = "csaf",
DisplayName = "CSAF Aggregator",
Category = SourceCategory.Csaf,
Type = SourceType.Upstream,
Description = "Common Security Advisory Framework",
BaseEndpoint = "https://csaf-aggregator.oasis-open.org/",
HealthCheckEndpoint = "https://csaf-aggregator.oasis-open.org/",
HttpClientName = "CsafClient",
RequiresAuthentication = false,
DefaultPriority = 70,
Tags = ImmutableArray.Create("csaf", "vex", "structured")
};
public static readonly SourceDefinition CsafTc = new()
{
Id = "csaf-tc",
DisplayName = "CSAF TC Trusted Publishers",
Category = SourceCategory.Csaf,
Type = SourceType.Upstream,
Description = "OASIS CSAF TC Trusted Publisher List",
BaseEndpoint = "https://csaf.io/",
HealthCheckEndpoint = "https://csaf.io/",
HttpClientName = "CsafTcClient",
RequiresAuthentication = false,
DefaultPriority = 72,
Tags = ImmutableArray.Create("csaf", "oasis", "trusted")
};
public static readonly SourceDefinition Vex = new()
{
Id = "vex",
DisplayName = "VEX Hub",
Category = SourceCategory.Csaf,
Type = SourceType.Upstream,
Description = "Vulnerability Exploitability eXchange documents",
BaseEndpoint = "https://vexhub.example.com/",
HealthCheckEndpoint = "https://vexhub.example.com/",
HttpClientName = "VexClient",
RequiresAuthentication = false,
DefaultPriority = 74,
Tags = ImmutableArray.Create("vex", "exploitability")
};
// ===== CERTs =====
public static readonly SourceDefinition CertFr = new()
{
Id = "cert-fr",
DisplayName = "CERT-FR",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "French CERT",
BaseEndpoint = "https://www.cert.ssi.gouv.fr/",
HealthCheckEndpoint = "https://www.cert.ssi.gouv.fr/",
HttpClientName = "CertFrClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("FR", "EU"),
DefaultPriority = 80,
Tags = ImmutableArray.Create("cert", "france", "eu")
};
public static readonly SourceDefinition CertDe = new()
{
Id = "cert-de",
DisplayName = "CERT-Bund (Germany)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "German Federal CERT",
BaseEndpoint = "https://www.bsi.bund.de/DE/Themen/Unternehmen-und-Organisationen/Cyber-Sicherheitslage/Technische-Sicherheitshinweise/",
HealthCheckEndpoint = "https://www.bsi.bund.de/",
HttpClientName = "CertDeClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("DE", "EU"),
DefaultPriority = 82,
Tags = ImmutableArray.Create("cert", "germany", "eu")
};
public static readonly SourceDefinition CertAt = new()
{
Id = "cert-at",
DisplayName = "CERT.at (Austria)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Austrian CERT",
BaseEndpoint = "https://cert.at/",
HealthCheckEndpoint = "https://cert.at/",
HttpClientName = "CertAtClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("AT", "EU"),
DefaultPriority = 84,
Tags = ImmutableArray.Create("cert", "austria", "eu")
};
public static readonly SourceDefinition CertBe = new()
{
Id = "cert-be",
DisplayName = "CERT.be (Belgium)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Belgian CERT",
BaseEndpoint = "https://cert.be/",
HealthCheckEndpoint = "https://cert.be/",
HttpClientName = "CertBeClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("BE", "EU"),
DefaultPriority = 86,
Tags = ImmutableArray.Create("cert", "belgium", "eu")
};
public static readonly SourceDefinition CertCh = new()
{
Id = "cert-ch",
DisplayName = "NCSC-CH (Switzerland)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Swiss National Cyber Security Centre",
BaseEndpoint = "https://www.ncsc.admin.ch/",
HealthCheckEndpoint = "https://www.ncsc.admin.ch/",
HttpClientName = "CertChClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("CH"),
DefaultPriority = 88,
Tags = ImmutableArray.Create("cert", "switzerland")
};
public static readonly SourceDefinition CertEu = new()
{
Id = "cert-eu",
DisplayName = "CERT-EU",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "EU CERT for EU Institutions",
BaseEndpoint = "https://cert.europa.eu/",
HealthCheckEndpoint = "https://cert.europa.eu/",
HttpClientName = "CertEuClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("EU"),
DefaultPriority = 90,
Tags = ImmutableArray.Create("cert", "eu")
};
public static readonly SourceDefinition JpCert = new()
{
Id = "jpcert",
DisplayName = "JPCERT/CC (Japan)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "Japan Computer Emergency Response Team",
BaseEndpoint = "https://www.jpcert.or.jp/english/",
HealthCheckEndpoint = "https://www.jpcert.or.jp/english/",
HttpClientName = "JpCertClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("JP", "APAC"),
DefaultPriority = 92,
Tags = ImmutableArray.Create("cert", "japan", "apac")
};
public static readonly SourceDefinition UsCert = new()
{
Id = "us-cert",
DisplayName = "CISA (US-CERT)",
Category = SourceCategory.Cert,
Type = SourceType.Upstream,
Description = "US Cybersecurity and Infrastructure Security Agency",
BaseEndpoint = "https://www.cisa.gov/news-events/cybersecurity-advisories",
HealthCheckEndpoint = "https://www.cisa.gov/",
HttpClientName = "UsCertClient",
RequiresAuthentication = false,
Regions = ImmutableArray.Create("US", "NA"),
DefaultPriority = 94,
Tags = ImmutableArray.Create("cert", "us", "cisa")
};
// ===== StellaOps Mirror =====
public static readonly SourceDefinition StellaMirror = new()
{
Id = "stella-mirror",
DisplayName = "StellaOps Mirror",
Category = SourceCategory.Mirror,
Type = SourceType.StellaMirror,
Description = "StellaOps Pre-aggregated Advisory Mirror",
BaseEndpoint = "https://mirror.stella-ops.org/api/v1",
HealthCheckEndpoint = "https://mirror.stella-ops.org/api/v1/health",
HttpClientName = "StellaMirrorClient",
RequiresAuthentication = false, // Can be configured for OAuth
StatusPageUrl = "https://status.stella-ops.org/",
DocumentationUrl = "https://docs.stella-ops.org/mirror/",
DefaultPriority = 1, // Highest priority when using mirror mode
Tags = ImmutableArray.Create("stella", "mirror", "aggregated")
};
// ===== All Sources Collection =====
/// <summary>
/// All registered source definitions.
/// Must be declared after all individual sources due to static initialization order.
/// </summary>
public static readonly ImmutableArray<SourceDefinition> All = ImmutableArray.Create(
// Primary databases
Nvd, Osv, Ghsa, Cve, Epss, Kev,
// Vendor advisories
RedHat, Microsoft, Amazon, Google, Oracle, Apple, Cisco, Fortinet, Juniper, Palo, Vmware,
// Linux distributions
Debian, Ubuntu, Alpine, Suse, Rhel, Centos, Fedora, Arch, Gentoo,
// Ecosystems
Npm, PyPi, Go, RubyGems, Nuget, Maven, Crates, Packagist, Hex,
// CSAF/VEX
Csaf, CsafTc, Vex,
// CERTs
CertFr, CertDe, CertAt, CertBe, CertCh, CertEu, JpCert, UsCert,
// Mirrors
StellaMirror);
// ===== Helper Methods =====
/// <summary>
/// Get sources by category.
/// </summary>
public static ImmutableArray<SourceDefinition> GetByCategory(SourceCategory category)
=> All.Where(s => s.Category == category).ToImmutableArray();
/// <summary>
/// Get sources by tag.
/// </summary>
public static ImmutableArray<SourceDefinition> GetByTag(string tag)
=> All.Where(s => s.Tags.Contains(tag, StringComparer.OrdinalIgnoreCase)).ToImmutableArray();
/// <summary>
/// Get sources by region.
/// </summary>
public static ImmutableArray<SourceDefinition> GetByRegion(string region)
=> All.Where(s => s.Regions.Length == 0 || s.Regions.Contains(region, StringComparer.OrdinalIgnoreCase))
.ToImmutableArray();
/// <summary>
/// Get sources that require authentication.
/// </summary>
public static ImmutableArray<SourceDefinition> GetAuthenticatedSources()
=> All.Where(s => s.RequiresAuthentication).ToImmutableArray();
/// <summary>
/// Find a source by ID.
/// </summary>
public static SourceDefinition? FindById(string sourceId)
=> All.FirstOrDefault(s => s.Id.Equals(sourceId, StringComparison.OrdinalIgnoreCase));
}

427
src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceErrorDetails.cs Normal file
View File

@@ -0,0 +1,427 @@
// -----------------------------------------------------------------------------
// SourceErrorDetails.cs
// Sprint: SPRINT_20260114_SOURCES_SETUP
// Task: 3.3 - Source Error Details
// Description: Detailed error information with why/how explanations
// -----------------------------------------------------------------------------
using System.Collections.Immutable;
using System.Net;
namespace StellaOps.Concelier.Core.Sources;
/// <summary>
/// Detailed error information for a source connectivity failure.
/// Provides "why" and "how to fix" explanations for users.
/// </summary>
public sealed record SourceErrorDetails
{
/// <summary>
/// Error code for categorization.
/// </summary>
public required string Code { get; init; }
/// <summary>
/// Human-readable error message.
/// </summary>
public required string Message { get; init; }
/// <summary>
/// HTTP status code if applicable.
/// </summary>
public int? HttpStatusCode { get; init; }
/// <summary>
/// Possible reasons for the failure.
/// </summary>
public ImmutableArray<string> PossibleReasons { get; init; }
= ImmutableArray<string>.Empty;
/// <summary>
/// Steps to remediate the issue.
/// </summary>
public ImmutableArray<RemediationStep> RemediationSteps { get; init; }
= ImmutableArray<RemediationStep>.Empty;
}
/// <summary>
/// Factory for creating source-specific error details.
/// </summary>
public static class SourceErrorFactory
{
/// <summary>
/// Create error details from an HTTP response.
/// </summary>
public static SourceErrorDetails FromHttpResponse(
SourceDefinition source,
HttpStatusCode statusCode,
string? responseBody = null)
{
return statusCode switch
{
HttpStatusCode.Unauthorized => CreateAuthError(source, statusCode),
HttpStatusCode.Forbidden => CreateForbiddenError(source, statusCode),
HttpStatusCode.NotFound => CreateNotFoundError(source, statusCode),
HttpStatusCode.TooManyRequests => CreateRateLimitError(source, statusCode),
HttpStatusCode.ServiceUnavailable => CreateServiceDownError(source, statusCode),
HttpStatusCode.BadGateway => CreateNetworkError(source, statusCode),
HttpStatusCode.GatewayTimeout => CreateTimeoutError(source, statusCode),
_ => CreateGenericHttpError(source, statusCode, responseBody)
};
}
/// <summary>
/// Create error details from a network exception.
/// </summary>
public static SourceErrorDetails FromNetworkException(
SourceDefinition source,
Exception exception)
{
return exception switch
{
HttpRequestException httpEx when httpEx.InnerException is System.Net.Sockets.SocketException =>
CreateDnsError(source),
HttpRequestException httpEx when httpEx.Message.Contains("SSL", StringComparison.OrdinalIgnoreCase) =>
CreateSslError(source),
TaskCanceledException =>
CreateTimeoutError(source, null),
_ => CreateNetworkError(source, null)
};
}
private static SourceErrorDetails CreateAuthError(SourceDefinition source, HttpStatusCode statusCode)
{
var (envVar, tokenUrl, tokenScopes) = GetSourceCredentialInfo(source.Id);
return new SourceErrorDetails
{
Code = "AUTH_REQUIRED",
Message = $"Authentication failed for {source.DisplayName}",
HttpStatusCode = (int)statusCode,
PossibleReasons = ImmutableArray.Create(
$"API key or token for {source.DisplayName} is not set",
"The API key may have expired",
"The API key may have been revoked"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = $"Obtain an API key from {source.DisplayName}",
DocumentationUrl = tokenUrl,
CommandType = CommandType.Url
},
new RemediationStep
{
Order = 2,
Description = $"Set the {envVar} environment variable",
Command = $"export {envVar}=\"your-api-key\"",
CommandType = CommandType.Bash
},
new RemediationStep
{
Order = 3,
Description = "Retry the connectivity check",
Command = $"stella sources check --source {source.Id}",
CommandType = CommandType.StellaCli
})
};
}
private static SourceErrorDetails CreateForbiddenError(SourceDefinition source, HttpStatusCode statusCode)
{
var (envVar, tokenUrl, tokenScopes) = GetSourceCredentialInfo(source.Id);
return new SourceErrorDetails
{
Code = "ACCESS_DENIED",
Message = $"Access denied to {source.DisplayName}",
HttpStatusCode = (int)statusCode,
PossibleReasons = ImmutableArray.Create(
"The API key lacks required permissions/scopes",
"Your account may not have access to this resource",
"IP-based access restrictions may be in place"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = $"Verify your API key has the required scopes: {tokenScopes}",
DocumentationUrl = tokenUrl,
CommandType = CommandType.Url
},
new RemediationStep
{
Order = 2,
Description = "Generate a new API key with correct permissions",
CommandType = CommandType.Url,
DocumentationUrl = tokenUrl
})
};
}
private static SourceErrorDetails CreateNotFoundError(SourceDefinition source, HttpStatusCode statusCode)
{
return new SourceErrorDetails
{
Code = "ENDPOINT_NOT_FOUND",
Message = $"Endpoint not found for {source.DisplayName}",
HttpStatusCode = (int)statusCode,
PossibleReasons = ImmutableArray.Create(
"The API endpoint may have changed",
"The source may be temporarily unavailable",
"A custom endpoint URL may be incorrect"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = $"Check the {source.DisplayName} status page",
DocumentationUrl = source.StatusPageUrl,
CommandType = CommandType.Url
},
new RemediationStep
{
Order = 2,
Description = "Verify custom endpoint configuration if set",
Command = "stella config get concelier.sources",
CommandType = CommandType.StellaCli
})
};
}
private static SourceErrorDetails CreateRateLimitError(SourceDefinition source, HttpStatusCode statusCode)
{
var (envVar, tokenUrl, _) = GetSourceCredentialInfo(source.Id);
return new SourceErrorDetails
{
Code = "RATE_LIMITED",
Message = $"Rate limit exceeded for {source.DisplayName}",
HttpStatusCode = (int)statusCode,
PossibleReasons = ImmutableArray.Create(
"Too many requests in a short period",
"API key may have lower rate limits",
"Multiple instances may be sharing the same key"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = "Wait a few minutes and retry",
CommandType = CommandType.Bash
},
new RemediationStep
{
Order = 2,
Description = "Consider using an API key for higher rate limits",
DocumentationUrl = tokenUrl,
CommandType = CommandType.Url
},
new RemediationStep
{
Order = 3,
Description = "Adjust request delay in configuration",
Command = $"stella config set concelier.sources.{source.Id}.requestDelay 00:00:01",
CommandType = CommandType.StellaCli
})
};
}
private static SourceErrorDetails CreateServiceDownError(SourceDefinition source, HttpStatusCode statusCode)
{
return new SourceErrorDetails
{
Code = "SERVICE_DOWN",
Message = $"{source.DisplayName} is currently unavailable",
HttpStatusCode = (int)statusCode,
PossibleReasons = ImmutableArray.Create(
"The service may be undergoing maintenance",
"There may be an outage affecting the service",
"Network routing issues may be occurring"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = $"Check {source.DisplayName} status page",
DocumentationUrl = source.StatusPageUrl,
CommandType = CommandType.Url
},
new RemediationStep
{
Order = 2,
Description = "Wait and retry later",
CommandType = CommandType.Bash
},
new RemediationStep
{
Order = 3,
Description = "Consider using cached/offline data temporarily",
Command = "stella sources enable-offline-fallback",
CommandType = CommandType.StellaCli
})
};
}
private static SourceErrorDetails CreateNetworkError(SourceDefinition source, HttpStatusCode? statusCode)
{
return new SourceErrorDetails
{
Code = "NETWORK_ERROR",
Message = $"Network error connecting to {source.DisplayName}",
HttpStatusCode = statusCode.HasValue ? (int)statusCode.Value : null,
PossibleReasons = ImmutableArray.Create(
"Firewall may be blocking the connection",
"Proxy configuration may be required",
"Network connectivity issues"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = "Check network connectivity",
Command = $"curl -v {source.HealthCheckEndpoint}",
CommandType = CommandType.Bash
},
new RemediationStep
{
Order = 2,
Description = "Verify firewall rules allow outbound HTTPS",
CommandType = CommandType.Bash
},
new RemediationStep
{
Order = 3,
Description = "Configure proxy if required",
Command = "export HTTPS_PROXY=http://proxy:8080",
CommandType = CommandType.Bash
})
};
}
private static SourceErrorDetails CreateDnsError(SourceDefinition source)
{
return new SourceErrorDetails
{
Code = "DNS_ERROR",
Message = $"DNS resolution failed for {source.DisplayName}",
PossibleReasons = ImmutableArray.Create(
"DNS server may be unreachable",
"The hostname may be incorrect",
"Network configuration issues"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = "Check DNS resolution",
Command = $"nslookup {new Uri(source.HealthCheckEndpoint).Host}",
CommandType = CommandType.Bash
},
new RemediationStep
{
Order = 2,
Description = "Verify DNS server configuration",
CommandType = CommandType.Bash
})
};
}
private static SourceErrorDetails CreateSslError(SourceDefinition source)
{
return new SourceErrorDetails
{
Code = "SSL_ERROR",
Message = $"SSL/TLS error connecting to {source.DisplayName}",
PossibleReasons = ImmutableArray.Create(
"SSL certificate may be invalid or expired",
"Certificate chain verification failed",
"TLS version mismatch"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = "Verify the SSL certificate",
Command = $"openssl s_client -connect {new Uri(source.HealthCheckEndpoint).Host}:443",
CommandType = CommandType.Bash
},
new RemediationStep
{
Order = 2,
Description = "Update CA certificates if outdated",
Command = "sudo update-ca-certificates",
CommandType = CommandType.Bash
})
};
}
private static SourceErrorDetails CreateTimeoutError(SourceDefinition source, HttpStatusCode? statusCode)
{
return new SourceErrorDetails
{
Code = "TIMEOUT",
Message = $"Connection to {source.DisplayName} timed out",
HttpStatusCode = statusCode.HasValue ? (int)statusCode.Value : null,
PossibleReasons = ImmutableArray.Create(
"The service may be slow or overloaded",
"Network latency may be high",
"Connection timeout may be too short"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = "Increase connection timeout",
Command = "stella config set concelier.connectivityCheckTimeoutSeconds 60",
CommandType = CommandType.StellaCli
},
new RemediationStep
{
Order = 2,
Description = "Check network latency",
Command = $"ping {new Uri(source.HealthCheckEndpoint).Host}",
CommandType = CommandType.Bash
})
};
}
private static SourceErrorDetails CreateGenericHttpError(
SourceDefinition source,
HttpStatusCode statusCode,
string? responseBody)
{
return new SourceErrorDetails
{
Code = "HTTP_ERROR",
Message = $"HTTP {(int)statusCode} response from {source.DisplayName}",
HttpStatusCode = (int)statusCode,
PossibleReasons = ImmutableArray.Create(
"An unexpected error occurred",
"The API may be experiencing issues"),
RemediationSteps = ImmutableArray.Create(
new RemediationStep
{
Order = 1,
Description = $"Check {source.DisplayName} status page",
DocumentationUrl = source.StatusPageUrl,
CommandType = CommandType.Url
},
new RemediationStep
{
Order = 2,
Description = "Retry the connectivity check",
Command = $"stella sources check --source {source.Id}",
CommandType = CommandType.StellaCli
})
};
}
private static (string EnvVar, string TokenUrl, string Scopes) GetSourceCredentialInfo(string sourceId)
{
return sourceId.ToUpperInvariant() switch
{
"NVD" => ("NVD_API_KEY", "https://nvd.nist.gov/developers/request-an-api-key", "N/A"),
"GHSA" => ("GITHUB_PAT", "https://github.com/settings/tokens", "read:packages, read:org"),
"OSV" => ("OSV_API_KEY", "https://osv.dev", "N/A (optional)"),
"EPSS" => ("EPSS_API_KEY", "https://www.first.org/epss/api", "N/A (no auth required)"),
"KEV" => ("KEV_API_KEY", "https://www.cisa.gov/known-exploited-vulnerabilities-catalog", "N/A (no auth)"),
"REDHAT" => ("REDHAT_API_KEY", "https://access.redhat.com/hydra/rest/securitydata/", "N/A"),
"DEBIAN" => ("DEBIAN_API_KEY", "https://security-tracker.debian.org/", "N/A (no auth)"),
"UBUNTU" => ("UBUNTU_API_KEY", "https://ubuntu.com/security/cves", "N/A (no auth)"),
_ => ($"{sourceId.ToUpperInvariant()}_API_KEY", "", "")
};
}
}

369
src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourceRegistry.cs Normal file
View File

@@ -0,0 +1,369 @@
// -----------------------------------------------------------------------------
// SourceRegistry.cs
// Sprint: SPRINT_20260114_SOURCES_SETUP
// Task: 3.3 - Source Registry Implementation
// Description: Registry for managing and checking advisory source connectivity
// -----------------------------------------------------------------------------
using System.Collections.Concurrent;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Net;
using System.Net.Http;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.Core.Configuration;
namespace StellaOps.Concelier.Core.Sources;
/// <summary>
/// Default implementation of <see cref="ISourceRegistry"/>.
/// Manages source connectivity checking with auto-enable for healthy sources.
/// </summary>
public sealed class SourceRegistry : ISourceRegistry
{
private readonly ImmutableArray<SourceDefinition> _sources;
private readonly IHttpClientFactory _httpClientFactory;
private readonly ILogger<SourceRegistry> _logger;
private readonly TimeProvider _timeProvider;
private readonly SourcesConfiguration _configuration;
private readonly ConcurrentDictionary<string, bool> _enabledSources;
private readonly ConcurrentDictionary<string, SourceConnectivityResult> _lastCheckResults;
public SourceRegistry(
IHttpClientFactory httpClientFactory,
ILogger<SourceRegistry> logger,
TimeProvider? timeProvider = null,
SourcesConfiguration? configuration = null)
{
_httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory));
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
_timeProvider = timeProvider ?? TimeProvider.System;
_configuration = configuration ?? new SourcesConfiguration();
_sources = SourceDefinitions.All;
_enabledSources = new ConcurrentDictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
_lastCheckResults = new ConcurrentDictionary<string, SourceConnectivityResult>(StringComparer.OrdinalIgnoreCase);
// Initialize enabled state from definitions
foreach (var source in _sources)
{
_enabledSources[source.Id] = source.EnabledByDefault;
}
}
/// <inheritdoc />
public IReadOnlyList<SourceDefinition> GetAllSources() => _sources;
/// <inheritdoc />
public SourceDefinition? GetSource(string sourceId)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceId);
return _sources.FirstOrDefault(s => s.Id.Equals(sourceId, StringComparison.OrdinalIgnoreCase));
}
/// <inheritdoc />
public IReadOnlyList<SourceDefinition> GetSourcesByCategory(SourceCategory category)
=> _sources.Where(s => s.Category == category).ToList();
/// <inheritdoc />
public async Task<SourceConnectivityResult> CheckConnectivityAsync(
string sourceId,
CancellationToken cancellationToken = default)
{
var source = GetSource(sourceId);
if (source is null)
{
var notFound = SourceConnectivityResult.NotFound(sourceId);
_lastCheckResults[sourceId] = notFound;
return notFound;
}
var stopwatch = Stopwatch.StartNew();
var checkedAt = _timeProvider.GetUtcNow();
try
{
using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
cts.CancelAfter(TimeSpan.FromSeconds(_configuration.ConnectivityCheckTimeoutSeconds));
var client = _httpClientFactory.CreateClient(source.HttpClientName);
// Set appropriate headers for the source
ConfigureClientHeaders(client, source);
_logger.LogDebug(
"Checking connectivity for source {SourceId} at {Endpoint}",
sourceId, source.HealthCheckEndpoint);
var response = await client.GetAsync(
source.HealthCheckEndpoint,
HttpCompletionOption.ResponseHeadersRead,
cts.Token);
stopwatch.Stop();
if (response.IsSuccessStatusCode)
{
var result = SourceConnectivityResult.Healthy(sourceId, stopwatch.Elapsed, checkedAt);
_lastCheckResults[sourceId] = result;
_logger.LogInformation(
"Source {SourceId} is healthy (latency: {Latency}ms)",
sourceId, stopwatch.Elapsed.TotalMilliseconds);
return result;
}
// Handle non-success status codes
var errorDetails = SourceErrorFactory.FromHttpResponse(source, response.StatusCode);
var failedResult = SourceConnectivityResult.Failed(
sourceId,
errorDetails.Code,
errorDetails.Message,
errorDetails.PossibleReasons,
errorDetails.RemediationSteps,
checkedAt,
stopwatch.Elapsed,
(int)response.StatusCode);
_lastCheckResults[sourceId] = failedResult;
_logger.LogWarning(
"Source {SourceId} failed connectivity check: {StatusCode} - {ErrorMessage}",
sourceId, response.StatusCode, errorDetails.Message);
return failedResult;
}
catch (HttpRequestException ex)
{
stopwatch.Stop();
var errorDetails = SourceErrorFactory.FromNetworkException(source, ex);
var failedResult = SourceConnectivityResult.Failed(
sourceId,
errorDetails.Code,
errorDetails.Message,
errorDetails.PossibleReasons,
errorDetails.RemediationSteps,
checkedAt,
stopwatch.Elapsed);
_lastCheckResults[sourceId] = failedResult;
_logger.LogWarning(ex,
"Source {SourceId} failed connectivity check (network error): {ErrorMessage}",
sourceId, errorDetails.Message);
return failedResult;
}
catch (TaskCanceledException) when (cancellationToken.IsCancellationRequested)
{
throw;
}
catch (TaskCanceledException)
{
stopwatch.Stop();
var errorDetails = SourceErrorFactory.FromNetworkException(source, new TaskCanceledException());
var failedResult = SourceConnectivityResult.Failed(
sourceId,
errorDetails.Code,
errorDetails.Message,
errorDetails.PossibleReasons,
errorDetails.RemediationSteps,
checkedAt,
stopwatch.Elapsed);
_lastCheckResults[sourceId] = failedResult;
_logger.LogWarning(
"Source {SourceId} connectivity check timed out after {Timeout}s",
sourceId, _configuration.ConnectivityCheckTimeoutSeconds);
return failedResult;
}
catch (Exception ex)
{
stopwatch.Stop();
var failedResult = SourceConnectivityResult.Failed(
sourceId,
"UNEXPECTED_ERROR",
$"Unexpected error: {ex.Message}",
ImmutableArray.Create("An unexpected error occurred during connectivity check"),
ImmutableArray.Create(new RemediationStep
{
Order = 1,
Description = "Check the logs for detailed error information",
CommandType = CommandType.Bash
}),
checkedAt,
stopwatch.Elapsed);
_lastCheckResults[sourceId] = failedResult;
_logger.LogError(ex,
"Source {SourceId} connectivity check failed with unexpected error",
sourceId);
return failedResult;
}
}
/// <inheritdoc />
public async Task<SourceCheckResult> CheckAllAndAutoConfigureAsync(
CancellationToken cancellationToken = default)
{
var startTime = _timeProvider.GetUtcNow();
var stopwatch = Stopwatch.StartNew();
_logger.LogInformation(
"Starting connectivity check for {SourceCount} sources",
_sources.Length);
// Check all sources in parallel with limited concurrency
var semaphore = new SemaphoreSlim(10); // Max 10 concurrent checks
var tasks = _sources.Select(async source =>
{
await semaphore.WaitAsync(cancellationToken);
try
{
return await CheckConnectivityAsync(source.Id, cancellationToken);
}
finally
{
semaphore.Release();
}
});
var results = await Task.WhenAll(tasks);
stopwatch.Stop();
// Auto-configure based on results
if (_configuration.AutoEnableHealthySources)
{
foreach (var result in results)
{
_enabledSources[result.SourceId] = result.IsHealthy;
}
}
var checkResult = SourceCheckResult.FromResults(results, startTime, stopwatch.Elapsed);
_logger.LogInformation(
"Connectivity check completed: {HealthyCount}/{Total} healthy, {FailedCount} failed (duration: {Duration}ms)",
checkResult.HealthyCount,
checkResult.TotalChecked,
checkResult.FailedCount,
stopwatch.Elapsed.TotalMilliseconds);
return checkResult;
}
/// <inheritdoc />
public async Task<ImmutableArray<SourceConnectivityResult>> CheckMultipleAsync(
IEnumerable<string> sourceIds,
CancellationToken cancellationToken = default)
{
var ids = sourceIds.ToList();
var tasks = ids.Select(id => CheckConnectivityAsync(id, cancellationToken));
var results = await Task.WhenAll(tasks);
return results.ToImmutableArray();
}
/// <inheritdoc />
public Task<bool> EnableSourceAsync(
string sourceId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceId);
var source = GetSource(sourceId);
if (source is null)
{
_logger.LogWarning("Attempted to enable unknown source: {SourceId}", sourceId);
return Task.FromResult(false);
}
_enabledSources[sourceId] = true;
_logger.LogInformation("Enabled source: {SourceId}", sourceId);
return Task.FromResult(true);
}
/// <inheritdoc />
public Task<bool> DisableSourceAsync(
string sourceId,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(sourceId);
var source = GetSource(sourceId);
if (source is null)
{
_logger.LogWarning("Attempted to disable unknown source: {SourceId}", sourceId);
return Task.FromResult(false);
}
_enabledSources[sourceId] = false;
_logger.LogInformation("Disabled source: {SourceId}", sourceId);
return Task.FromResult(true);
}
/// <inheritdoc />
public Task<ImmutableArray<string>> GetEnabledSourcesAsync(
CancellationToken cancellationToken = default)
{
var enabled = _enabledSources
.Where(kvp => kvp.Value)
.Select(kvp => kvp.Key)
.ToImmutableArray();
return Task.FromResult(enabled);
}
/// <inheritdoc />
public Task<SourceConnectivityResult> RetryCheckAsync(
string sourceId,
CancellationToken cancellationToken = default)
{
_logger.LogInformation("Retrying connectivity check for source: {SourceId}", sourceId);
return CheckConnectivityAsync(sourceId, cancellationToken);
}
/// <summary>
/// Get the last connectivity check result for a source.
/// </summary>
public SourceConnectivityResult? GetLastCheckResult(string sourceId)
{
return _lastCheckResults.GetValueOrDefault(sourceId);
}
/// <summary>
/// Check if a source is currently enabled.
/// </summary>
public bool IsEnabled(string sourceId)
{
return _enabledSources.GetValueOrDefault(sourceId);
}
private static void ConfigureClientHeaders(HttpClient client, SourceDefinition source)
{
// Set a reasonable timeout
client.Timeout = TimeSpan.FromSeconds(30);
// Set User-Agent
if (!client.DefaultRequestHeaders.Contains("User-Agent"))
{
client.DefaultRequestHeaders.Add(
"User-Agent",
$"StellaOps.Concelier/{typeof(SourceRegistry).Assembly.GetName().Version} (+https://stella-ops.org)");
}
// Set Accept header for JSON APIs
if (!client.DefaultRequestHeaders.Contains("Accept"))
{
client.DefaultRequestHeaders.Add("Accept", "application/json");
}
// Source-specific headers would be configured via named HttpClient in DI
}
}

174
src/Concelier/__Libraries/StellaOps.Concelier.Core/Sources/SourcesServiceCollectionExtensions.cs Normal file
View File

@@ -0,0 +1,174 @@
// -----------------------------------------------------------------------------
// SourcesServiceCollectionExtensions.cs
// Sprint: SPRINT_20260114_SOURCES_SETUP
// Task: 3.3 - Source Registry DI Registration
// Description: Extension methods for registering source services
// -----------------------------------------------------------------------------
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.DependencyInjection.Extensions;
using StellaOps.Concelier.Core.Configuration;
namespace StellaOps.Concelier.Core.Sources;
/// <summary>
/// Extension methods for registering advisory source services.
/// </summary>
public static class SourcesServiceCollectionExtensions
{
/// <summary>
/// Adds advisory source registry and related services.
/// </summary>
/// <param name="services">Service collection.</param>
/// <param name="configuration">Configuration instance.</param>
/// <returns>Service collection for chaining.</returns>
public static IServiceCollection AddSourcesRegistry(
this IServiceCollection services,
IConfiguration configuration)
{
// Bind configuration
services.Configure<SourcesConfiguration>(
configuration.GetSection("sources"));
// Register TimeProvider if not already registered
services.TryAddSingleton(TimeProvider.System);
// Register the source registry
services.AddSingleton<ISourceRegistry, SourceRegistry>();
// Configure HTTP clients for sources
ConfigureSourceHttpClients(services);
return services;
}
/// <summary>
/// Adds advisory source registry with custom configuration.
/// </summary>
/// <param name="services">Service collection.</param>
/// <param name="configure">Configuration action.</param>
/// <returns>Service collection for chaining.</returns>
public static IServiceCollection AddSourcesRegistry(
this IServiceCollection services,
Action<SourcesConfiguration> configure)
{
var config = new SourcesConfiguration();
configure(config);
services.AddSingleton(config);
// Register TimeProvider if not already registered
services.TryAddSingleton(TimeProvider.System);
// Register the source registry
services.AddSingleton<ISourceRegistry, SourceRegistry>();
// Configure HTTP clients for sources
ConfigureSourceHttpClients(services);
return services;
}
private static void ConfigureSourceHttpClients(IServiceCollection services)
{
// Configure named HTTP clients for each source
// These can be overridden by the application for custom configuration
// NVD client
services.AddHttpClient("NvdClient", client =>
{
client.BaseAddress = new Uri("https://services.nvd.nist.gov/rest/json/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// OSV client
services.AddHttpClient("OsvClient", client =>
{
client.BaseAddress = new Uri("https://api.osv.dev/v1/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// GHSA client
services.AddHttpClient("GhsaClient", client =>
{
client.BaseAddress = new Uri("https://api.github.com/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.DefaultRequestHeaders.Add("User-Agent", "StellaOps-Concelier");
client.Timeout = TimeSpan.FromSeconds(30);
});
// KEV client
services.AddHttpClient("KevClient", client =>
{
client.BaseAddress = new Uri("https://www.cisa.gov/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// EPSS client
services.AddHttpClient("EpssClient", client =>
{
client.BaseAddress = new Uri("https://api.first.org/data/v1/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// CVE/MITRE client
services.AddHttpClient("CveClient", client =>
{
client.BaseAddress = new Uri("https://cveawg.mitre.org/api/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Red Hat client
services.AddHttpClient("RedHatClient", client =>
{
client.BaseAddress = new Uri("https://access.redhat.com/hydra/rest/securitydata/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Debian client
services.AddHttpClient("DebianClient", client =>
{
client.BaseAddress = new Uri("https://security-tracker.debian.org/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Ubuntu client
services.AddHttpClient("UbuntuClient", client =>
{
client.BaseAddress = new Uri("https://ubuntu.com/security/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// Alpine client
services.AddHttpClient("AlpineClient", client =>
{
client.BaseAddress = new Uri("https://secdb.alpinelinux.org/");
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
// StellaOps Mirror client
services.AddHttpClient("StellaMirrorClient", client =>
{
// Base address would be configured from settings
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(60);
});
// Default client for sources without specific configuration
services.AddHttpClient("DefaultSourceClient", client =>
{
client.DefaultRequestHeaders.Add("Accept", "application/json");
client.Timeout = TimeSpan.FromSeconds(30);
});
}
}

1
src/Concelier/__Libraries/StellaOps.Concelier.Core/StellaOps.Concelier.Core.csproj
View File

@@ -9,6 +9,7 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Caching.Memory" />
<PackageReference Include="Microsoft.Extensions.Http" />
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
<PackageReference Include="Microsoft.Extensions.Options" />
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />