release orchestrator v1 draft and build fixes

This commit is contained in:
master
2026-01-12 12:24:17 +02:00
parent f3de858c59
commit 9873f80830
1598 changed files with 240385 additions and 5944 deletions

View File

@@ -0,0 +1,380 @@
namespace StellaOps.Concelier.Plugin.Unified;
using System.Runtime.CompilerServices;
using StellaOps.Plugin;
using StellaOps.Plugin.Abstractions;
using StellaOps.Plugin.Abstractions.Capabilities;
using StellaOps.Plugin.Abstractions.Context;
using StellaOps.Plugin.Abstractions.Health;
using StellaOps.Plugin.Abstractions.Lifecycle;
using UnifiedFeedCapability = StellaOps.Plugin.Abstractions.Capabilities.IFeedCapability;
using LegacyFeedConnector = StellaOps.Plugin.IFeedConnector;
/// <summary>
/// Adapts an existing IConnectorPlugin to the unified IPlugin and IFeedCapability interfaces.
/// This enables gradual migration of Concelier connectors to the unified plugin architecture.
/// </summary>
/// <remarks>
/// The adapter bridges the Concelier-specific IConnectorPlugin/IFeedConnector interfaces to
/// the Plugin.Abstractions IFeedCapability interface. The underlying connector operations
/// are delegated to the wrapped connector.
/// </remarks>
public sealed class FeedPluginAdapter : IPlugin, UnifiedFeedCapability
{
private readonly IConnectorPlugin _plugin;
private readonly FeedType _feedType;
private readonly IReadOnlyList<string> _ecosystems;
private IPluginContext? _context;
private IServiceProvider? _serviceProvider;
private LegacyFeedConnector? _connector;
private PluginLifecycleState _state = PluginLifecycleState.Discovered;
/// <summary>
/// Creates a new adapter for an existing connector plugin.
/// </summary>
/// <param name="plugin">The existing connector plugin to wrap.</param>
/// <param name="feedType">The feed type classification.</param>
/// <param name="ecosystems">Ecosystems covered by this feed.</param>
public FeedPluginAdapter(
IConnectorPlugin plugin,
FeedType feedType,
IReadOnlyList<string> ecosystems)
{
_plugin = plugin ?? throw new ArgumentNullException(nameof(plugin));
_feedType = feedType;
_ecosystems = ecosystems ?? Array.Empty<string>();
}
#region IPlugin
/// <inheritdoc />
public PluginInfo Info => new(
Id: $"com.stellaops.feed.{_plugin.Name}",
Name: _plugin.Name,
Version: "1.0.0",
Vendor: "Stella Ops",
Description: $"Vulnerability feed connector for {_plugin.Name}");
/// <inheritdoc />
public PluginTrustLevel TrustLevel => PluginTrustLevel.BuiltIn;
/// <inheritdoc />
public PluginCapabilities Capabilities => PluginCapabilities.Feed | PluginCapabilities.Network;
/// <inheritdoc />
public PluginLifecycleState State => _state;
/// <inheritdoc />
public async Task InitializeAsync(IPluginContext context, CancellationToken ct)
{
_context = context;
_state = PluginLifecycleState.Initializing;
try
{
// Get service provider from context
// Note: This would require IPluginContext to expose the service provider
// For now we create a minimal service provider
_serviceProvider = CreateServiceProvider(context);
if (!_plugin.IsAvailable(_serviceProvider))
{
_state = PluginLifecycleState.Failed;
throw new InvalidOperationException(
$"Feed connector '{_plugin.Name}' is not available.");
}
// Create the actual connector
_connector = _plugin.Create(_serviceProvider);
_state = PluginLifecycleState.Active;
context.Logger.Info("Feed connector adapter initialized for {FeedId}", _plugin.Name);
}
catch (Exception ex)
{
_state = PluginLifecycleState.Failed;
context.Logger.Error(ex, "Failed to initialize feed connector {FeedId}", _plugin.Name);
throw;
}
await Task.CompletedTask;
}
/// <inheritdoc />
public async Task<HealthCheckResult> HealthCheckAsync(CancellationToken ct)
{
try
{
if (_state != PluginLifecycleState.Active)
{
return HealthCheckResult.Unhealthy($"Connector is in state {_state}");
}
if (_connector == null)
{
return HealthCheckResult.Unhealthy("Connector not initialized");
}
return HealthCheckResult.Healthy()
.WithDetails(new Dictionary<string, object>
{
["feedId"] = FeedId,
["feedType"] = _feedType.ToString(),
["ecosystems"] = string.Join(", ", _ecosystems),
["sourceName"] = _connector.SourceName
});
}
catch (Exception ex)
{
return HealthCheckResult.Unhealthy(ex);
}
}
/// <inheritdoc />
public ValueTask DisposeAsync()
{
if (_connector is IAsyncDisposable asyncDisposable)
{
return asyncDisposable.DisposeAsync();
}
if (_connector is IDisposable disposable)
{
disposable.Dispose();
}
_connector = null;
_state = PluginLifecycleState.Stopped;
return ValueTask.CompletedTask;
}
#endregion
#region IConnectorCapability
/// <inheritdoc />
public string ConnectorType => $"feed.{_plugin.Name}";
/// <inheritdoc />
public string DisplayName => _plugin.Name;
/// <inheritdoc />
public async Task<ConnectionTestResult> TestConnectionAsync(CancellationToken ct)
{
if (_state != PluginLifecycleState.Active)
{
return ConnectionTestResult.Failed($"Connector is in state {_state}");
}
try
{
// For feed connectors, we can't easily test connection without a full fetch
// Return success if the connector is initialized
return ConnectionTestResult.Succeeded();
}
catch (Exception ex)
{
return ConnectionTestResult.Failed(ex.Message, ex);
}
}
/// <inheritdoc />
public async Task<ConnectionInfo> GetConnectionInfoAsync(CancellationToken ct)
{
return new ConnectionInfo(
EndpointUrl: $"feed://{_plugin.Name}",
AuthenticatedAs: null,
ConnectedSince: null,
Metadata: new Dictionary<string, object>
{
["feedType"] = _feedType.ToString(),
["ecosystems"] = _ecosystems
});
}
#endregion
#region IFeedCapability
/// <inheritdoc />
public string FeedId => _plugin.Name;
/// <inheritdoc />
public FeedType FeedType => _feedType;
/// <inheritdoc />
public IReadOnlyList<string> CoveredEcosystems => _ecosystems;
/// <inheritdoc />
public async Task<FeedMetadata> GetMetadataAsync(CancellationToken ct)
{
EnsureActive();
// The legacy IFeedConnector doesn't have direct metadata access
// Return basic metadata from the plugin
return new FeedMetadata(
FeedId: FeedId,
LastModified: null,
TotalEntries: null,
SchemaVersion: null,
Description: $"Vulnerability feed from {FeedId}");
}
/// <inheritdoc />
public IAsyncEnumerable<FeedEntry> FetchEntriesAsync(
DateTimeOffset? since,
CancellationToken ct)
{
EnsureActive();
// The legacy IFeedConnector uses FetchAsync/ParseAsync/MapAsync pipeline
// which doesn't directly stream entries. We document this limitation.
return EmptyAsyncEnumerable<FeedEntry>.ThrowNotSupported(
$"Direct entry streaming is not supported for {FeedId}. " +
"Use the legacy FetchAsync/ParseAsync/MapAsync pipeline via the connector directly.");
}
/// <inheritdoc />
public async Task<FeedEntry?> GetEntryAsync(string id, CancellationToken ct)
{
EnsureActive();
// The legacy connector doesn't support individual entry lookup
throw new NotSupportedException(
$"Individual entry lookup is not supported for {FeedId}. " +
"Use the legacy pipeline for bulk data processing.");
}
/// <inheritdoc />
public async Task<string?> GetRawEntryAsync(string id, CancellationToken ct)
{
EnsureActive();
// Not supported by legacy connector
return null;
}
#endregion
#region Extended Operations
/// <summary>
/// Gets the underlying connector for extended operations.
/// </summary>
public LegacyFeedConnector? Connector => _connector;
/// <summary>
/// Executes the fetch phase of the legacy pipeline.
/// </summary>
public async Task FetchAsync(CancellationToken ct)
{
EnsureActive();
await _connector!.FetchAsync(_serviceProvider!, ct);
}
/// <summary>
/// Executes the parse phase of the legacy pipeline.
/// </summary>
public async Task ParseAsync(CancellationToken ct)
{
EnsureActive();
await _connector!.ParseAsync(_serviceProvider!, ct);
}
/// <summary>
/// Executes the map phase of the legacy pipeline.
/// </summary>
public async Task MapAsync(CancellationToken ct)
{
EnsureActive();
await _connector!.MapAsync(_serviceProvider!, ct);
}
/// <summary>
/// Executes the full pipeline (fetch, parse, map).
/// </summary>
public async Task RunPipelineAsync(CancellationToken ct)
{
await FetchAsync(ct);
await ParseAsync(ct);
await MapAsync(ct);
}
#endregion
#region Helpers
private void EnsureActive()
{
if (_state != PluginLifecycleState.Active || _connector == null)
{
throw new InvalidOperationException(
$"Feed connector '{_plugin.Name}' is not active (state: {_state})");
}
}
private IServiceProvider CreateServiceProvider(IPluginContext context)
{
// The adapter needs to provide services that the legacy connector expects
// For now, return a minimal provider that wraps the context
return new MinimalServiceProvider(context);
}
private sealed class MinimalServiceProvider : IServiceProvider
{
private readonly IPluginContext _context;
public MinimalServiceProvider(IPluginContext context)
{
_context = context;
}
public object? GetService(Type serviceType)
{
if (serviceType == typeof(IPluginContext))
return _context;
if (serviceType == typeof(IPluginLogger))
return _context.Logger;
return null;
}
}
#endregion
}
/// <summary>
/// Helper for creating async enumerables that throw exceptions.
/// </summary>
internal static class EmptyAsyncEnumerable<T>
{
public static IAsyncEnumerable<T> ThrowNotSupported(string message)
{
return new ThrowingAsyncEnumerable(message);
}
private sealed class ThrowingAsyncEnumerable : IAsyncEnumerable<T>
{
private readonly string _message;
public ThrowingAsyncEnumerable(string message) => _message = message;
public IAsyncEnumerator<T> GetAsyncEnumerator(CancellationToken ct) =>
new ThrowingAsyncEnumerator(_message);
}
private sealed class ThrowingAsyncEnumerator : IAsyncEnumerator<T>
{
private readonly string _message;
public ThrowingAsyncEnumerator(string message) => _message = message;
public T Current => throw new NotSupportedException(_message);
public ValueTask<bool> MoveNextAsync() =>
throw new NotSupportedException(_message);
public ValueTask DisposeAsync() => ValueTask.CompletedTask;
}
}

View File

@@ -0,0 +1,230 @@
namespace StellaOps.Concelier.Plugin.Unified;
using Microsoft.Extensions.DependencyInjection;
using StellaOps.Plugin;
using StellaOps.Plugin.Abstractions;
using StellaOps.Plugin.Abstractions.Capabilities;
using UnifiedFeedCapability = StellaOps.Plugin.Abstractions.Capabilities.IFeedCapability;
/// <summary>
/// Factory for creating unified feed plugin adapters from existing Concelier connectors.
/// </summary>
public sealed class FeedPluginAdapterFactory
{
private readonly IEnumerable<IConnectorPlugin> _plugins;
private readonly Dictionary<string, FeedPluginAdapter> _adapters = new(StringComparer.OrdinalIgnoreCase);
private readonly object _lock = new();
/// <summary>
/// Known feed types for each connector.
/// </summary>
private static readonly Dictionary<string, FeedType> KnownFeedTypes = new(StringComparer.OrdinalIgnoreCase)
{
// CVE databases
["nvd"] = FeedType.Database,
["cve"] = FeedType.Database,
["osv"] = FeedType.Database,
["ghsa"] = FeedType.Advisory,
// OVAL feeds (distro-specific)
["redhat"] = FeedType.Oval,
["ubuntu"] = FeedType.Oval,
["debian"] = FeedType.Oval,
["suse"] = FeedType.Oval,
["alpine"] = FeedType.EcosystemSpecific,
// Vendor advisories
["msrc"] = FeedType.Advisory,
["cisco"] = FeedType.Advisory,
["adobe"] = FeedType.Advisory,
["apple"] = FeedType.Advisory,
["oracle"] = FeedType.Advisory,
["vmware"] = FeedType.Advisory,
["chromium"] = FeedType.Advisory,
// Government/CERT feeds
["kev"] = FeedType.Advisory,
["certfr"] = FeedType.Advisory,
["certbund"] = FeedType.Advisory,
["certcc"] = FeedType.Advisory,
["certin"] = FeedType.Advisory,
["acsc"] = FeedType.Advisory,
["cccs"] = FeedType.Advisory,
["jvn"] = FeedType.Advisory,
["kisa"] = FeedType.Advisory,
// Russian feeds
["rubdu"] = FeedType.Database,
["runkcki"] = FeedType.Advisory,
// ICS feeds
["icscisa"] = FeedType.Advisory,
["kaspersky"] = FeedType.Advisory,
// Mirror/EPSS
["stellaops-mirror"] = FeedType.Database,
["epss"] = FeedType.EcosystemSpecific
};
/// <summary>
/// Known ecosystems for each connector.
/// </summary>
private static readonly Dictionary<string, string[]> KnownEcosystems = new(StringComparer.OrdinalIgnoreCase)
{
["nvd"] = new[] { "all" },
["cve"] = new[] { "all" },
["osv"] = new[] { "npm", "pypi", "crates.io", "go", "maven", "rubygems", "nuget" },
["ghsa"] = new[] { "npm", "pypi", "go", "maven", "rubygems", "nuget", "composer" },
["redhat"] = new[] { "rpm:redhat" },
["ubuntu"] = new[] { "deb:ubuntu" },
["debian"] = new[] { "deb:debian" },
["suse"] = new[] { "rpm:suse" },
["alpine"] = new[] { "apk:alpine" },
["kev"] = new[] { "all" },
["epss"] = new[] { "all" }
};
/// <summary>
/// Creates a new factory instance.
/// </summary>
/// <param name="plugins">The available connector plugins.</param>
public FeedPluginAdapterFactory(IEnumerable<IConnectorPlugin> plugins)
{
_plugins = plugins ?? throw new ArgumentNullException(nameof(plugins));
}
/// <summary>
/// Gets all available unified feed plugins.
/// </summary>
/// <param name="serviceProvider">Service provider for availability checking.</param>
/// <returns>List of unified feed plugins.</returns>
public IReadOnlyList<IPlugin> GetAllPlugins(IServiceProvider serviceProvider)
{
var result = new List<IPlugin>();
foreach (var plugin in _plugins)
{
if (plugin.IsAvailable(serviceProvider))
{
var adapter = GetOrCreateAdapter(plugin);
if (adapter != null)
{
result.Add(adapter);
}
}
}
return result;
}
/// <summary>
/// Gets a unified feed plugin by feed ID.
/// </summary>
/// <param name="feedId">Feed identifier (e.g., "nvd", "osv", "ghsa").</param>
/// <returns>Unified feed plugin, or null if not found.</returns>
public IPlugin? GetPlugin(string feedId)
{
var plugin = _plugins.FirstOrDefault(p =>
p.Name.Equals(feedId, StringComparison.OrdinalIgnoreCase));
if (plugin == null)
{
return null;
}
return GetOrCreateAdapter(plugin);
}
/// <summary>
/// Gets the feed capability for a connector.
/// </summary>
/// <param name="feedId">Feed identifier.</param>
/// <returns>Feed capability, or null if not found.</returns>
public UnifiedFeedCapability? GetCapability(string feedId)
{
return GetPlugin(feedId) as UnifiedFeedCapability;
}
/// <summary>
/// Gets all available feed IDs.
/// </summary>
/// <returns>List of feed IDs.</returns>
public IReadOnlyList<string> GetAvailableFeedIds()
{
return _plugins.Select(p => p.Name).ToList();
}
/// <summary>
/// Gets feed plugins by type.
/// </summary>
/// <param name="feedType">Feed type to filter by.</param>
/// <param name="serviceProvider">Service provider for availability checking.</param>
/// <returns>List of plugins matching the feed type.</returns>
public IReadOnlyList<IPlugin> GetPluginsByType(FeedType feedType, IServiceProvider serviceProvider)
{
var result = new List<IPlugin>();
foreach (var plugin in _plugins)
{
if (!plugin.IsAvailable(serviceProvider))
continue;
var type = KnownFeedTypes.TryGetValue(plugin.Name, out var t)
? t
: FeedType.Database;
if (type == feedType)
{
var adapter = GetOrCreateAdapter(plugin);
if (adapter != null)
{
result.Add(adapter);
}
}
}
return result;
}
private FeedPluginAdapter? GetOrCreateAdapter(IConnectorPlugin plugin)
{
lock (_lock)
{
if (_adapters.TryGetValue(plugin.Name, out var existing))
{
return existing;
}
var feedType = KnownFeedTypes.TryGetValue(plugin.Name, out var ft)
? ft
: FeedType.Database;
var ecosystems = KnownEcosystems.TryGetValue(plugin.Name, out var eco)
? eco
: Array.Empty<string>();
var adapter = new FeedPluginAdapter(plugin, feedType, ecosystems);
_adapters[plugin.Name] = adapter;
return adapter;
}
}
}
/// <summary>
/// Extension methods for registering unified feed plugin services.
/// </summary>
public static class FeedPluginAdapterExtensions
{
/// <summary>
/// Adds unified feed plugin adapter services to the service collection.
/// </summary>
/// <param name="services">Service collection.</param>
/// <returns>Service collection for chaining.</returns>
public static IServiceCollection AddUnifiedFeedPlugins(this IServiceCollection services)
{
services.AddSingleton<FeedPluginAdapterFactory>();
return services;
}
}

View File

@@ -0,0 +1,21 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
<LangVersion>preview</LangVersion>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<Description>Unified plugin adapter for Concelier feed connectors</Description>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\..\__Libraries\StellaOps.Plugin\StellaOps.Plugin.csproj" />
<ProjectReference Include="..\..\Plugin\StellaOps.Plugin.Abstractions\StellaOps.Plugin.Abstractions.csproj" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
</ItemGroup>
</Project>

View File

@@ -12,6 +12,12 @@ public sealed class InMemoryOrchestratorRegistryStore : IOrchestratorRegistrySto
private readonly ConcurrentDictionary<(string Tenant, string ConnectorId, Guid RunId), List<OrchestratorHeartbeatRecord>> _heartbeats = new();
private readonly ConcurrentDictionary<(string Tenant, string ConnectorId, Guid RunId), List<OrchestratorCommandRecord>> _commands = new();
private readonly ConcurrentDictionary<(string Tenant, string ConnectorId, Guid RunId), OrchestratorRunManifest> _manifests = new();
private readonly TimeProvider _timeProvider;
public InMemoryOrchestratorRegistryStore(TimeProvider? timeProvider = null)
{
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <inheritdoc />
public Task UpsertAsync(OrchestratorRegistryRecord record, CancellationToken cancellationToken)
@@ -99,7 +105,7 @@ public sealed class InMemoryOrchestratorRegistryStore : IOrchestratorRegistrySto
lock (commands)
{
var now = DateTimeOffset.UtcNow;
var now = _timeProvider.GetUtcNow();
var pending = commands
.Where(c => (afterSequence is null || c.Sequence > afterSequence)
&& (c.ExpiresAt is null || c.ExpiresAt > now))

View File

@@ -19,8 +19,11 @@ public sealed record TenantScope(
/// <summary>
/// Validates that the tenant scope is well-formed.
/// </summary>
public void Validate()
/// <param name="asOf">The time to check expiry against. Defaults to current UTC time.</param>
public void Validate(DateTimeOffset? asOf = null)
{
var now = asOf ?? DateTimeOffset.UtcNow;
if (string.IsNullOrWhiteSpace(TenantId))
{
throw new TenantScopeException("auth/tenant-scope-missing", "TenantId is required");
@@ -41,7 +44,7 @@ public sealed record TenantScope(
throw new TenantScopeException("auth/tenant-scope-missing", "Required concelier scope missing");
}
if (ExpiresAt <= DateTimeOffset.UtcNow)
if (ExpiresAt <= now)
{
throw new TenantScopeException("auth/token-expired", "Token has expired");
}

View File

@@ -21,17 +21,20 @@ public sealed class BundleExportService : IBundleExportService
private readonly IBundleSigner _signer;
private readonly FederationOptions _options;
private readonly ILogger<BundleExportService> _logger;
private readonly TimeProvider _timeProvider;
public BundleExportService(
IDeltaQueryService deltaQuery,
IBundleSigner signer,
IOptions<FederationOptions> options,
ILogger<BundleExportService> logger)
ILogger<BundleExportService> logger,
TimeProvider? timeProvider = null)
{
_deltaQuery = deltaQuery;
_signer = signer;
_options = options.Value;
_logger = logger;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <inheritdoc />
@@ -164,7 +167,7 @@ public sealed class BundleExportService : IBundleExportService
await WriteEntryAsync(tarWriter, "deletions.ndjson", deletionBuffer, ct);
// Generate new cursor
var exportCursor = CursorFormat.Create(DateTimeOffset.UtcNow);
var exportCursor = CursorFormat.Create(_timeProvider.GetUtcNow());
// Build manifest
var manifest = new BundleManifest
@@ -173,7 +176,7 @@ public sealed class BundleExportService : IBundleExportService
SiteId = _options.SiteId,
ExportCursor = exportCursor,
SinceCursor = sinceCursor,
ExportedAt = DateTimeOffset.UtcNow,
ExportedAt = _timeProvider.GetUtcNow(),
Counts = new BundleCounts
{
Canonicals = canonicalCount,

View File

@@ -14,13 +14,16 @@ public sealed class DeltaQueryService : IDeltaQueryService
{
private readonly ICanonicalAdvisoryStore _canonicalStore;
private readonly ILogger<DeltaQueryService> _logger;
private readonly TimeProvider _timeProvider;
public DeltaQueryService(
ICanonicalAdvisoryStore canonicalStore,
ILogger<DeltaQueryService> logger)
ILogger<DeltaQueryService> logger,
TimeProvider? timeProvider = null)
{
_canonicalStore = canonicalStore;
_logger = logger;
_timeProvider = timeProvider ?? TimeProvider.System;
}
/// <inheritdoc />
@@ -32,7 +35,7 @@ public sealed class DeltaQueryService : IDeltaQueryService
options ??= new DeltaQueryOptions();
var sinceTimestamp = ParseCursor(sinceCursor);
var newCursor = CursorFormat.Create(DateTimeOffset.UtcNow);
var newCursor = CursorFormat.Create(_timeProvider.GetUtcNow());
_logger.LogInformation(
"Querying changes since {Cursor} (timestamp: {Since})",