Refactor and enhance scanner worker functionality
- Cleaned up code formatting and organization across multiple files for improved readability. - Introduced `OsScanAnalyzerDispatcher` to handle OS analyzer execution and plugin loading. - Updated `ScanJobContext` to include an `Analysis` property for storing scan results. - Enhanced `ScanJobProcessor` to utilize the new `OsScanAnalyzerDispatcher`. - Improved logging and error handling in `ScanProgressReporter` for better traceability. - Updated project dependencies and added references to new analyzer plugins. - Revised task documentation to reflect current status and dependencies.
This commit is contained in:
@@ -2,141 +2,162 @@ using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.ObjectModel;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Options;
|
||||
|
||||
public sealed class ScannerWorkerOptions
|
||||
{
|
||||
public const string SectionName = "Scanner:Worker";
|
||||
|
||||
public int MaxConcurrentJobs { get; set; } = 2;
|
||||
|
||||
public QueueOptions Queue { get; } = new();
|
||||
|
||||
public PollingOptions Polling { get; } = new();
|
||||
|
||||
public AuthorityOptions Authority { get; } = new();
|
||||
|
||||
using System.IO;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Options;
|
||||
|
||||
public sealed class ScannerWorkerOptions
|
||||
{
|
||||
public const string SectionName = "Scanner:Worker";
|
||||
|
||||
public int MaxConcurrentJobs { get; set; } = 2;
|
||||
|
||||
public QueueOptions Queue { get; } = new();
|
||||
|
||||
public PollingOptions Polling { get; } = new();
|
||||
|
||||
public AuthorityOptions Authority { get; } = new();
|
||||
|
||||
public TelemetryOptions Telemetry { get; } = new();
|
||||
|
||||
public ShutdownOptions Shutdown { get; } = new();
|
||||
|
||||
public sealed class QueueOptions
|
||||
{
|
||||
public int MaxAttempts { get; set; } = 5;
|
||||
|
||||
public double HeartbeatSafetyFactor { get; set; } = 3.0;
|
||||
|
||||
public int MaxHeartbeatJitterMilliseconds { get; set; } = 750;
|
||||
|
||||
public IReadOnlyList<TimeSpan> HeartbeatRetryDelays => _heartbeatRetryDelays;
|
||||
|
||||
public TimeSpan MinHeartbeatInterval { get; set; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
public TimeSpan MaxHeartbeatInterval { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
public void SetHeartbeatRetryDelays(IEnumerable<TimeSpan> delays)
|
||||
{
|
||||
_heartbeatRetryDelays = NormalizeDelays(delays);
|
||||
}
|
||||
|
||||
internal IReadOnlyList<TimeSpan> NormalizedHeartbeatRetryDelays => _heartbeatRetryDelays;
|
||||
|
||||
private static IReadOnlyList<TimeSpan> NormalizeDelays(IEnumerable<TimeSpan> delays)
|
||||
{
|
||||
var buffer = new List<TimeSpan>();
|
||||
foreach (var delay in delays)
|
||||
{
|
||||
if (delay <= TimeSpan.Zero)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
buffer.Add(delay);
|
||||
}
|
||||
|
||||
buffer.Sort();
|
||||
return new ReadOnlyCollection<TimeSpan>(buffer);
|
||||
}
|
||||
|
||||
private IReadOnlyList<TimeSpan> _heartbeatRetryDelays = new ReadOnlyCollection<TimeSpan>(new TimeSpan[]
|
||||
{
|
||||
TimeSpan.FromSeconds(2),
|
||||
TimeSpan.FromSeconds(5),
|
||||
TimeSpan.FromSeconds(10),
|
||||
});
|
||||
}
|
||||
|
||||
public sealed class PollingOptions
|
||||
{
|
||||
public TimeSpan InitialDelay { get; set; } = TimeSpan.FromMilliseconds(200);
|
||||
|
||||
public TimeSpan MaxDelay { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
public double JitterRatio { get; set; } = 0.2;
|
||||
}
|
||||
|
||||
public sealed class AuthorityOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public string? Issuer { get; set; }
|
||||
|
||||
public string? ClientId { get; set; }
|
||||
|
||||
public string? ClientSecret { get; set; }
|
||||
|
||||
public bool RequireHttpsMetadata { get; set; } = true;
|
||||
|
||||
public string? MetadataAddress { get; set; }
|
||||
|
||||
public int BackchannelTimeoutSeconds { get; set; } = 20;
|
||||
|
||||
public int TokenClockSkewSeconds { get; set; } = 30;
|
||||
|
||||
public IList<string> Scopes { get; } = new List<string> { "scanner.scan" };
|
||||
|
||||
public ResilienceOptions Resilience { get; } = new();
|
||||
}
|
||||
|
||||
public sealed class ResilienceOptions
|
||||
{
|
||||
public bool? EnableRetries { get; set; }
|
||||
|
||||
public IList<TimeSpan> RetryDelays { get; } = new List<TimeSpan>
|
||||
{
|
||||
TimeSpan.FromMilliseconds(250),
|
||||
TimeSpan.FromMilliseconds(500),
|
||||
TimeSpan.FromSeconds(1),
|
||||
TimeSpan.FromSeconds(5),
|
||||
};
|
||||
|
||||
public bool? AllowOfflineCacheFallback { get; set; }
|
||||
|
||||
public TimeSpan? OfflineCacheTolerance { get; set; }
|
||||
}
|
||||
|
||||
public sealed class TelemetryOptions
|
||||
{
|
||||
public bool EnableLogging { get; set; } = true;
|
||||
|
||||
public bool EnableTelemetry { get; set; } = true;
|
||||
|
||||
public bool EnableTracing { get; set; }
|
||||
|
||||
public bool EnableMetrics { get; set; } = true;
|
||||
|
||||
public string ServiceName { get; set; } = "stellaops-scanner-worker";
|
||||
|
||||
public string? OtlpEndpoint { get; set; }
|
||||
|
||||
public bool ExportConsole { get; set; }
|
||||
|
||||
public IDictionary<string, string?> ResourceAttributes { get; } = new ConcurrentDictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public AnalyzerOptions Analyzers { get; } = new();
|
||||
|
||||
public sealed class QueueOptions
|
||||
{
|
||||
public int MaxAttempts { get; set; } = 5;
|
||||
|
||||
public double HeartbeatSafetyFactor { get; set; } = 3.0;
|
||||
|
||||
public int MaxHeartbeatJitterMilliseconds { get; set; } = 750;
|
||||
|
||||
public IReadOnlyList<TimeSpan> HeartbeatRetryDelays => _heartbeatRetryDelays;
|
||||
|
||||
public TimeSpan MinHeartbeatInterval { get; set; } = TimeSpan.FromSeconds(10);
|
||||
|
||||
public TimeSpan MaxHeartbeatInterval { get; set; } = TimeSpan.FromSeconds(30);
|
||||
|
||||
public void SetHeartbeatRetryDelays(IEnumerable<TimeSpan> delays)
|
||||
{
|
||||
_heartbeatRetryDelays = NormalizeDelays(delays);
|
||||
}
|
||||
|
||||
internal IReadOnlyList<TimeSpan> NormalizedHeartbeatRetryDelays => _heartbeatRetryDelays;
|
||||
|
||||
private static IReadOnlyList<TimeSpan> NormalizeDelays(IEnumerable<TimeSpan> delays)
|
||||
{
|
||||
var buffer = new List<TimeSpan>();
|
||||
foreach (var delay in delays)
|
||||
{
|
||||
if (delay <= TimeSpan.Zero)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
buffer.Add(delay);
|
||||
}
|
||||
|
||||
buffer.Sort();
|
||||
return new ReadOnlyCollection<TimeSpan>(buffer);
|
||||
}
|
||||
|
||||
private IReadOnlyList<TimeSpan> _heartbeatRetryDelays = new ReadOnlyCollection<TimeSpan>(new TimeSpan[]
|
||||
{
|
||||
TimeSpan.FromSeconds(2),
|
||||
TimeSpan.FromSeconds(5),
|
||||
TimeSpan.FromSeconds(10),
|
||||
});
|
||||
}
|
||||
|
||||
public sealed class PollingOptions
|
||||
{
|
||||
public TimeSpan InitialDelay { get; set; } = TimeSpan.FromMilliseconds(200);
|
||||
|
||||
public TimeSpan MaxDelay { get; set; } = TimeSpan.FromSeconds(5);
|
||||
|
||||
public double JitterRatio { get; set; } = 0.2;
|
||||
}
|
||||
|
||||
public sealed class AuthorityOptions
|
||||
{
|
||||
public bool Enabled { get; set; }
|
||||
|
||||
public string? Issuer { get; set; }
|
||||
|
||||
public string? ClientId { get; set; }
|
||||
|
||||
public string? ClientSecret { get; set; }
|
||||
|
||||
public bool RequireHttpsMetadata { get; set; } = true;
|
||||
|
||||
public string? MetadataAddress { get; set; }
|
||||
|
||||
public int BackchannelTimeoutSeconds { get; set; } = 20;
|
||||
|
||||
public int TokenClockSkewSeconds { get; set; } = 30;
|
||||
|
||||
public IList<string> Scopes { get; } = new List<string> { "scanner.scan" };
|
||||
|
||||
public ResilienceOptions Resilience { get; } = new();
|
||||
}
|
||||
|
||||
public sealed class ResilienceOptions
|
||||
{
|
||||
public bool? EnableRetries { get; set; }
|
||||
|
||||
public IList<TimeSpan> RetryDelays { get; } = new List<TimeSpan>
|
||||
{
|
||||
TimeSpan.FromMilliseconds(250),
|
||||
TimeSpan.FromMilliseconds(500),
|
||||
TimeSpan.FromSeconds(1),
|
||||
TimeSpan.FromSeconds(5),
|
||||
};
|
||||
|
||||
public bool? AllowOfflineCacheFallback { get; set; }
|
||||
|
||||
public TimeSpan? OfflineCacheTolerance { get; set; }
|
||||
}
|
||||
|
||||
public sealed class TelemetryOptions
|
||||
{
|
||||
public bool EnableLogging { get; set; } = true;
|
||||
|
||||
public bool EnableTelemetry { get; set; } = true;
|
||||
|
||||
public bool EnableTracing { get; set; }
|
||||
|
||||
public bool EnableMetrics { get; set; } = true;
|
||||
|
||||
public string ServiceName { get; set; } = "stellaops-scanner-worker";
|
||||
|
||||
public string? OtlpEndpoint { get; set; }
|
||||
|
||||
public bool ExportConsole { get; set; }
|
||||
|
||||
public IDictionary<string, string?> ResourceAttributes { get; } = new ConcurrentDictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public sealed class ShutdownOptions
|
||||
{
|
||||
public TimeSpan Timeout { get; set; } = TimeSpan.FromSeconds(30);
|
||||
}
|
||||
|
||||
public sealed class AnalyzerOptions
|
||||
{
|
||||
public AnalyzerOptions()
|
||||
{
|
||||
PluginDirectories = new List<string>
|
||||
{
|
||||
Path.Combine("plugins", "scanner", "analyzers", "os"),
|
||||
};
|
||||
}
|
||||
|
||||
public IList<string> PluginDirectories { get; }
|
||||
|
||||
public string RootFilesystemMetadataKey { get; set; } = ScanMetadataKeys.RootFilesystemPath;
|
||||
|
||||
public string WorkspaceMetadataKey { get; set; } = ScanMetadataKeys.WorkspacePath;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,91 +1,91 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Options;
|
||||
|
||||
public sealed class ScannerWorkerOptionsValidator : IValidateOptions<ScannerWorkerOptions>
|
||||
{
|
||||
public ValidateOptionsResult Validate(string? name, ScannerWorkerOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var failures = new List<string>();
|
||||
|
||||
if (options.MaxConcurrentJobs <= 0)
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using Microsoft.Extensions.Options;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Options;
|
||||
|
||||
public sealed class ScannerWorkerOptionsValidator : IValidateOptions<ScannerWorkerOptions>
|
||||
{
|
||||
public ValidateOptionsResult Validate(string? name, ScannerWorkerOptions options)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var failures = new List<string>();
|
||||
|
||||
if (options.MaxConcurrentJobs <= 0)
|
||||
{
|
||||
failures.Add("Scanner.Worker:MaxConcurrentJobs must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Queue.HeartbeatSafetyFactor < 3.0)
|
||||
{
|
||||
failures.Add("Scanner.Worker:MaxConcurrentJobs must be greater than zero.");
|
||||
failures.Add("Scanner.Worker:Queue:HeartbeatSafetyFactor must be at least 3.");
|
||||
}
|
||||
|
||||
if (options.Queue.HeartbeatSafetyFactor < 2.0)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Queue:HeartbeatSafetyFactor must be at least 2.");
|
||||
}
|
||||
|
||||
if (options.Queue.MaxAttempts <= 0)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Queue:MaxAttempts must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Queue.MinHeartbeatInterval <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Queue:MinHeartbeatInterval must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Queue.MaxHeartbeatInterval <= options.Queue.MinHeartbeatInterval)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Queue:MaxHeartbeatInterval must be greater than MinHeartbeatInterval.");
|
||||
}
|
||||
|
||||
if (options.Polling.InitialDelay <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Polling:InitialDelay must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Polling.MaxDelay < options.Polling.InitialDelay)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Polling:MaxDelay must be greater than or equal to InitialDelay.");
|
||||
}
|
||||
|
||||
if (options.Polling.JitterRatio is < 0 or > 1)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Polling:JitterRatio must be between 0 and 1.");
|
||||
}
|
||||
|
||||
if (options.Authority.Enabled)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.Authority.Issuer))
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority requires Issuer when Enabled is true.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.Authority.ClientId))
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority requires ClientId when Enabled is true.");
|
||||
}
|
||||
|
||||
if (options.Authority.BackchannelTimeoutSeconds <= 0)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority:BackchannelTimeoutSeconds must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Authority.TokenClockSkewSeconds < 0)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority:TokenClockSkewSeconds cannot be negative.");
|
||||
}
|
||||
|
||||
if (options.Authority.Resilience.RetryDelays.Any(delay => delay <= TimeSpan.Zero))
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority:Resilience:RetryDelays must be positive durations.");
|
||||
}
|
||||
}
|
||||
|
||||
if (options.Shutdown.Timeout < TimeSpan.FromSeconds(5))
|
||||
{
|
||||
failures.Add("Scanner.Worker:Shutdown:Timeout must be at least 5 seconds to allow lease completion.");
|
||||
}
|
||||
|
||||
|
||||
if (options.Queue.MaxAttempts <= 0)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Queue:MaxAttempts must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Queue.MinHeartbeatInterval <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Queue:MinHeartbeatInterval must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Queue.MaxHeartbeatInterval <= options.Queue.MinHeartbeatInterval)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Queue:MaxHeartbeatInterval must be greater than MinHeartbeatInterval.");
|
||||
}
|
||||
|
||||
if (options.Polling.InitialDelay <= TimeSpan.Zero)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Polling:InitialDelay must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Polling.MaxDelay < options.Polling.InitialDelay)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Polling:MaxDelay must be greater than or equal to InitialDelay.");
|
||||
}
|
||||
|
||||
if (options.Polling.JitterRatio is < 0 or > 1)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Polling:JitterRatio must be between 0 and 1.");
|
||||
}
|
||||
|
||||
if (options.Authority.Enabled)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(options.Authority.Issuer))
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority requires Issuer when Enabled is true.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.Authority.ClientId))
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority requires ClientId when Enabled is true.");
|
||||
}
|
||||
|
||||
if (options.Authority.BackchannelTimeoutSeconds <= 0)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority:BackchannelTimeoutSeconds must be greater than zero.");
|
||||
}
|
||||
|
||||
if (options.Authority.TokenClockSkewSeconds < 0)
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority:TokenClockSkewSeconds cannot be negative.");
|
||||
}
|
||||
|
||||
if (options.Authority.Resilience.RetryDelays.Any(delay => delay <= TimeSpan.Zero))
|
||||
{
|
||||
failures.Add("Scanner.Worker:Authority:Resilience:RetryDelays must be positive durations.");
|
||||
}
|
||||
}
|
||||
|
||||
if (options.Shutdown.Timeout < TimeSpan.FromSeconds(5))
|
||||
{
|
||||
failures.Add("Scanner.Worker:Shutdown:Timeout must be at least 5 seconds to allow lease completion.");
|
||||
}
|
||||
|
||||
if (options.Telemetry.EnableTelemetry)
|
||||
{
|
||||
if (!options.Telemetry.EnableMetrics && !options.Telemetry.EnableTracing)
|
||||
@@ -94,6 +94,11 @@ public sealed class ScannerWorkerOptionsValidator : IValidateOptions<ScannerWork
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(options.Analyzers.RootFilesystemMetadataKey))
|
||||
{
|
||||
failures.Add("Scanner.Worker:Analyzers:RootFilesystemMetadataKey must be provided.");
|
||||
}
|
||||
|
||||
return failures.Count == 0 ? ValidateOptionsResult.Success : ValidateOptionsResult.Fail(failures);
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user