save progress
This commit is contained in:
StellaOps Bot
@@ -23,6 +23,9 @@ public static class ScanStageNames
|
||||
// Sprint: SPRINT_20251226_014_BINIDX - Binary Vulnerability Lookup
|
||||
public const string BinaryLookup = "binary-lookup";
|
||||
|
||||
// Sprint: SPRINT_20251229_046_BE - Secrets Leak Detection
|
||||
public const string ScanSecrets = "scan-secrets";
|
||||
|
||||
public static readonly IReadOnlyList<string> Ordered = new[]
|
||||
{
|
||||
IngestReplay,
|
||||
@@ -30,6 +33,7 @@ public static class ScanStageNames
|
||||
PullLayers,
|
||||
BuildFilesystem,
|
||||
ExecuteAnalyzers,
|
||||
ScanSecrets,
|
||||
BinaryLookup,
|
||||
EpssEnrichment,
|
||||
ComposeArtifacts,
|
||||
|
||||
@@ -0,0 +1,235 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Analyzers.Secrets;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Stage executor that scans filesystem for hardcoded secrets and credentials.
|
||||
/// </summary>
|
||||
internal sealed class SecretsAnalyzerStageExecutor : IScanStageExecutor
|
||||
{
|
||||
private static readonly string[] RootFsMetadataKeys =
|
||||
{
|
||||
"filesystem.rootfs",
|
||||
"rootfs.path",
|
||||
"scanner.rootfs",
|
||||
};
|
||||
|
||||
private readonly ISecretsAnalyzer _secretsAnalyzer;
|
||||
private readonly ScannerWorkerMetrics _metrics;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
private readonly IOptions<ScannerWorkerOptions> _options;
|
||||
private readonly ILogger<SecretsAnalyzerStageExecutor> _logger;
|
||||
|
||||
public SecretsAnalyzerStageExecutor(
|
||||
ISecretsAnalyzer secretsAnalyzer,
|
||||
ScannerWorkerMetrics metrics,
|
||||
TimeProvider timeProvider,
|
||||
IOptions<ScannerWorkerOptions> options,
|
||||
ILogger<SecretsAnalyzerStageExecutor> logger)
|
||||
{
|
||||
_secretsAnalyzer = secretsAnalyzer ?? throw new ArgumentNullException(nameof(secretsAnalyzer));
|
||||
_metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string StageName => ScanStageNames.ScanSecrets;
|
||||
|
||||
public async ValueTask ExecuteAsync(ScanJobContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
|
||||
var secretsOptions = _options.Value.Secrets;
|
||||
if (!secretsOptions.Enabled)
|
||||
{
|
||||
_logger.LogDebug("Secrets scanning is disabled; skipping stage.");
|
||||
return;
|
||||
}
|
||||
|
||||
// Get file entries from analyzer stage
|
||||
if (!context.Analysis.TryGet<IReadOnlyList<ScanFileEntry>>(ScanAnalysisKeys.FileEntries, out var files) || files is null)
|
||||
{
|
||||
_logger.LogDebug("No file entries available; skipping secrets scan.");
|
||||
return;
|
||||
}
|
||||
|
||||
var rootfsPath = ResolveRootfsPath(context.Lease.Metadata);
|
||||
if (string.IsNullOrWhiteSpace(rootfsPath))
|
||||
{
|
||||
_logger.LogWarning("No rootfs path found in job metadata; skipping secrets scan for job {JobId}.", context.JobId);
|
||||
return;
|
||||
}
|
||||
|
||||
var startTime = _timeProvider.GetTimestamp();
|
||||
var allFindings = new List<SecretFinding>();
|
||||
|
||||
try
|
||||
{
|
||||
// Filter to text-like files only
|
||||
var textFiles = files
|
||||
.Where(f => ShouldScanFile(f))
|
||||
.ToList();
|
||||
|
||||
_logger.LogInformation(
|
||||
"Scanning {FileCount} files for secrets in job {JobId}.",
|
||||
textFiles.Count,
|
||||
context.JobId);
|
||||
|
||||
foreach (var file in textFiles)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
var filePath = Path.Combine(rootfsPath, file.Path.TrimStart('/'));
|
||||
if (!File.Exists(filePath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var content = await File.ReadAllBytesAsync(filePath, cancellationToken).ConfigureAwait(false);
|
||||
if (content.Length == 0 || content.Length > secretsOptions.MaxFileSizeBytes)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var findings = await _secretsAnalyzer.AnalyzeAsync(
|
||||
content,
|
||||
file.Path,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
if (findings.Count > 0)
|
||||
{
|
||||
allFindings.AddRange(findings);
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (!cancellationToken.IsCancellationRequested)
|
||||
{
|
||||
_logger.LogDebug(ex, "Error scanning file {Path} for secrets: {Message}", file.Path, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
var elapsed = _timeProvider.GetElapsedTime(startTime);
|
||||
|
||||
// Store findings in analysis store
|
||||
var report = new SecretsAnalysisReport
|
||||
{
|
||||
JobId = context.JobId,
|
||||
ScanId = context.ScanId,
|
||||
Findings = allFindings.ToImmutableArray(),
|
||||
FilesScanned = textFiles.Count,
|
||||
RulesetVersion = _secretsAnalyzer.RulesetVersion,
|
||||
AnalyzedAtUtc = _timeProvider.GetUtcNow(),
|
||||
ElapsedMilliseconds = elapsed.TotalMilliseconds
|
||||
};
|
||||
|
||||
context.Analysis.Set(ScanAnalysisKeys.SecretFindings, report);
|
||||
context.Analysis.Set(ScanAnalysisKeys.SecretRulesetVersion, _secretsAnalyzer.RulesetVersion);
|
||||
|
||||
_metrics.RecordSecretsAnalysisCompleted(
|
||||
context,
|
||||
allFindings.Count,
|
||||
textFiles.Count,
|
||||
elapsed,
|
||||
_timeProvider);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Secrets scan completed for job {JobId}: {FindingCount} findings in {FileCount} files ({ElapsedMs:F0}ms).",
|
||||
context.JobId,
|
||||
allFindings.Count,
|
||||
textFiles.Count,
|
||||
elapsed.TotalMilliseconds);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
_logger.LogDebug("Secrets scan cancelled for job {JobId}.", context.JobId);
|
||||
throw;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_metrics.RecordSecretsAnalysisFailed(context, _timeProvider);
|
||||
_logger.LogError(ex, "Secrets scan failed for job {JobId}: {Message}", context.JobId, ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ShouldScanFile(ScanFileEntry file)
|
||||
{
|
||||
if (file is null || file.SizeBytes == 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Skip binary files
|
||||
if (file.Kind is "elf" or "pe" or "mach-o" or "blob")
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Skip very large files
|
||||
if (file.SizeBytes > 10 * 1024 * 1024)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var ext = Path.GetExtension(file.Path).ToLowerInvariant();
|
||||
|
||||
// Include common text/config file extensions
|
||||
return ext is ".json" or ".yaml" or ".yml" or ".xml" or ".properties" or ".conf" or ".config"
|
||||
or ".env" or ".ini" or ".toml" or ".cfg"
|
||||
or ".js" or ".ts" or ".jsx" or ".tsx" or ".mjs" or ".cjs"
|
||||
or ".py" or ".rb" or ".php" or ".go" or ".java" or ".cs" or ".rs" or ".swift" or ".kt"
|
||||
or ".sh" or ".bash" or ".zsh" or ".ps1" or ".bat" or ".cmd"
|
||||
or ".sql" or ".graphql" or ".gql"
|
||||
or ".tf" or ".tfvars" or ".hcl"
|
||||
or ".dockerfile" or ".dockerignore"
|
||||
or ".gitignore" or ".npmrc" or ".yarnrc" or ".pypirc"
|
||||
or ".pem" or ".key" or ".crt" or ".cer"
|
||||
or ".md" or ".txt" or ".log"
|
||||
|| string.IsNullOrEmpty(ext);
|
||||
}
|
||||
|
||||
private static string? ResolveRootfsPath(IReadOnlyDictionary<string, string> metadata)
|
||||
{
|
||||
if (metadata is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
foreach (var key in RootFsMetadataKeys)
|
||||
{
|
||||
if (metadata.TryGetValue(key, out var value) && !string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return value.Trim();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Report of secrets analysis for a scan job.
|
||||
/// </summary>
|
||||
public sealed record SecretsAnalysisReport
|
||||
{
|
||||
public required string JobId { get; init; }
|
||||
public required string ScanId { get; init; }
|
||||
public required ImmutableArray<SecretFinding> Findings { get; init; }
|
||||
public required int FilesScanned { get; init; }
|
||||
public required string RulesetVersion { get; init; }
|
||||
public required DateTimeOffset AnalyzedAtUtc { get; init; }
|
||||
public required double ElapsedMilliseconds { get; init; }
|
||||
}
|
||||
Reference in New Issue
Block a user