save progress
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<EnableDefaultItems>false</EnableDefaultItems>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Include="**\*.cs" Exclude="obj\**;bin\**" />
|
||||
<EmbeddedResource Include="**\*.json" Exclude="obj\**;bin\**" />
|
||||
<None Include="**\*" Exclude="**\*.cs;**\*.json;bin\**;obj\**" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\StellaOps.Scanner.Analyzers.Lang\StellaOps.Scanner.Analyzers.Lang.csproj" />
|
||||
<ProjectReference Include="..\StellaOps.Scanner.Surface.Validation\StellaOps.Scanner.Surface.Validation.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
# Scanner Secrets Analyzer Guild Charter
|
||||
|
||||
## Mission
|
||||
|
||||
Detect accidentally committed secrets in container layers during scans using deterministic, DSSE-signed rule bundles. Ensure findings are reproducible, masked before output, and integrated with the Policy Engine for policy-driven decisions.
|
||||
|
||||
## Scope
|
||||
|
||||
- Secret detection plugin implementing `ILayerAnalyzer`
|
||||
- Regex and entropy-based detection strategies
|
||||
- Rule bundle loading, verification, and execution
|
||||
- Payload masking engine
|
||||
- Evidence emission (`secret.leak`) for policy integration
|
||||
- Integration with Scanner Worker pipeline
|
||||
|
||||
## Required Reading
|
||||
|
||||
- `docs/modules/scanner/operations/secret-leak-detection.md` - Target specification
|
||||
- `docs/modules/scanner/design/surface-secrets.md` - Credential delivery (different from leak detection)
|
||||
- `docs/modules/scanner/architecture.md` - Scanner module architecture
|
||||
- `docs/modules/policy/secret-leak-detection-readiness.md` - Policy integration requirements
|
||||
- `docs/implplan/SPRINT_20260104_002_SCANNER_secret_leak_detection_core.md` - Implementation sprint
|
||||
- `docs/implplan/SPRINT_20260104_003_SCANNER_secret_rule_bundles.md` - Bundle infrastructure sprint
|
||||
- CLAUDE.md Section 8 (Code Quality & Determinism Rules)
|
||||
|
||||
## Working Agreement
|
||||
|
||||
1. **Status synchronisation**: Update task state in sprint file and local `TASKS.md` when starting or completing work.
|
||||
|
||||
2. **Determinism**:
|
||||
- Sort rules by ID for deterministic execution order
|
||||
- Use `CultureInfo.InvariantCulture` for all parsing
|
||||
- Inject `TimeProvider` for timestamps
|
||||
- Same inputs must produce same outputs
|
||||
|
||||
3. **Security posture**:
|
||||
- NEVER log secret payloads
|
||||
- Apply masking BEFORE any output or persistence
|
||||
- Verify bundle signatures on load
|
||||
- Enforce feature flag for gradual rollout
|
||||
|
||||
4. **Testing requirements**:
|
||||
- Unit tests for all detectors, masking, and rule loading
|
||||
- Integration tests with Scanner Worker
|
||||
- Golden fixture tests for determinism verification
|
||||
- Security tests ensuring secrets are not leaked
|
||||
|
||||
5. **Offline readiness**:
|
||||
- Support local bundle verification without network
|
||||
- Document Attestor mirror configuration
|
||||
- Ensure bundles ship with Offline Kit
|
||||
|
||||
## Key Interfaces
|
||||
|
||||
```csharp
|
||||
// Detection interface
|
||||
public interface ISecretDetector
|
||||
{
|
||||
string DetectorId { get; }
|
||||
ValueTask<IReadOnlyList<SecretMatch>> DetectAsync(
|
||||
ReadOnlyMemory<byte> content,
|
||||
string filePath,
|
||||
SecretRule rule,
|
||||
CancellationToken ct = default);
|
||||
}
|
||||
|
||||
// Masking interface
|
||||
public interface IPayloadMasker
|
||||
{
|
||||
string Mask(ReadOnlySpan<byte> payload, string? hint = null);
|
||||
}
|
||||
|
||||
// Bundle verification
|
||||
public interface IBundleVerifier
|
||||
{
|
||||
Task<BundleVerificationResult> VerifyAsync(
|
||||
string bundleDirectory,
|
||||
VerificationOptions options,
|
||||
CancellationToken ct = default);
|
||||
}
|
||||
```
|
||||
|
||||
## Metrics
|
||||
|
||||
| Metric | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| `scanner.secret.finding_total` | Counter | Total findings by tenant, ruleId, severity |
|
||||
| `scanner.secret.scan_duration_seconds` | Histogram | Detection time per scan |
|
||||
| `scanner.secret.rules_loaded` | Gauge | Number of active rules |
|
||||
|
||||
## Directory Structure
|
||||
|
||||
```
|
||||
StellaOps.Scanner.Analyzers.Secrets/
|
||||
├── AGENTS.md # This file
|
||||
├── StellaOps.Scanner.Analyzers.Secrets.csproj
|
||||
├── Detectors/
|
||||
│ ├── ISecretDetector.cs
|
||||
│ ├── RegexDetector.cs
|
||||
│ ├── EntropyDetector.cs
|
||||
│ └── CompositeSecretDetector.cs
|
||||
├── Rules/
|
||||
│ ├── SecretRule.cs
|
||||
│ ├── SecretRuleset.cs
|
||||
│ └── RulesetLoader.cs
|
||||
├── Bundles/
|
||||
│ ├── BundleBuilder.cs
|
||||
│ ├── BundleVerifier.cs
|
||||
│ └── Schemas/
|
||||
├── Masking/
|
||||
│ ├── IPayloadMasker.cs
|
||||
│ └── PayloadMasker.cs
|
||||
├── Evidence/
|
||||
│ ├── SecretLeakEvidence.cs
|
||||
│ └── SecretFinding.cs
|
||||
├── SecretsAnalyzer.cs
|
||||
├── SecretsAnalyzerHost.cs
|
||||
├── SecretsAnalyzerOptions.cs
|
||||
└── ServiceCollectionExtensions.cs
|
||||
```
|
||||
|
||||
## Implementation Status
|
||||
|
||||
See sprint files for current implementation status:
|
||||
- SPRINT_20260104_002_SCANNER_secret_leak_detection_core.md
|
||||
- SPRINT_20260104_003_SCANNER_secret_rule_bundles.md
|
||||
- SPRINT_20260104_004_POLICY_secret_dsl_integration.md
|
||||
- SPRINT_20260104_005_AIRGAP_secret_offline_kit.md
|
||||
@@ -0,0 +1,3 @@
|
||||
using System.Runtime.CompilerServices;
|
||||
|
||||
[assembly: InternalsVisibleTo("StellaOps.Scanner.Analyzers.Secrets.Tests")]
|
||||
@@ -0,0 +1,345 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Globalization;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets.Bundles;
|
||||
|
||||
/// <summary>
|
||||
/// Builds secrets detection rule bundles from individual rule files.
|
||||
/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles
|
||||
/// </summary>
|
||||
public interface IBundleBuilder
|
||||
{
|
||||
/// <summary>
|
||||
/// Creates a bundle from individual rule files.
|
||||
/// </summary>
|
||||
Task<BundleArtifact> BuildAsync(
|
||||
BundleBuildOptions options,
|
||||
CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Options for bundle creation.
|
||||
/// </summary>
|
||||
public sealed record BundleBuildOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Directory where the bundle will be written.
|
||||
/// </summary>
|
||||
public required string OutputDirectory { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Bundle identifier (e.g., "secrets.ruleset").
|
||||
/// </summary>
|
||||
public required string BundleId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Bundle version (e.g., "2026.01").
|
||||
/// </summary>
|
||||
public required string Version { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Paths to individual rule JSON files to include.
|
||||
/// </summary>
|
||||
public required IReadOnlyList<string> RuleFiles { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional description for the bundle.
|
||||
/// </summary>
|
||||
public string Description { get; init; } = "StellaOps Secret Detection Rules";
|
||||
|
||||
/// <summary>
|
||||
/// Time provider for deterministic timestamps.
|
||||
/// </summary>
|
||||
public TimeProvider? TimeProvider { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to validate rules during build.
|
||||
/// </summary>
|
||||
public bool ValidateRules { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to fail on validation warnings.
|
||||
/// </summary>
|
||||
public bool FailOnWarnings { get; init; } = false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of bundle creation.
|
||||
/// </summary>
|
||||
public sealed record BundleArtifact
|
||||
{
|
||||
/// <summary>
|
||||
/// Path to the manifest file.
|
||||
/// </summary>
|
||||
public required string ManifestPath { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Path to the rules JSONL file.
|
||||
/// </summary>
|
||||
public required string RulesPath { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 hash of the rules file (lowercase hex).
|
||||
/// </summary>
|
||||
public required string RulesSha256 { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Total number of rules in the bundle.
|
||||
/// </summary>
|
||||
public required int TotalRules { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of enabled rules in the bundle.
|
||||
/// </summary>
|
||||
public required int EnabledRules { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The generated manifest.
|
||||
/// </summary>
|
||||
public required BundleManifest Manifest { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of bundle builder.
|
||||
/// </summary>
|
||||
public sealed class BundleBuilder : IBundleBuilder
|
||||
{
|
||||
private static readonly JsonSerializerOptions ManifestSerializerOptions = new()
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingNull
|
||||
};
|
||||
|
||||
private static readonly JsonSerializerOptions RuleSerializerOptions = new()
|
||||
{
|
||||
WriteIndented = false,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = System.Text.Json.Serialization.JsonIgnoreCondition.WhenWritingDefault
|
||||
};
|
||||
|
||||
private static readonly JsonSerializerOptions RuleReaderOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
AllowTrailingCommas = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip
|
||||
};
|
||||
|
||||
private readonly IRuleValidator _validator;
|
||||
private readonly ILogger<BundleBuilder> _logger;
|
||||
|
||||
public BundleBuilder(IRuleValidator validator, ILogger<BundleBuilder> logger)
|
||||
{
|
||||
_validator = validator ?? throw new ArgumentNullException(nameof(validator));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<BundleArtifact> BuildAsync(
|
||||
BundleBuildOptions options,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var timeProvider = options.TimeProvider ?? TimeProvider.System;
|
||||
var createdAt = timeProvider.GetUtcNow();
|
||||
|
||||
_logger.LogInformation(
|
||||
"Building bundle {BundleId} v{Version} from {FileCount} rule files",
|
||||
options.BundleId,
|
||||
options.Version,
|
||||
options.RuleFiles.Count);
|
||||
|
||||
// Load and validate rules
|
||||
var rules = new List<SecretRule>();
|
||||
var validationErrors = new List<string>();
|
||||
var validationWarnings = new List<string>();
|
||||
|
||||
foreach (var ruleFile in options.RuleFiles)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
if (!File.Exists(ruleFile))
|
||||
{
|
||||
validationErrors.Add($"Rule file not found: {ruleFile}");
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var json = await File.ReadAllTextAsync(ruleFile, ct).ConfigureAwait(false);
|
||||
var rule = JsonSerializer.Deserialize<SecretRule>(json, RuleReaderOptions);
|
||||
|
||||
if (rule is null)
|
||||
{
|
||||
validationErrors.Add($"Failed to deserialize rule from {ruleFile}");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (options.ValidateRules)
|
||||
{
|
||||
var validation = _validator.Validate(rule);
|
||||
if (!validation.IsValid)
|
||||
{
|
||||
foreach (var error in validation.Errors)
|
||||
{
|
||||
validationErrors.Add($"{ruleFile}: {error}");
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var warning in validation.Warnings)
|
||||
{
|
||||
validationWarnings.Add($"{ruleFile}: {warning}");
|
||||
}
|
||||
}
|
||||
|
||||
rules.Add(rule);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
validationErrors.Add($"JSON parse error in {ruleFile}: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
// Handle warnings
|
||||
if (validationWarnings.Count > 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Bundle build has {WarningCount} warnings: {Warnings}",
|
||||
validationWarnings.Count,
|
||||
string.Join("; ", validationWarnings.Take(5)));
|
||||
|
||||
if (options.FailOnWarnings)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Bundle build failed due to warnings: {string.Join("; ", validationWarnings)}");
|
||||
}
|
||||
}
|
||||
|
||||
// Handle errors
|
||||
if (validationErrors.Count > 0)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Bundle build failed with {validationErrors.Count} errors: {string.Join("; ", validationErrors)}");
|
||||
}
|
||||
|
||||
if (rules.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("No valid rules found to include in bundle.");
|
||||
}
|
||||
|
||||
// Sort rules by ID for deterministic output
|
||||
rules.Sort((a, b) => string.Compare(a.Id, b.Id, StringComparison.Ordinal));
|
||||
|
||||
// Ensure output directory exists
|
||||
Directory.CreateDirectory(options.OutputDirectory);
|
||||
|
||||
// Write rules JSONL file
|
||||
var rulesPath = Path.Combine(options.OutputDirectory, "secrets.ruleset.rules.jsonl");
|
||||
await WriteRulesJsonlAsync(rulesPath, rules, ct).ConfigureAwait(false);
|
||||
|
||||
// Compute SHA-256 of rules file
|
||||
var rulesSha256 = await ComputeFileSha256Async(rulesPath, ct).ConfigureAwait(false);
|
||||
|
||||
// Build manifest
|
||||
var manifest = new BundleManifest
|
||||
{
|
||||
SchemaVersion = "1.0",
|
||||
Id = options.BundleId,
|
||||
Version = options.Version,
|
||||
CreatedAt = createdAt,
|
||||
Description = options.Description,
|
||||
Rules = rules.Select(r => new BundleRuleSummary
|
||||
{
|
||||
Id = r.Id,
|
||||
Version = r.Version,
|
||||
Severity = r.Severity.ToString().ToLowerInvariant(),
|
||||
Enabled = r.Enabled
|
||||
}).ToImmutableArray(),
|
||||
Integrity = new BundleIntegrity
|
||||
{
|
||||
RulesFile = "secrets.ruleset.rules.jsonl",
|
||||
RulesSha256 = rulesSha256,
|
||||
TotalRules = rules.Count,
|
||||
EnabledRules = rules.Count(r => r.Enabled)
|
||||
}
|
||||
};
|
||||
|
||||
// Write manifest
|
||||
var manifestPath = Path.Combine(options.OutputDirectory, "secrets.ruleset.manifest.json");
|
||||
await WriteManifestAsync(manifestPath, manifest, ct).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Bundle {BundleId} v{Version} created with {RuleCount} rules ({EnabledCount} enabled)",
|
||||
options.BundleId,
|
||||
options.Version,
|
||||
rules.Count,
|
||||
rules.Count(r => r.Enabled));
|
||||
|
||||
return new BundleArtifact
|
||||
{
|
||||
ManifestPath = manifestPath,
|
||||
RulesPath = rulesPath,
|
||||
RulesSha256 = rulesSha256,
|
||||
TotalRules = rules.Count,
|
||||
EnabledRules = rules.Count(r => r.Enabled),
|
||||
Manifest = manifest
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task WriteRulesJsonlAsync(
|
||||
string path,
|
||||
IReadOnlyList<SecretRule> rules,
|
||||
CancellationToken ct)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
path,
|
||||
FileMode.Create,
|
||||
FileAccess.Write,
|
||||
FileShare.None,
|
||||
bufferSize: 4096,
|
||||
useAsync: true);
|
||||
|
||||
await using var writer = new StreamWriter(stream, Encoding.UTF8, leaveOpen: true);
|
||||
|
||||
foreach (var rule in rules)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
var json = JsonSerializer.Serialize(rule, RuleSerializerOptions);
|
||||
await writer.WriteLineAsync(json).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WriteManifestAsync(
|
||||
string path,
|
||||
BundleManifest manifest,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var json = JsonSerializer.Serialize(manifest, ManifestSerializerOptions);
|
||||
await File.WriteAllTextAsync(path, json, Encoding.UTF8, ct).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
private static async Task<string> ComputeFileSha256Async(string path, CancellationToken ct)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 4096,
|
||||
useAsync: true);
|
||||
|
||||
var hash = await SHA256.HashDataAsync(stream, ct).ConfigureAwait(false);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets.Bundles;
|
||||
|
||||
/// <summary>
|
||||
/// Represents the manifest of a secrets detection rule bundle.
|
||||
/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles
|
||||
/// </summary>
|
||||
public sealed record BundleManifest
|
||||
{
|
||||
/// <summary>
|
||||
/// Schema version identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("schemaVersion")]
|
||||
public string SchemaVersion { get; init; } = "1.0";
|
||||
|
||||
/// <summary>
|
||||
/// Unique identifier for the bundle (e.g., "secrets.ruleset").
|
||||
/// </summary>
|
||||
[JsonPropertyName("id")]
|
||||
public required string Id { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Bundle version using CalVer (e.g., "2026.01").
|
||||
/// </summary>
|
||||
[JsonPropertyName("version")]
|
||||
public required string Version { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// UTC timestamp when the bundle was created.
|
||||
/// </summary>
|
||||
[JsonPropertyName("createdAt")]
|
||||
public required DateTimeOffset CreatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Human-readable description of the bundle.
|
||||
/// </summary>
|
||||
[JsonPropertyName("description")]
|
||||
public string Description { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Summary of rules included in the bundle.
|
||||
/// </summary>
|
||||
[JsonPropertyName("rules")]
|
||||
public ImmutableArray<BundleRuleSummary> Rules { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Integrity information for the bundle.
|
||||
/// </summary>
|
||||
[JsonPropertyName("integrity")]
|
||||
public required BundleIntegrity Integrity { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Signature information for the bundle.
|
||||
/// </summary>
|
||||
[JsonPropertyName("signatures")]
|
||||
public BundleSignatures? Signatures { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Summary of a rule included in the bundle manifest.
|
||||
/// </summary>
|
||||
public sealed record BundleRuleSummary
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique rule identifier.
|
||||
/// </summary>
|
||||
[JsonPropertyName("id")]
|
||||
public required string Id { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Rule version (SemVer).
|
||||
/// </summary>
|
||||
[JsonPropertyName("version")]
|
||||
public required string Version { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Rule severity level.
|
||||
/// </summary>
|
||||
[JsonPropertyName("severity")]
|
||||
public required string Severity { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the rule is enabled by default.
|
||||
/// </summary>
|
||||
[JsonPropertyName("enabled")]
|
||||
public bool Enabled { get; init; } = true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Integrity information for bundle verification.
|
||||
/// </summary>
|
||||
public sealed record BundleIntegrity
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the rules file within the bundle.
|
||||
/// </summary>
|
||||
[JsonPropertyName("rulesFile")]
|
||||
public string RulesFile { get; init; } = "secrets.ruleset.rules.jsonl";
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 hash of the rules file (lowercase hex).
|
||||
/// </summary>
|
||||
[JsonPropertyName("rulesSha256")]
|
||||
public required string RulesSha256 { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Total number of rules in the bundle.
|
||||
/// </summary>
|
||||
[JsonPropertyName("totalRules")]
|
||||
public required int TotalRules { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Number of enabled rules in the bundle.
|
||||
/// </summary>
|
||||
[JsonPropertyName("enabledRules")]
|
||||
public required int EnabledRules { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Signature references for the bundle.
|
||||
/// </summary>
|
||||
public sealed record BundleSignatures
|
||||
{
|
||||
/// <summary>
|
||||
/// Path to the DSSE envelope file within the bundle.
|
||||
/// </summary>
|
||||
[JsonPropertyName("dsseEnvelope")]
|
||||
public string DsseEnvelope { get; init; } = "secrets.ruleset.dsse.json";
|
||||
|
||||
/// <summary>
|
||||
/// Key ID used for signing (for informational purposes).
|
||||
/// </summary>
|
||||
[JsonPropertyName("keyId")]
|
||||
public string? KeyId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// UTC timestamp when the bundle was signed.
|
||||
/// </summary>
|
||||
[JsonPropertyName("signedAt")]
|
||||
public DateTimeOffset? SignedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Rekor transparency log entry ID (if applicable).
|
||||
/// </summary>
|
||||
[JsonPropertyName("rekorLogId")]
|
||||
public string? RekorLogId { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,349 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets.Bundles;
|
||||
|
||||
/// <summary>
|
||||
/// Signs secrets detection rule bundles using DSSE envelopes.
|
||||
/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles
|
||||
/// </summary>
|
||||
public interface IBundleSigner
|
||||
{
|
||||
/// <summary>
|
||||
/// Signs a bundle artifact producing a DSSE envelope.
|
||||
/// </summary>
|
||||
Task<BundleSigningResult> SignAsync(
|
||||
BundleArtifact artifact,
|
||||
BundleSigningOptions options,
|
||||
CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Options for bundle signing.
|
||||
/// </summary>
|
||||
public sealed record BundleSigningOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Key identifier for the signature.
|
||||
/// </summary>
|
||||
public required string KeyId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Signing algorithm (e.g., "HMAC-SHA256", "ES256").
|
||||
/// </summary>
|
||||
public string Algorithm { get; init; } = "HMAC-SHA256";
|
||||
|
||||
/// <summary>
|
||||
/// Shared secret for HMAC signing (base64 or hex encoded).
|
||||
/// Required for HMAC-SHA256 algorithm.
|
||||
/// </summary>
|
||||
public string? SharedSecret { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Path to file containing the shared secret.
|
||||
/// </summary>
|
||||
public string? SharedSecretFile { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Payload type for the DSSE envelope.
|
||||
/// </summary>
|
||||
public string PayloadType { get; init; } = "application/vnd.stellaops.secrets-ruleset+json";
|
||||
|
||||
/// <summary>
|
||||
/// Time provider for deterministic timestamps.
|
||||
/// </summary>
|
||||
public TimeProvider? TimeProvider { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of bundle signing.
|
||||
/// </summary>
|
||||
public sealed record BundleSigningResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Path to the generated DSSE envelope file.
|
||||
/// </summary>
|
||||
public required string EnvelopePath { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The generated DSSE envelope.
|
||||
/// </summary>
|
||||
public required DsseEnvelope Envelope { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Updated manifest with signature information.
|
||||
/// </summary>
|
||||
public required BundleManifest UpdatedManifest { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// DSSE envelope structure for bundle signatures.
|
||||
/// </summary>
|
||||
public sealed record DsseEnvelope
|
||||
{
|
||||
/// <summary>
|
||||
/// Base64url-encoded payload.
|
||||
/// </summary>
|
||||
public required string Payload { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Payload type URI.
|
||||
/// </summary>
|
||||
public required string PayloadType { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Signatures over the PAE.
|
||||
/// </summary>
|
||||
public required ImmutableArray<DsseSignature> Signatures { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A signature within a DSSE envelope.
|
||||
/// </summary>
|
||||
public sealed record DsseSignature
|
||||
{
|
||||
/// <summary>
|
||||
/// Base64url-encoded signature bytes.
|
||||
/// </summary>
|
||||
public required string Sig { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Key identifier.
|
||||
/// </summary>
|
||||
public string? KeyId { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of bundle signing using HMAC-SHA256.
|
||||
/// </summary>
|
||||
public sealed class BundleSigner : IBundleSigner
|
||||
{
|
||||
private const string DssePrefix = "DSSEv1";
|
||||
|
||||
private static readonly JsonSerializerOptions EnvelopeSerializerOptions = new()
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
||||
};
|
||||
|
||||
private readonly ILogger<BundleSigner> _logger;
|
||||
|
||||
public BundleSigner(ILogger<BundleSigner> logger)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<BundleSigningResult> SignAsync(
|
||||
BundleArtifact artifact,
|
||||
BundleSigningOptions options,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(artifact);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var timeProvider = options.TimeProvider ?? TimeProvider.System;
|
||||
var signedAt = timeProvider.GetUtcNow();
|
||||
|
||||
_logger.LogInformation(
|
||||
"Signing bundle {BundleId} v{Version} with key {KeyId}",
|
||||
artifact.Manifest.Id,
|
||||
artifact.Manifest.Version,
|
||||
options.KeyId);
|
||||
|
||||
// Read manifest as payload
|
||||
var manifestJson = await File.ReadAllBytesAsync(artifact.ManifestPath, ct).ConfigureAwait(false);
|
||||
|
||||
// Encode payload as base64url
|
||||
var payloadBase64 = ToBase64Url(manifestJson);
|
||||
|
||||
// Build PAE (Pre-Authentication Encoding)
|
||||
var pae = BuildPae(options.PayloadType, manifestJson);
|
||||
|
||||
// Sign the PAE
|
||||
var signature = await SignPaeAsync(pae, options, ct).ConfigureAwait(false);
|
||||
var signatureBase64 = ToBase64Url(signature);
|
||||
|
||||
// Build DSSE envelope
|
||||
var envelope = new DsseEnvelope
|
||||
{
|
||||
Payload = payloadBase64,
|
||||
PayloadType = options.PayloadType,
|
||||
Signatures =
|
||||
[
|
||||
new DsseSignature
|
||||
{
|
||||
Sig = signatureBase64,
|
||||
KeyId = options.KeyId
|
||||
}
|
||||
]
|
||||
};
|
||||
|
||||
// Write envelope to file
|
||||
var bundleDir = Path.GetDirectoryName(artifact.ManifestPath)!;
|
||||
var envelopePath = Path.Combine(bundleDir, "secrets.ruleset.dsse.json");
|
||||
var envelopeJson = JsonSerializer.Serialize(envelope, EnvelopeSerializerOptions);
|
||||
await File.WriteAllTextAsync(envelopePath, envelopeJson, Encoding.UTF8, ct).ConfigureAwait(false);
|
||||
|
||||
// Update manifest with signature info
|
||||
var updatedManifest = artifact.Manifest with
|
||||
{
|
||||
Signatures = new BundleSignatures
|
||||
{
|
||||
DsseEnvelope = "secrets.ruleset.dsse.json",
|
||||
KeyId = options.KeyId,
|
||||
SignedAt = signedAt
|
||||
}
|
||||
};
|
||||
|
||||
// Rewrite manifest with signature info
|
||||
var updatedManifestJson = JsonSerializer.Serialize(updatedManifest, EnvelopeSerializerOptions);
|
||||
await File.WriteAllTextAsync(artifact.ManifestPath, updatedManifestJson, Encoding.UTF8, ct).ConfigureAwait(false);
|
||||
|
||||
_logger.LogInformation(
|
||||
"Bundle signed successfully. Envelope: {EnvelopePath}",
|
||||
envelopePath);
|
||||
|
||||
return new BundleSigningResult
|
||||
{
|
||||
EnvelopePath = envelopePath,
|
||||
Envelope = envelope,
|
||||
UpdatedManifest = updatedManifest
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<byte[]> SignPaeAsync(
|
||||
byte[] pae,
|
||||
BundleSigningOptions options,
|
||||
CancellationToken ct)
|
||||
{
|
||||
if (!options.Algorithm.Equals("HMAC-SHA256", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new NotSupportedException($"Algorithm '{options.Algorithm}' is not supported. Use HMAC-SHA256.");
|
||||
}
|
||||
|
||||
var secret = await LoadSecretAsync(options, ct).ConfigureAwait(false);
|
||||
if (secret is null || secret.Length == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Shared secret is required for HMAC-SHA256 signing.");
|
||||
}
|
||||
|
||||
using var hmac = new HMACSHA256(secret);
|
||||
return hmac.ComputeHash(pae);
|
||||
}
|
||||
|
||||
private static async Task<byte[]?> LoadSecretAsync(BundleSigningOptions options, CancellationToken ct)
|
||||
{
|
||||
// Try file first
|
||||
if (!string.IsNullOrWhiteSpace(options.SharedSecretFile) && File.Exists(options.SharedSecretFile))
|
||||
{
|
||||
var content = (await File.ReadAllTextAsync(options.SharedSecretFile, ct).ConfigureAwait(false)).Trim();
|
||||
return DecodeSecret(content);
|
||||
}
|
||||
|
||||
// Then inline secret
|
||||
if (!string.IsNullOrWhiteSpace(options.SharedSecret))
|
||||
{
|
||||
return DecodeSecret(options.SharedSecret);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static byte[] DecodeSecret(string value)
|
||||
{
|
||||
// Try base64 first
|
||||
try
|
||||
{
|
||||
return Convert.FromBase64String(value);
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
// Not base64
|
||||
}
|
||||
|
||||
// Try hex
|
||||
if (value.Length % 2 == 0 && IsHexString(value))
|
||||
{
|
||||
return Convert.FromHexString(value);
|
||||
}
|
||||
|
||||
// Treat as raw UTF-8
|
||||
return Encoding.UTF8.GetBytes(value);
|
||||
}
|
||||
|
||||
private static bool IsHexString(string value)
|
||||
{
|
||||
foreach (var c in value)
|
||||
{
|
||||
if (!char.IsAsciiHexDigit(c))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds DSSE v1 Pre-Authentication Encoding.
|
||||
/// Format: "DSSEv1" SP LEN(type) SP type SP LEN(payload) SP payload
|
||||
/// </summary>
|
||||
private static byte[] BuildPae(string payloadType, byte[] payload)
|
||||
{
|
||||
var typeBytes = Encoding.UTF8.GetBytes(payloadType);
|
||||
var typeLenStr = typeBytes.Length.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
var payloadLenStr = payload.Length.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
|
||||
// Calculate total size
|
||||
var prefixBytes = Encoding.UTF8.GetBytes(DssePrefix);
|
||||
var typeLenBytes = Encoding.UTF8.GetBytes(typeLenStr);
|
||||
var payloadLenBytes = Encoding.UTF8.GetBytes(payloadLenStr);
|
||||
|
||||
var totalSize = prefixBytes.Length + 1 // prefix + SP
|
||||
+ typeLenBytes.Length + 1 // type len + SP
|
||||
+ typeBytes.Length + 1 // type + SP
|
||||
+ payloadLenBytes.Length + 1 // payload len + SP
|
||||
+ payload.Length;
|
||||
|
||||
var pae = new byte[totalSize];
|
||||
var offset = 0;
|
||||
|
||||
// DSSEv1
|
||||
Buffer.BlockCopy(prefixBytes, 0, pae, offset, prefixBytes.Length);
|
||||
offset += prefixBytes.Length;
|
||||
pae[offset++] = 0x20; // SP
|
||||
|
||||
// type length
|
||||
Buffer.BlockCopy(typeLenBytes, 0, pae, offset, typeLenBytes.Length);
|
||||
offset += typeLenBytes.Length;
|
||||
pae[offset++] = 0x20; // SP
|
||||
|
||||
// type
|
||||
Buffer.BlockCopy(typeBytes, 0, pae, offset, typeBytes.Length);
|
||||
offset += typeBytes.Length;
|
||||
pae[offset++] = 0x20; // SP
|
||||
|
||||
// payload length
|
||||
Buffer.BlockCopy(payloadLenBytes, 0, pae, offset, payloadLenBytes.Length);
|
||||
offset += payloadLenBytes.Length;
|
||||
pae[offset++] = 0x20; // SP
|
||||
|
||||
// payload
|
||||
Buffer.BlockCopy(payload, 0, pae, offset, payload.Length);
|
||||
|
||||
return pae;
|
||||
}
|
||||
|
||||
private static string ToBase64Url(byte[] data)
|
||||
{
|
||||
return Convert.ToBase64String(data)
|
||||
.TrimEnd('=')
|
||||
.Replace('+', '-')
|
||||
.Replace('/', '_');
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,527 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets.Bundles;
|
||||
|
||||
/// <summary>
|
||||
/// Verifies secrets detection rule bundle signatures and integrity.
|
||||
/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles
|
||||
/// </summary>
|
||||
public interface IBundleVerifier
|
||||
{
|
||||
/// <summary>
|
||||
/// Verifies a bundle's DSSE signature and integrity.
|
||||
/// </summary>
|
||||
Task<BundleVerificationResult> VerifyAsync(
|
||||
string bundleDirectory,
|
||||
BundleVerificationOptions options,
|
||||
CancellationToken ct = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Options for bundle verification.
|
||||
/// </summary>
|
||||
public sealed record BundleVerificationOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// URL of the attestor service for online verification.
|
||||
/// </summary>
|
||||
public string? AttestorUrl { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to require Rekor transparency log proof.
|
||||
/// </summary>
|
||||
public bool RequireRekorProof { get; init; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// List of trusted key IDs. If empty, any key is accepted.
|
||||
/// </summary>
|
||||
public IReadOnlyList<string>? TrustedKeyIds { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Shared secret for HMAC verification (base64 or hex encoded).
|
||||
/// </summary>
|
||||
public string? SharedSecret { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Path to file containing the shared secret.
|
||||
/// </summary>
|
||||
public string? SharedSecretFile { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether to verify file integrity (SHA-256).
|
||||
/// </summary>
|
||||
public bool VerifyIntegrity { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to skip signature verification (integrity only).
|
||||
/// </summary>
|
||||
public bool SkipSignatureVerification { get; init; } = false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of bundle verification.
|
||||
/// </summary>
|
||||
public sealed record BundleVerificationResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the bundle is valid.
|
||||
/// </summary>
|
||||
public required bool IsValid { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Bundle version that was verified.
|
||||
/// </summary>
|
||||
public string? BundleVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Bundle ID that was verified.
|
||||
/// </summary>
|
||||
public string? BundleId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// UTC timestamp when the bundle was signed.
|
||||
/// </summary>
|
||||
public DateTimeOffset? SignedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Key ID that signed the bundle.
|
||||
/// </summary>
|
||||
public string? SignerKeyId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Rekor transparency log entry ID (if available).
|
||||
/// </summary>
|
||||
public string? RekorLogId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Total number of rules in the bundle.
|
||||
/// </summary>
|
||||
public int? RuleCount { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Validation errors encountered.
|
||||
/// </summary>
|
||||
public ImmutableArray<string> ValidationErrors { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Validation warnings (non-fatal).
|
||||
/// </summary>
|
||||
public ImmutableArray<string> ValidationWarnings { get; init; } = [];
|
||||
|
||||
public static BundleVerificationResult Success(BundleManifest manifest, string? keyId) => new()
|
||||
{
|
||||
IsValid = true,
|
||||
BundleId = manifest.Id,
|
||||
BundleVersion = manifest.Version,
|
||||
SignedAt = manifest.Signatures?.SignedAt,
|
||||
SignerKeyId = keyId ?? manifest.Signatures?.KeyId,
|
||||
RekorLogId = manifest.Signatures?.RekorLogId,
|
||||
RuleCount = manifest.Integrity.TotalRules
|
||||
};
|
||||
|
||||
public static BundleVerificationResult Failure(params string[] errors) => new()
|
||||
{
|
||||
IsValid = false,
|
||||
ValidationErrors = [.. errors]
|
||||
};
|
||||
|
||||
public static BundleVerificationResult Failure(IEnumerable<string> errors) => new()
|
||||
{
|
||||
IsValid = false,
|
||||
ValidationErrors = [.. errors]
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of bundle verification.
|
||||
/// </summary>
|
||||
public sealed class BundleVerifier : IBundleVerifier
|
||||
{
|
||||
private const string DssePrefix = "DSSEv1";
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
AllowTrailingCommas = true
|
||||
};
|
||||
|
||||
private readonly ILogger<BundleVerifier> _logger;
|
||||
|
||||
public BundleVerifier(ILogger<BundleVerifier> logger)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public async Task<BundleVerificationResult> VerifyAsync(
|
||||
string bundleDirectory,
|
||||
BundleVerificationOptions options,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrWhiteSpace(bundleDirectory);
|
||||
ArgumentNullException.ThrowIfNull(options);
|
||||
|
||||
var errors = new List<string>();
|
||||
var warnings = new List<string>();
|
||||
|
||||
_logger.LogDebug("Verifying bundle at {BundleDir}", bundleDirectory);
|
||||
|
||||
// Check directory exists
|
||||
if (!Directory.Exists(bundleDirectory))
|
||||
{
|
||||
return BundleVerificationResult.Failure($"Bundle directory not found: {bundleDirectory}");
|
||||
}
|
||||
|
||||
// Load manifest
|
||||
var manifestPath = Path.Combine(bundleDirectory, "secrets.ruleset.manifest.json");
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
return BundleVerificationResult.Failure($"Manifest not found: {manifestPath}");
|
||||
}
|
||||
|
||||
BundleManifest manifest;
|
||||
try
|
||||
{
|
||||
var manifestJson = await File.ReadAllTextAsync(manifestPath, ct).ConfigureAwait(false);
|
||||
manifest = JsonSerializer.Deserialize<BundleManifest>(manifestJson, JsonOptions)!;
|
||||
if (manifest is null)
|
||||
{
|
||||
return BundleVerificationResult.Failure("Failed to parse manifest.");
|
||||
}
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
return BundleVerificationResult.Failure($"Invalid manifest JSON: {ex.Message}");
|
||||
}
|
||||
|
||||
_logger.LogDebug(
|
||||
"Loaded manifest: {BundleId} v{Version} with {RuleCount} rules",
|
||||
manifest.Id,
|
||||
manifest.Version,
|
||||
manifest.Integrity.TotalRules);
|
||||
|
||||
// Verify file integrity
|
||||
if (options.VerifyIntegrity)
|
||||
{
|
||||
var rulesPath = Path.Combine(bundleDirectory, manifest.Integrity.RulesFile);
|
||||
if (!File.Exists(rulesPath))
|
||||
{
|
||||
errors.Add($"Rules file not found: {rulesPath}");
|
||||
}
|
||||
else
|
||||
{
|
||||
var actualHash = await ComputeFileSha256Async(rulesPath, ct).ConfigureAwait(false);
|
||||
if (!string.Equals(actualHash, manifest.Integrity.RulesSha256, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
errors.Add($"Rules file integrity check failed. Expected: {manifest.Integrity.RulesSha256}, Actual: {actualHash}");
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("Rules file integrity verified.");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify signature
|
||||
if (!options.SkipSignatureVerification)
|
||||
{
|
||||
if (manifest.Signatures is null)
|
||||
{
|
||||
errors.Add("Bundle is not signed.");
|
||||
}
|
||||
else
|
||||
{
|
||||
var envelopePath = Path.Combine(bundleDirectory, manifest.Signatures.DsseEnvelope);
|
||||
if (!File.Exists(envelopePath))
|
||||
{
|
||||
errors.Add($"DSSE envelope not found: {envelopePath}");
|
||||
}
|
||||
else
|
||||
{
|
||||
var signatureResult = await VerifySignatureAsync(
|
||||
manifestPath,
|
||||
envelopePath,
|
||||
options,
|
||||
ct).ConfigureAwait(false);
|
||||
|
||||
if (!signatureResult.IsValid)
|
||||
{
|
||||
errors.AddRange(signatureResult.Errors);
|
||||
}
|
||||
else
|
||||
{
|
||||
// Check trusted key IDs
|
||||
if (options.TrustedKeyIds is { Count: > 0 } trustedKeys)
|
||||
{
|
||||
if (signatureResult.KeyId is null || !trustedKeys.Contains(signatureResult.KeyId))
|
||||
{
|
||||
errors.Add($"Signature key '{signatureResult.KeyId}' is not in the trusted keys list.");
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogDebug("Signature verified with key: {KeyId}", signatureResult.KeyId);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Check Rekor requirement
|
||||
if (options.RequireRekorProof)
|
||||
{
|
||||
if (manifest.Signatures?.RekorLogId is null)
|
||||
{
|
||||
errors.Add("Rekor transparency log proof is required but not present.");
|
||||
}
|
||||
else
|
||||
{
|
||||
// TODO: Implement Rekor verification via Attestor client
|
||||
warnings.Add("Rekor verification not yet implemented; proof present but not verified.");
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.Count > 0)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Bundle verification failed for {BundleId} v{Version}: {Errors}",
|
||||
manifest.Id,
|
||||
manifest.Version,
|
||||
string.Join("; ", errors));
|
||||
|
||||
return new BundleVerificationResult
|
||||
{
|
||||
IsValid = false,
|
||||
BundleId = manifest.Id,
|
||||
BundleVersion = manifest.Version,
|
||||
ValidationErrors = [.. errors],
|
||||
ValidationWarnings = [.. warnings]
|
||||
};
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Bundle verified: {BundleId} v{Version} ({RuleCount} rules)",
|
||||
manifest.Id,
|
||||
manifest.Version,
|
||||
manifest.Integrity.TotalRules);
|
||||
|
||||
return new BundleVerificationResult
|
||||
{
|
||||
IsValid = true,
|
||||
BundleId = manifest.Id,
|
||||
BundleVersion = manifest.Version,
|
||||
SignedAt = manifest.Signatures?.SignedAt,
|
||||
SignerKeyId = manifest.Signatures?.KeyId,
|
||||
RekorLogId = manifest.Signatures?.RekorLogId,
|
||||
RuleCount = manifest.Integrity.TotalRules,
|
||||
ValidationWarnings = [.. warnings]
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<SignatureVerificationResult> VerifySignatureAsync(
|
||||
string manifestPath,
|
||||
string envelopePath,
|
||||
BundleVerificationOptions options,
|
||||
CancellationToken ct)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Load envelope
|
||||
var envelopeJson = await File.ReadAllTextAsync(envelopePath, ct).ConfigureAwait(false);
|
||||
var envelope = JsonSerializer.Deserialize<DsseEnvelope>(envelopeJson, JsonOptions);
|
||||
|
||||
if (envelope is null || envelope.Signatures.IsDefaultOrEmpty)
|
||||
{
|
||||
return SignatureVerificationResult.Failure("Invalid or empty DSSE envelope.");
|
||||
}
|
||||
|
||||
// Decode payload - this is the original manifest (before signature was added)
|
||||
var payloadBytes = FromBase64Url(envelope.Payload);
|
||||
var payloadManifest = JsonSerializer.Deserialize<BundleManifest>(payloadBytes, JsonOptions);
|
||||
|
||||
if (payloadManifest is null)
|
||||
{
|
||||
return SignatureVerificationResult.Failure("Failed to parse envelope payload as manifest.");
|
||||
}
|
||||
|
||||
// Load current manifest and verify it matches the signed version (ignoring the Signatures field
|
||||
// which was added after signing)
|
||||
var manifestJson = await File.ReadAllTextAsync(manifestPath, ct).ConfigureAwait(false);
|
||||
var currentManifest = JsonSerializer.Deserialize<BundleManifest>(manifestJson, JsonOptions);
|
||||
|
||||
if (currentManifest is null)
|
||||
{
|
||||
return SignatureVerificationResult.Failure("Failed to parse current manifest.");
|
||||
}
|
||||
|
||||
// Compare all fields except Signatures (which is added after signing)
|
||||
if (!ManifestsMatchIgnoringSignatures(payloadManifest, currentManifest))
|
||||
{
|
||||
return SignatureVerificationResult.Failure("Envelope payload does not match manifest content.");
|
||||
}
|
||||
|
||||
// Build PAE
|
||||
var pae = BuildPae(envelope.PayloadType, payloadBytes);
|
||||
|
||||
// Verify each signature (at least one must be valid)
|
||||
var secret = await LoadSecretAsync(options, ct).ConfigureAwait(false);
|
||||
if (secret is null || secret.Length == 0)
|
||||
{
|
||||
return SignatureVerificationResult.Failure("Shared secret is required for signature verification.");
|
||||
}
|
||||
|
||||
foreach (var sig in envelope.Signatures)
|
||||
{
|
||||
var signatureBytes = FromBase64Url(sig.Sig);
|
||||
|
||||
using var hmac = new HMACSHA256(secret);
|
||||
var expectedSignature = hmac.ComputeHash(pae);
|
||||
|
||||
if (CryptographicOperations.FixedTimeEquals(expectedSignature, signatureBytes))
|
||||
{
|
||||
return SignatureVerificationResult.Success(sig.KeyId);
|
||||
}
|
||||
}
|
||||
|
||||
return SignatureVerificationResult.Failure("Signature verification failed.");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return SignatureVerificationResult.Failure($"Signature verification error: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ManifestsMatchIgnoringSignatures(BundleManifest a, BundleManifest b)
|
||||
{
|
||||
// Compare all fields except Signatures
|
||||
return a.SchemaVersion == b.SchemaVersion
|
||||
&& a.Id == b.Id
|
||||
&& a.Version == b.Version
|
||||
&& a.CreatedAt == b.CreatedAt
|
||||
&& a.Description == b.Description
|
||||
&& a.Integrity.RulesFile == b.Integrity.RulesFile
|
||||
&& a.Integrity.RulesSha256 == b.Integrity.RulesSha256
|
||||
&& a.Integrity.TotalRules == b.Integrity.TotalRules
|
||||
&& a.Integrity.EnabledRules == b.Integrity.EnabledRules;
|
||||
}
|
||||
|
||||
private static async Task<byte[]?> LoadSecretAsync(BundleVerificationOptions options, CancellationToken ct)
|
||||
{
|
||||
// Try file first
|
||||
if (!string.IsNullOrWhiteSpace(options.SharedSecretFile) && File.Exists(options.SharedSecretFile))
|
||||
{
|
||||
var content = (await File.ReadAllTextAsync(options.SharedSecretFile, ct).ConfigureAwait(false)).Trim();
|
||||
return DecodeSecret(content);
|
||||
}
|
||||
|
||||
// Then inline secret
|
||||
if (!string.IsNullOrWhiteSpace(options.SharedSecret))
|
||||
{
|
||||
return DecodeSecret(options.SharedSecret);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
private static byte[] DecodeSecret(string value)
|
||||
{
|
||||
// Try base64 first
|
||||
try
|
||||
{
|
||||
return Convert.FromBase64String(value);
|
||||
}
|
||||
catch (FormatException)
|
||||
{
|
||||
// Not base64
|
||||
}
|
||||
|
||||
// Try hex
|
||||
if (value.Length % 2 == 0 && IsHexString(value))
|
||||
{
|
||||
return Convert.FromHexString(value);
|
||||
}
|
||||
|
||||
// Treat as raw UTF-8
|
||||
return Encoding.UTF8.GetBytes(value);
|
||||
}
|
||||
|
||||
private static bool IsHexString(string value)
|
||||
{
|
||||
foreach (var c in value)
|
||||
{
|
||||
if (!char.IsAsciiHexDigit(c))
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private static byte[] BuildPae(string payloadType, byte[] payload)
|
||||
{
|
||||
var typeBytes = Encoding.UTF8.GetBytes(payloadType);
|
||||
var typeLenStr = typeBytes.Length.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
var payloadLenStr = payload.Length.ToString(System.Globalization.CultureInfo.InvariantCulture);
|
||||
|
||||
var prefixBytes = Encoding.UTF8.GetBytes(DssePrefix);
|
||||
var typeLenBytes = Encoding.UTF8.GetBytes(typeLenStr);
|
||||
var payloadLenBytes = Encoding.UTF8.GetBytes(payloadLenStr);
|
||||
|
||||
var totalSize = prefixBytes.Length + 1
|
||||
+ typeLenBytes.Length + 1
|
||||
+ typeBytes.Length + 1
|
||||
+ payloadLenBytes.Length + 1
|
||||
+ payload.Length;
|
||||
|
||||
var pae = new byte[totalSize];
|
||||
var offset = 0;
|
||||
|
||||
Buffer.BlockCopy(prefixBytes, 0, pae, offset, prefixBytes.Length);
|
||||
offset += prefixBytes.Length;
|
||||
pae[offset++] = 0x20;
|
||||
|
||||
Buffer.BlockCopy(typeLenBytes, 0, pae, offset, typeLenBytes.Length);
|
||||
offset += typeLenBytes.Length;
|
||||
pae[offset++] = 0x20;
|
||||
|
||||
Buffer.BlockCopy(typeBytes, 0, pae, offset, typeBytes.Length);
|
||||
offset += typeBytes.Length;
|
||||
pae[offset++] = 0x20;
|
||||
|
||||
Buffer.BlockCopy(payloadLenBytes, 0, pae, offset, payloadLenBytes.Length);
|
||||
offset += payloadLenBytes.Length;
|
||||
pae[offset++] = 0x20;
|
||||
|
||||
Buffer.BlockCopy(payload, 0, pae, offset, payload.Length);
|
||||
|
||||
return pae;
|
||||
}
|
||||
|
||||
private static byte[] FromBase64Url(string value)
|
||||
{
|
||||
var padded = value.Replace('-', '+').Replace('_', '/');
|
||||
switch (padded.Length % 4)
|
||||
{
|
||||
case 2: padded += "=="; break;
|
||||
case 3: padded += "="; break;
|
||||
}
|
||||
return Convert.FromBase64String(padded);
|
||||
}
|
||||
|
||||
private static async Task<string> ComputeFileSha256Async(string path, CancellationToken ct)
|
||||
{
|
||||
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, useAsync: true);
|
||||
var hash = await SHA256.HashDataAsync(stream, ct).ConfigureAwait(false);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
private sealed record SignatureVerificationResult(bool IsValid, string? KeyId, ImmutableArray<string> Errors)
|
||||
{
|
||||
public static SignatureVerificationResult Success(string? keyId) => new(true, keyId, []);
|
||||
public static SignatureVerificationResult Failure(params string[] errors) => new(false, null, [.. errors]);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.RegularExpressions;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets.Bundles;
|
||||
|
||||
/// <summary>
|
||||
/// Validates secret detection rules against the schema requirements.
|
||||
/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles
|
||||
/// </summary>
|
||||
public interface IRuleValidator
|
||||
{
|
||||
/// <summary>
|
||||
/// Validates a rule and returns validation errors, if any.
|
||||
/// </summary>
|
||||
RuleValidationResult Validate(SecretRule rule);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of rule validation.
|
||||
/// </summary>
|
||||
public sealed record RuleValidationResult
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the rule is valid.
|
||||
/// </summary>
|
||||
public required bool IsValid { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Validation errors encountered.
|
||||
/// </summary>
|
||||
public ImmutableArray<string> Errors { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Validation warnings (non-fatal).
|
||||
/// </summary>
|
||||
public ImmutableArray<string> Warnings { get; init; } = [];
|
||||
|
||||
public static RuleValidationResult Success() => new() { IsValid = true };
|
||||
|
||||
public static RuleValidationResult Failure(params string[] errors) => new()
|
||||
{
|
||||
IsValid = false,
|
||||
Errors = [.. errors]
|
||||
};
|
||||
|
||||
public static RuleValidationResult Failure(IEnumerable<string> errors) => new()
|
||||
{
|
||||
IsValid = false,
|
||||
Errors = [.. errors]
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of rule validation.
|
||||
/// </summary>
|
||||
public sealed class RuleValidator : IRuleValidator
|
||||
{
|
||||
private static readonly Regex NamespacedIdPattern = new(
|
||||
@"^[a-z][a-z0-9-]*(\.[a-z][a-z0-9-]*)+$",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
private static readonly Regex SemVerPattern = new(
|
||||
@"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
private readonly ILogger<RuleValidator> _logger;
|
||||
|
||||
public RuleValidator(ILogger<RuleValidator> logger)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public RuleValidationResult Validate(SecretRule rule)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rule);
|
||||
|
||||
var errors = new List<string>();
|
||||
var warnings = new List<string>();
|
||||
|
||||
// Validate ID
|
||||
if (string.IsNullOrWhiteSpace(rule.Id))
|
||||
{
|
||||
errors.Add("Rule ID is required.");
|
||||
}
|
||||
else if (!NamespacedIdPattern.IsMatch(rule.Id))
|
||||
{
|
||||
errors.Add($"Rule ID '{rule.Id}' must be namespaced (e.g., 'stellaops.secrets.aws-key').");
|
||||
}
|
||||
|
||||
// Validate version
|
||||
if (string.IsNullOrWhiteSpace(rule.Version))
|
||||
{
|
||||
errors.Add("Rule version is required.");
|
||||
}
|
||||
else if (!SemVerPattern.IsMatch(rule.Version))
|
||||
{
|
||||
errors.Add($"Rule version '{rule.Version}' must be valid SemVer.");
|
||||
}
|
||||
|
||||
// Validate name
|
||||
if (string.IsNullOrWhiteSpace(rule.Name))
|
||||
{
|
||||
warnings.Add("Rule name is recommended for documentation.");
|
||||
}
|
||||
|
||||
// Validate pattern for regex/composite rules
|
||||
if (rule.Type is SecretRuleType.Regex or SecretRuleType.Composite)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rule.Pattern))
|
||||
{
|
||||
errors.Add("Pattern is required for regex/composite rules.");
|
||||
}
|
||||
else
|
||||
{
|
||||
try
|
||||
{
|
||||
// Validate regex compiles
|
||||
_ = new Regex(rule.Pattern, RegexOptions.Compiled, TimeSpan.FromSeconds(1));
|
||||
}
|
||||
catch (RegexParseException ex)
|
||||
{
|
||||
errors.Add($"Invalid regex pattern: {ex.Message}");
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
errors.Add($"Invalid regex pattern: {ex.Message}");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Validate entropy threshold for entropy rules
|
||||
if (rule.Type is SecretRuleType.Entropy or SecretRuleType.Composite)
|
||||
{
|
||||
if (rule.EntropyThreshold <= 0 || rule.EntropyThreshold > 8)
|
||||
{
|
||||
warnings.Add($"Entropy threshold {rule.EntropyThreshold} may be out of typical range (3.0-6.0).");
|
||||
}
|
||||
}
|
||||
|
||||
// Validate min/max length
|
||||
if (rule.MinLength < 0)
|
||||
{
|
||||
errors.Add("MinLength cannot be negative.");
|
||||
}
|
||||
|
||||
if (rule.MaxLength < rule.MinLength)
|
||||
{
|
||||
errors.Add("MaxLength must be greater than or equal to MinLength.");
|
||||
}
|
||||
|
||||
// Validate file patterns if provided
|
||||
foreach (var pattern in rule.FilePatterns)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(pattern))
|
||||
{
|
||||
warnings.Add("Empty file pattern will be ignored.");
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.Count > 0)
|
||||
{
|
||||
_logger.LogDebug("Rule {RuleId} validation failed: {Errors}", rule.Id, string.Join("; ", errors));
|
||||
return new RuleValidationResult
|
||||
{
|
||||
IsValid = false,
|
||||
Errors = [.. errors],
|
||||
Warnings = [.. warnings]
|
||||
};
|
||||
}
|
||||
|
||||
if (warnings.Count > 0)
|
||||
{
|
||||
_logger.LogDebug("Rule {RuleId} validated with warnings: {Warnings}", rule.Id, string.Join("; ", warnings));
|
||||
}
|
||||
|
||||
return new RuleValidationResult
|
||||
{
|
||||
IsValid = true,
|
||||
Errors = [],
|
||||
Warnings = [.. warnings]
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Combines multiple detection strategies for comprehensive secret detection.
|
||||
/// </summary>
|
||||
public sealed class CompositeSecretDetector : ISecretDetector
|
||||
{
|
||||
private readonly RegexDetector _regexDetector;
|
||||
private readonly EntropyDetector _entropyDetector;
|
||||
private readonly ILogger<CompositeSecretDetector> _logger;
|
||||
|
||||
public CompositeSecretDetector(
|
||||
RegexDetector regexDetector,
|
||||
EntropyDetector entropyDetector,
|
||||
ILogger<CompositeSecretDetector> logger)
|
||||
{
|
||||
_regexDetector = regexDetector ?? throw new ArgumentNullException(nameof(regexDetector));
|
||||
_entropyDetector = entropyDetector ?? throw new ArgumentNullException(nameof(entropyDetector));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string DetectorId => "composite";
|
||||
|
||||
public bool CanHandle(SecretRuleType ruleType) => true;
|
||||
|
||||
public async ValueTask<IReadOnlyList<SecretMatch>> DetectAsync(
|
||||
ReadOnlyMemory<byte> content,
|
||||
string filePath,
|
||||
SecretRule rule,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
var results = new List<SecretMatch>();
|
||||
|
||||
// Choose detector based on rule type
|
||||
switch (rule.Type)
|
||||
{
|
||||
case SecretRuleType.Regex:
|
||||
var regexMatches = await _regexDetector.DetectAsync(content, filePath, rule, ct);
|
||||
results.AddRange(regexMatches);
|
||||
break;
|
||||
|
||||
case SecretRuleType.Entropy:
|
||||
var entropyMatches = await _entropyDetector.DetectAsync(content, filePath, rule, ct);
|
||||
results.AddRange(entropyMatches);
|
||||
break;
|
||||
|
||||
case SecretRuleType.Composite:
|
||||
// Run both detectors and merge results
|
||||
var regexTask = _regexDetector.DetectAsync(content, filePath, rule, ct);
|
||||
var entropyTask = _entropyDetector.DetectAsync(content, filePath, rule, ct);
|
||||
|
||||
var regexResults = await regexTask;
|
||||
var entropyResults = await entropyTask;
|
||||
|
||||
// Add regex matches
|
||||
results.AddRange(regexResults);
|
||||
|
||||
// Add entropy matches, boosting confidence if they overlap with regex
|
||||
foreach (var entropyMatch in entropyResults)
|
||||
{
|
||||
var overlappingRegex = regexResults.FirstOrDefault(r =>
|
||||
r.LineNumber == entropyMatch.LineNumber &&
|
||||
OverlapsColumn(r, entropyMatch));
|
||||
|
||||
if (overlappingRegex is not null)
|
||||
{
|
||||
// Boost confidence for overlapping matches
|
||||
results.Add(entropyMatch with
|
||||
{
|
||||
ConfidenceScore = Math.Min(0.99, entropyMatch.ConfidenceScore + 0.1)
|
||||
});
|
||||
}
|
||||
else
|
||||
{
|
||||
results.Add(entropyMatch);
|
||||
}
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
// Deduplicate overlapping matches
|
||||
return DeduplicateMatches(results);
|
||||
}
|
||||
|
||||
private static bool OverlapsColumn(SecretMatch a, SecretMatch b)
|
||||
{
|
||||
return a.ColumnStart <= b.ColumnEnd && b.ColumnStart <= a.ColumnEnd;
|
||||
}
|
||||
|
||||
private static IReadOnlyList<SecretMatch> DeduplicateMatches(List<SecretMatch> matches)
|
||||
{
|
||||
if (matches.Count <= 1)
|
||||
{
|
||||
return matches;
|
||||
}
|
||||
|
||||
// Sort by position
|
||||
matches.Sort((a, b) =>
|
||||
{
|
||||
var lineComp = a.LineNumber.CompareTo(b.LineNumber);
|
||||
return lineComp != 0 ? lineComp : a.ColumnStart.CompareTo(b.ColumnStart);
|
||||
});
|
||||
|
||||
var deduplicated = new List<SecretMatch>();
|
||||
SecretMatch? previous = null;
|
||||
|
||||
foreach (var match in matches)
|
||||
{
|
||||
if (previous is null)
|
||||
{
|
||||
previous = match;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this match overlaps with the previous one
|
||||
if (match.LineNumber == previous.LineNumber && OverlapsColumn(previous, match))
|
||||
{
|
||||
// Keep the one with higher confidence
|
||||
if (match.ConfidenceScore > previous.ConfidenceScore)
|
||||
{
|
||||
previous = match;
|
||||
}
|
||||
// Otherwise keep previous
|
||||
}
|
||||
else
|
||||
{
|
||||
deduplicated.Add(previous);
|
||||
previous = match;
|
||||
}
|
||||
}
|
||||
|
||||
if (previous is not null)
|
||||
{
|
||||
deduplicated.Add(previous);
|
||||
}
|
||||
|
||||
return deduplicated;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,161 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Calculates Shannon entropy for detecting high-entropy strings that may be secrets.
|
||||
/// </summary>
|
||||
public static class EntropyCalculator
|
||||
{
|
||||
/// <summary>
|
||||
/// Calculates Shannon entropy in bits per character for the given data.
|
||||
/// </summary>
|
||||
/// <param name="data">The data to analyze.</param>
|
||||
/// <returns>Entropy in bits per character (0.0 to 8.0 for byte data).</returns>
|
||||
public static double Calculate(ReadOnlySpan<byte> data)
|
||||
{
|
||||
if (data.IsEmpty)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// Count occurrences of each byte value
|
||||
Span<int> counts = stackalloc int[256];
|
||||
counts.Clear();
|
||||
|
||||
foreach (byte b in data)
|
||||
{
|
||||
counts[b]++;
|
||||
}
|
||||
|
||||
// Calculate entropy using Shannon's formula
|
||||
double entropy = 0.0;
|
||||
double length = data.Length;
|
||||
|
||||
for (int i = 0; i < 256; i++)
|
||||
{
|
||||
if (counts[i] > 0)
|
||||
{
|
||||
double probability = counts[i] / length;
|
||||
entropy -= probability * Math.Log2(probability);
|
||||
}
|
||||
}
|
||||
|
||||
return entropy;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Calculates Shannon entropy for a string.
|
||||
/// </summary>
|
||||
public static double Calculate(ReadOnlySpan<char> data)
|
||||
{
|
||||
if (data.IsEmpty)
|
||||
{
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
// For character data, we calculate based on unique characters seen
|
||||
var counts = new Dictionary<char, int>();
|
||||
|
||||
foreach (char c in data)
|
||||
{
|
||||
counts.TryGetValue(c, out int count);
|
||||
counts[c] = count + 1;
|
||||
}
|
||||
|
||||
double entropy = 0.0;
|
||||
double length = data.Length;
|
||||
|
||||
foreach (var count in counts.Values)
|
||||
{
|
||||
double probability = count / length;
|
||||
entropy -= probability * Math.Log2(probability);
|
||||
}
|
||||
|
||||
return entropy;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the data appears to be base64 encoded.
|
||||
/// </summary>
|
||||
public static bool IsBase64Like(ReadOnlySpan<char> data)
|
||||
{
|
||||
if (data.Length < 4)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
int validChars = 0;
|
||||
foreach (char c in data)
|
||||
{
|
||||
if (char.IsLetterOrDigit(c) || c is '+' or '/' or '=')
|
||||
{
|
||||
validChars++;
|
||||
}
|
||||
}
|
||||
|
||||
return validChars >= data.Length * 0.9;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the data appears to be hexadecimal.
|
||||
/// </summary>
|
||||
public static bool IsHexLike(ReadOnlySpan<char> data)
|
||||
{
|
||||
if (data.Length < 8)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
foreach (char c in data)
|
||||
{
|
||||
if (!char.IsAsciiHexDigit(c))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines if a string is likely a secret based on entropy and charset.
|
||||
/// </summary>
|
||||
/// <param name="data">The string to check.</param>
|
||||
/// <param name="threshold">Minimum entropy threshold (default 4.5).</param>
|
||||
/// <returns>True if the string appears to be a high-entropy secret.</returns>
|
||||
public static bool IsLikelySecret(ReadOnlySpan<char> data, double threshold = 4.5)
|
||||
{
|
||||
if (data.Length < 16)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
// Skip if it looks like a UUID (common false positive)
|
||||
if (LooksLikeUuid(data))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var entropy = Calculate(data);
|
||||
return entropy >= threshold;
|
||||
}
|
||||
|
||||
private static bool LooksLikeUuid(ReadOnlySpan<char> data)
|
||||
{
|
||||
// UUID format: xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx (36 chars)
|
||||
if (data.Length == 36)
|
||||
{
|
||||
if (data[8] == '-' && data[13] == '-' && data[18] == '-' && data[23] == '-')
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
// UUID without dashes: 32 hex chars
|
||||
if (data.Length == 32 && IsHexLike(data))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,199 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Entropy-based secret detector for high-entropy strings.
|
||||
/// </summary>
|
||||
public sealed class EntropyDetector : ISecretDetector
|
||||
{
|
||||
private readonly ILogger<EntropyDetector> _logger;
|
||||
|
||||
// Regex to find potential secret strings (alphanumeric with common secret characters)
|
||||
private static readonly Regex CandidatePattern = new(
|
||||
@"[A-Za-z0-9+/=_\-]{16,}",
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant,
|
||||
TimeSpan.FromSeconds(5));
|
||||
|
||||
public EntropyDetector(ILogger<EntropyDetector> logger)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string DetectorId => "entropy";
|
||||
|
||||
public bool CanHandle(SecretRuleType ruleType) =>
|
||||
ruleType is SecretRuleType.Entropy or SecretRuleType.Composite;
|
||||
|
||||
public ValueTask<IReadOnlyList<SecretMatch>> DetectAsync(
|
||||
ReadOnlyMemory<byte> content,
|
||||
string filePath,
|
||||
SecretRule rule,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<SecretMatch>>([]);
|
||||
}
|
||||
|
||||
// Decode content as UTF-8
|
||||
string text;
|
||||
try
|
||||
{
|
||||
text = Encoding.UTF8.GetString(content.Span);
|
||||
}
|
||||
catch (DecoderFallbackException)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<SecretMatch>>([]);
|
||||
}
|
||||
|
||||
var matches = new List<SecretMatch>();
|
||||
var lineStarts = ComputeLineStarts(text);
|
||||
var threshold = rule.EntropyThreshold > 0 ? rule.EntropyThreshold : 4.5;
|
||||
var minLength = rule.MinLength > 0 ? rule.MinLength : 16;
|
||||
var maxLength = rule.MaxLength > 0 ? rule.MaxLength : 1000;
|
||||
|
||||
try
|
||||
{
|
||||
foreach (Match candidate in CandidatePattern.Matches(text))
|
||||
{
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
var value = candidate.Value.AsSpan();
|
||||
|
||||
// Check length constraints
|
||||
if (value.Length < minLength || value.Length > maxLength)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip common false positives
|
||||
if (ShouldSkip(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Calculate entropy
|
||||
var entropy = EntropyCalculator.Calculate(value);
|
||||
if (entropy < threshold)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var (lineNumber, columnStart) = GetLineAndColumn(lineStarts, candidate.Index);
|
||||
var matchBytes = Encoding.UTF8.GetBytes(candidate.Value);
|
||||
|
||||
// Adjust confidence based on entropy level
|
||||
var confidenceScore = CalculateConfidence(entropy, threshold);
|
||||
|
||||
matches.Add(new SecretMatch
|
||||
{
|
||||
Rule = rule,
|
||||
FilePath = filePath,
|
||||
LineNumber = lineNumber,
|
||||
ColumnStart = columnStart,
|
||||
ColumnEnd = columnStart + candidate.Length - 1,
|
||||
RawMatch = matchBytes,
|
||||
ConfidenceScore = confidenceScore,
|
||||
DetectorId = DetectorId,
|
||||
Entropy = entropy
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (RegexMatchTimeoutException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Entropy detection timeout on file '{FilePath}'",
|
||||
filePath);
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<SecretMatch>>(matches);
|
||||
}
|
||||
|
||||
private static bool ShouldSkip(ReadOnlySpan<char> value)
|
||||
{
|
||||
// Skip UUIDs
|
||||
if (EntropyCalculator.IsHexLike(value) && value.Length == 32)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Skip if it looks like a UUID with dashes
|
||||
if (value.Length == 36 && value[8] == '-')
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Skip common hash prefixes that aren't secrets
|
||||
if (value.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase) ||
|
||||
value.StartsWith("sha512:", StringComparison.OrdinalIgnoreCase) ||
|
||||
value.StartsWith("md5:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
// Skip if it's all the same character repeated
|
||||
char first = value[0];
|
||||
bool allSame = true;
|
||||
for (int i = 1; i < value.Length; i++)
|
||||
{
|
||||
if (value[i] != first)
|
||||
{
|
||||
allSame = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (allSame)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static double CalculateConfidence(double entropy, double threshold)
|
||||
{
|
||||
// Scale confidence based on how far above threshold
|
||||
// entropy >= threshold + 1.5 => 0.95 (high)
|
||||
// entropy >= threshold + 0.5 => 0.75 (medium)
|
||||
// entropy >= threshold => 0.5 (low)
|
||||
var excess = entropy - threshold;
|
||||
return excess switch
|
||||
{
|
||||
>= 1.5 => 0.95,
|
||||
>= 0.5 => 0.75,
|
||||
_ => 0.5
|
||||
};
|
||||
}
|
||||
|
||||
private static List<int> ComputeLineStarts(string text)
|
||||
{
|
||||
var lineStarts = new List<int> { 0 };
|
||||
for (int i = 0; i < text.Length; i++)
|
||||
{
|
||||
if (text[i] == '\n')
|
||||
{
|
||||
lineStarts.Add(i + 1);
|
||||
}
|
||||
}
|
||||
return lineStarts;
|
||||
}
|
||||
|
||||
private static (int lineNumber, int column) GetLineAndColumn(List<int> lineStarts, int position)
|
||||
{
|
||||
int line = 1;
|
||||
for (int i = 1; i < lineStarts.Count; i++)
|
||||
{
|
||||
if (lineStarts[i] > position)
|
||||
{
|
||||
break;
|
||||
}
|
||||
line = i + 1;
|
||||
}
|
||||
|
||||
int lineStart = lineStarts[line - 1];
|
||||
int column = position - lineStart + 1;
|
||||
return (line, column);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Contract for secret detection strategies.
|
||||
/// Implementations must be thread-safe and deterministic.
|
||||
/// </summary>
|
||||
public interface ISecretDetector
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique identifier for this detector (e.g., "regex", "entropy").
|
||||
/// </summary>
|
||||
string DetectorId { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Detects secrets in the provided content using the specified rule.
|
||||
/// </summary>
|
||||
/// <param name="content">The file content to scan.</param>
|
||||
/// <param name="filePath">The file path (for reporting).</param>
|
||||
/// <param name="rule">The rule to apply.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>List of matches found.</returns>
|
||||
ValueTask<IReadOnlyList<SecretMatch>> DetectAsync(
|
||||
ReadOnlyMemory<byte> content,
|
||||
string filePath,
|
||||
SecretRule rule,
|
||||
CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Checks if this detector can handle the specified rule type.
|
||||
/// </summary>
|
||||
bool CanHandle(SecretRuleType ruleType);
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Regex-based secret detector.
|
||||
/// </summary>
|
||||
public sealed class RegexDetector : ISecretDetector
|
||||
{
|
||||
private readonly ILogger<RegexDetector> _logger;
|
||||
|
||||
public RegexDetector(ILogger<RegexDetector> logger)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
public string DetectorId => "regex";
|
||||
|
||||
public bool CanHandle(SecretRuleType ruleType) =>
|
||||
ruleType is SecretRuleType.Regex or SecretRuleType.Composite;
|
||||
|
||||
public ValueTask<IReadOnlyList<SecretMatch>> DetectAsync(
|
||||
ReadOnlyMemory<byte> content,
|
||||
string filePath,
|
||||
SecretRule rule,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<SecretMatch>>([]);
|
||||
}
|
||||
|
||||
var regex = rule.GetCompiledPattern();
|
||||
if (regex is null)
|
||||
{
|
||||
_logger.LogWarning("Rule '{RuleId}' has invalid regex pattern", rule.Id);
|
||||
return ValueTask.FromResult<IReadOnlyList<SecretMatch>>([]);
|
||||
}
|
||||
|
||||
// Decode content as UTF-8
|
||||
string text;
|
||||
try
|
||||
{
|
||||
text = Encoding.UTF8.GetString(content.Span);
|
||||
}
|
||||
catch (DecoderFallbackException)
|
||||
{
|
||||
// Not valid UTF-8, skip
|
||||
return ValueTask.FromResult<IReadOnlyList<SecretMatch>>([]);
|
||||
}
|
||||
|
||||
// Apply keyword pre-filter
|
||||
if (!rule.MightMatch(text.AsSpan()))
|
||||
{
|
||||
return ValueTask.FromResult<IReadOnlyList<SecretMatch>>([]);
|
||||
}
|
||||
|
||||
var matches = new List<SecretMatch>();
|
||||
var lineStarts = ComputeLineStarts(text);
|
||||
|
||||
try
|
||||
{
|
||||
foreach (Match match in regex.Matches(text))
|
||||
{
|
||||
if (ct.IsCancellationRequested)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (match.Length < rule.MinLength || match.Length > rule.MaxLength)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var (lineNumber, columnStart) = GetLineAndColumn(lineStarts, match.Index);
|
||||
var matchBytes = Encoding.UTF8.GetBytes(match.Value);
|
||||
|
||||
matches.Add(new SecretMatch
|
||||
{
|
||||
Rule = rule,
|
||||
FilePath = filePath,
|
||||
LineNumber = lineNumber,
|
||||
ColumnStart = columnStart,
|
||||
ColumnEnd = columnStart + match.Length - 1,
|
||||
RawMatch = matchBytes,
|
||||
ConfidenceScore = MapConfidenceToScore(rule.Confidence),
|
||||
DetectorId = DetectorId
|
||||
});
|
||||
}
|
||||
}
|
||||
catch (RegexMatchTimeoutException)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Regex timeout for rule '{RuleId}' on file '{FilePath}'",
|
||||
rule.Id,
|
||||
filePath);
|
||||
}
|
||||
|
||||
return ValueTask.FromResult<IReadOnlyList<SecretMatch>>(matches);
|
||||
}
|
||||
|
||||
private static List<int> ComputeLineStarts(string text)
|
||||
{
|
||||
var lineStarts = new List<int> { 0 };
|
||||
for (int i = 0; i < text.Length; i++)
|
||||
{
|
||||
if (text[i] == '\n')
|
||||
{
|
||||
lineStarts.Add(i + 1);
|
||||
}
|
||||
}
|
||||
return lineStarts;
|
||||
}
|
||||
|
||||
private static (int lineNumber, int column) GetLineAndColumn(List<int> lineStarts, int position)
|
||||
{
|
||||
int line = 1;
|
||||
for (int i = 1; i < lineStarts.Count; i++)
|
||||
{
|
||||
if (lineStarts[i] > position)
|
||||
{
|
||||
break;
|
||||
}
|
||||
line = i + 1;
|
||||
}
|
||||
|
||||
int lineStart = lineStarts[line - 1];
|
||||
int column = position - lineStart + 1;
|
||||
return (line, column);
|
||||
}
|
||||
|
||||
private static double MapConfidenceToScore(SecretConfidence confidence) => confidence switch
|
||||
{
|
||||
SecretConfidence.Low => 0.5,
|
||||
SecretConfidence.Medium => 0.75,
|
||||
SecretConfidence.High => 0.95,
|
||||
_ => 0.5
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Represents a potential secret match found by a detector.
|
||||
/// </summary>
|
||||
public sealed record SecretMatch
|
||||
{
|
||||
/// <summary>
|
||||
/// The rule that produced this match.
|
||||
/// </summary>
|
||||
public required SecretRule Rule { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The file path where the match was found.
|
||||
/// </summary>
|
||||
public required string FilePath { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The 1-based line number of the match.
|
||||
/// </summary>
|
||||
public required int LineNumber { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The 1-based column where the match starts.
|
||||
/// </summary>
|
||||
public required int ColumnStart { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The 1-based column where the match ends.
|
||||
/// </summary>
|
||||
public required int ColumnEnd { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The raw matched content (will be masked before output).
|
||||
/// </summary>
|
||||
public required ReadOnlyMemory<byte> RawMatch { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Confidence score from 0.0 to 1.0.
|
||||
/// </summary>
|
||||
public required double ConfidenceScore { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The detector that found this match.
|
||||
/// </summary>
|
||||
public required string DetectorId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional entropy value if entropy-based detection was used.
|
||||
/// </summary>
|
||||
public double? Entropy { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets the length of the matched content.
|
||||
/// </summary>
|
||||
public int MatchLength => RawMatch.Length;
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Aggregated finding for storage in ScanAnalysisStore.
|
||||
/// </summary>
|
||||
public sealed record SecretFinding
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique identifier for this finding.
|
||||
/// </summary>
|
||||
public required Guid Id { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The evidence record.
|
||||
/// </summary>
|
||||
public required SecretLeakEvidence Evidence { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The scan that produced this finding.
|
||||
/// </summary>
|
||||
public required string ScanId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The tenant that owns this finding.
|
||||
/// </summary>
|
||||
public required string TenantId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The artifact digest (container image or other artifact).
|
||||
/// </summary>
|
||||
public required string ArtifactDigest { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new finding from evidence.
|
||||
/// </summary>
|
||||
public static SecretFinding Create(
|
||||
SecretLeakEvidence evidence,
|
||||
string scanId,
|
||||
string tenantId,
|
||||
string artifactDigest,
|
||||
Guid? id = null)
|
||||
{
|
||||
return new SecretFinding
|
||||
{
|
||||
Id = id ?? Guid.NewGuid(),
|
||||
Evidence = evidence,
|
||||
ScanId = scanId,
|
||||
TenantId = tenantId,
|
||||
ArtifactDigest = artifactDigest
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Evidence record for a detected secret leak.
|
||||
/// This record is emitted to the policy engine for decision-making.
|
||||
/// </summary>
|
||||
public sealed record SecretLeakEvidence
|
||||
{
|
||||
/// <summary>
|
||||
/// The evidence type identifier.
|
||||
/// </summary>
|
||||
public const string EvidenceType = "secret.leak";
|
||||
|
||||
/// <summary>
|
||||
/// The rule ID that produced this finding.
|
||||
/// </summary>
|
||||
public required string RuleId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The rule version.
|
||||
/// </summary>
|
||||
public required string RuleVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The severity of the finding.
|
||||
/// </summary>
|
||||
public required SecretSeverity Severity { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The confidence level of the finding.
|
||||
/// </summary>
|
||||
public required SecretConfidence Confidence { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The file path where the secret was found (relative to scan root).
|
||||
/// </summary>
|
||||
public required string FilePath { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The 1-based line number.
|
||||
/// </summary>
|
||||
public required int LineNumber { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The 1-based column number.
|
||||
/// </summary>
|
||||
public int ColumnNumber { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The masked payload (e.g., "AKIA****B7"). Never contains the actual secret.
|
||||
/// </summary>
|
||||
public required string Mask { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The bundle ID that contained the rule.
|
||||
/// </summary>
|
||||
public required string BundleId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The bundle version.
|
||||
/// </summary>
|
||||
public required string BundleVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When this finding was detected.
|
||||
/// </summary>
|
||||
public required DateTimeOffset DetectedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The detector that found this secret.
|
||||
/// </summary>
|
||||
public required string DetectorId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Entropy value if entropy-based detection was used.
|
||||
/// </summary>
|
||||
public double? Entropy { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Additional metadata for the finding.
|
||||
/// </summary>
|
||||
public ImmutableDictionary<string, string> Metadata { get; init; } =
|
||||
ImmutableDictionary<string, string>.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// Creates evidence from a secret match and masker.
|
||||
/// </summary>
|
||||
public static SecretLeakEvidence FromMatch(
|
||||
SecretMatch match,
|
||||
IPayloadMasker masker,
|
||||
SecretRuleset ruleset,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(match);
|
||||
ArgumentNullException.ThrowIfNull(masker);
|
||||
ArgumentNullException.ThrowIfNull(ruleset);
|
||||
ArgumentNullException.ThrowIfNull(timeProvider);
|
||||
|
||||
var masked = masker.Mask(match.RawMatch.Span, match.Rule.MaskingHint);
|
||||
|
||||
return new SecretLeakEvidence
|
||||
{
|
||||
RuleId = match.Rule.Id,
|
||||
RuleVersion = match.Rule.Version,
|
||||
Severity = match.Rule.Severity,
|
||||
Confidence = MapScoreToConfidence(match.ConfidenceScore, match.Rule.Confidence),
|
||||
FilePath = match.FilePath,
|
||||
LineNumber = match.LineNumber,
|
||||
ColumnNumber = match.ColumnStart,
|
||||
Mask = masked,
|
||||
BundleId = ruleset.Id,
|
||||
BundleVersion = ruleset.Version,
|
||||
DetectedAt = timeProvider.GetUtcNow(),
|
||||
DetectorId = match.DetectorId,
|
||||
Entropy = match.Entropy
|
||||
};
|
||||
}
|
||||
|
||||
private static SecretConfidence MapScoreToConfidence(double score, SecretConfidence ruleDefault)
|
||||
{
|
||||
// Adjust confidence based on detection score
|
||||
if (score >= 0.9)
|
||||
{
|
||||
return SecretConfidence.High;
|
||||
}
|
||||
if (score >= 0.7)
|
||||
{
|
||||
return SecretConfidence.Medium;
|
||||
}
|
||||
if (score >= 0.5)
|
||||
{
|
||||
return ruleDefault;
|
||||
}
|
||||
return SecretConfidence.Low;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
global using System.Collections.Immutable;
|
||||
global using System.Diagnostics.CodeAnalysis;
|
||||
global using System.Globalization;
|
||||
global using System.Text;
|
||||
global using System.Text.Json;
|
||||
global using System.Text.RegularExpressions;
|
||||
global using Microsoft.Extensions.DependencyInjection;
|
||||
global using Microsoft.Extensions.DependencyInjection.Extensions;
|
||||
global using Microsoft.Extensions.Logging;
|
||||
global using Microsoft.Extensions.Options;
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Contract for secret payload masking.
|
||||
/// </summary>
|
||||
public interface IPayloadMasker
|
||||
{
|
||||
/// <summary>
|
||||
/// Masks a secret payload, preserving prefix/suffix for identification.
|
||||
/// </summary>
|
||||
/// <param name="payload">The raw secret bytes.</param>
|
||||
/// <param name="hint">Optional masking hint (e.g., "prefix:4,suffix:2").</param>
|
||||
/// <returns>Masked string (e.g., "AKIA****B7").</returns>
|
||||
string Mask(ReadOnlySpan<byte> payload, string? hint = null);
|
||||
|
||||
/// <summary>
|
||||
/// Masks a secret string, preserving prefix/suffix for identification.
|
||||
/// </summary>
|
||||
/// <param name="payload">The raw secret string.</param>
|
||||
/// <param name="hint">Optional masking hint (e.g., "prefix:4,suffix:2").</param>
|
||||
/// <returns>Masked string (e.g., "AKIA****B7").</returns>
|
||||
string Mask(ReadOnlySpan<char> payload, string? hint = null);
|
||||
}
|
||||
@@ -0,0 +1,151 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of payload masking for secrets.
|
||||
/// </summary>
|
||||
public sealed class PayloadMasker : IPayloadMasker
|
||||
{
|
||||
/// <summary>
|
||||
/// Default number of characters to preserve at the start.
|
||||
/// </summary>
|
||||
public const int DefaultPrefixLength = 4;
|
||||
|
||||
/// <summary>
|
||||
/// Default number of characters to preserve at the end.
|
||||
/// </summary>
|
||||
public const int DefaultSuffixLength = 2;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum number of mask characters to use.
|
||||
/// </summary>
|
||||
public const int MaxMaskLength = 8;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum output length for masked values.
|
||||
/// </summary>
|
||||
public const int MinOutputLength = 8;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum total characters to expose (prefix + suffix).
|
||||
/// </summary>
|
||||
public const int MaxExposedChars = 6;
|
||||
|
||||
/// <summary>
|
||||
/// The character used for masking.
|
||||
/// </summary>
|
||||
public const char MaskChar = '*';
|
||||
|
||||
public string Mask(ReadOnlySpan<byte> payload, string? hint = null)
|
||||
{
|
||||
if (payload.IsEmpty)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
// Try to decode as UTF-8
|
||||
try
|
||||
{
|
||||
var text = Encoding.UTF8.GetString(payload);
|
||||
return Mask(text.AsSpan(), hint);
|
||||
}
|
||||
catch (DecoderFallbackException)
|
||||
{
|
||||
// Not valid UTF-8, represent as hex
|
||||
var hex = Convert.ToHexString(payload);
|
||||
return Mask(hex.AsSpan(), hint);
|
||||
}
|
||||
}
|
||||
|
||||
public string Mask(ReadOnlySpan<char> payload, string? hint = null)
|
||||
{
|
||||
if (payload.IsEmpty)
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var (prefixLen, suffixLen) = ParseHint(hint);
|
||||
|
||||
// Enforce maximum exposed characters
|
||||
if (prefixLen + suffixLen > MaxExposedChars)
|
||||
{
|
||||
var ratio = (double)prefixLen / (prefixLen + suffixLen);
|
||||
prefixLen = (int)(MaxExposedChars * ratio);
|
||||
suffixLen = MaxExposedChars - prefixLen;
|
||||
}
|
||||
|
||||
// Handle short payloads
|
||||
if (payload.Length <= prefixLen + suffixLen)
|
||||
{
|
||||
// Too short to mask meaningfully, just return masked placeholder
|
||||
return new string(MaskChar, Math.Min(payload.Length, MinOutputLength));
|
||||
}
|
||||
|
||||
// Calculate mask length
|
||||
var middleLength = payload.Length - prefixLen - suffixLen;
|
||||
var maskLength = Math.Min(middleLength, MaxMaskLength);
|
||||
|
||||
// Build masked output
|
||||
var sb = new StringBuilder(prefixLen + maskLength + suffixLen);
|
||||
|
||||
// Prefix
|
||||
if (prefixLen > 0)
|
||||
{
|
||||
sb.Append(payload[..prefixLen]);
|
||||
}
|
||||
|
||||
// Mask
|
||||
sb.Append(MaskChar, maskLength);
|
||||
|
||||
// Suffix
|
||||
if (suffixLen > 0)
|
||||
{
|
||||
sb.Append(payload[^suffixLen..]);
|
||||
}
|
||||
|
||||
// Ensure minimum length
|
||||
while (sb.Length < MinOutputLength)
|
||||
{
|
||||
sb.Insert(prefixLen, MaskChar);
|
||||
}
|
||||
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
private static (int prefix, int suffix) ParseHint(string? hint)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(hint))
|
||||
{
|
||||
return (DefaultPrefixLength, DefaultSuffixLength);
|
||||
}
|
||||
|
||||
int prefix = DefaultPrefixLength;
|
||||
int suffix = DefaultSuffixLength;
|
||||
|
||||
// Parse hint format: "prefix:4,suffix:2"
|
||||
var parts = hint.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries);
|
||||
foreach (var part in parts)
|
||||
{
|
||||
var kv = part.Split(':', StringSplitOptions.TrimEntries);
|
||||
if (kv.Length != 2)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!int.TryParse(kv[1], NumberStyles.Integer, CultureInfo.InvariantCulture, out var value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (kv[0].Equals("prefix", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
prefix = Math.Max(0, Math.Min(value, MaxExposedChars));
|
||||
}
|
||||
else if (kv[0].Equals("suffix", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
suffix = Math.Max(0, Math.Min(value, MaxExposedChars));
|
||||
}
|
||||
}
|
||||
|
||||
return (prefix, suffix);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Contract for loading secret detection rulesets.
|
||||
/// </summary>
|
||||
public interface IRulesetLoader
|
||||
{
|
||||
/// <summary>
|
||||
/// Loads a ruleset from a directory containing bundle files.
|
||||
/// </summary>
|
||||
/// <param name="bundlePath">Path to the bundle directory.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The loaded ruleset.</returns>
|
||||
ValueTask<SecretRuleset> LoadAsync(string bundlePath, CancellationToken ct = default);
|
||||
|
||||
/// <summary>
|
||||
/// Loads a ruleset from a JSONL stream.
|
||||
/// </summary>
|
||||
/// <param name="rulesStream">Stream containing NDJSON rule definitions.</param>
|
||||
/// <param name="bundleId">The bundle identifier.</param>
|
||||
/// <param name="bundleVersion">The bundle version.</param>
|
||||
/// <param name="ct">Cancellation token.</param>
|
||||
/// <returns>The loaded ruleset.</returns>
|
||||
ValueTask<SecretRuleset> LoadFromJsonlAsync(
|
||||
Stream rulesStream,
|
||||
string bundleId,
|
||||
string bundleVersion,
|
||||
CancellationToken ct = default);
|
||||
}
|
||||
@@ -0,0 +1,227 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Loads secret detection rulesets from bundle files.
|
||||
/// </summary>
|
||||
public sealed class RulesetLoader : IRulesetLoader
|
||||
{
|
||||
private readonly ILogger<RulesetLoader> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
private static readonly JsonSerializerOptions JsonOptions = new()
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
Converters =
|
||||
{
|
||||
new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)
|
||||
}
|
||||
};
|
||||
|
||||
public RulesetLoader(ILogger<RulesetLoader> logger, TimeProvider timeProvider)
|
||||
{
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public async ValueTask<SecretRuleset> LoadAsync(string bundlePath, CancellationToken ct = default)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(bundlePath))
|
||||
{
|
||||
throw new ArgumentException("Bundle path is required", nameof(bundlePath));
|
||||
}
|
||||
|
||||
if (!Directory.Exists(bundlePath))
|
||||
{
|
||||
throw new DirectoryNotFoundException($"Bundle directory not found: {bundlePath}");
|
||||
}
|
||||
|
||||
// Load manifest
|
||||
var manifestPath = Path.Combine(bundlePath, "secrets.ruleset.manifest.json");
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
throw new FileNotFoundException("Bundle manifest not found", manifestPath);
|
||||
}
|
||||
|
||||
var manifestJson = await File.ReadAllTextAsync(manifestPath, ct);
|
||||
var manifest = JsonSerializer.Deserialize<BundleManifest>(manifestJson, JsonOptions)
|
||||
?? throw new InvalidOperationException("Failed to parse bundle manifest");
|
||||
|
||||
// Load rules
|
||||
var rulesPath = Path.Combine(bundlePath, "secrets.ruleset.rules.jsonl");
|
||||
if (!File.Exists(rulesPath))
|
||||
{
|
||||
throw new FileNotFoundException("Bundle rules file not found", rulesPath);
|
||||
}
|
||||
|
||||
await using var rulesStream = File.OpenRead(rulesPath);
|
||||
var ruleset = await LoadFromJsonlAsync(
|
||||
rulesStream,
|
||||
manifest.Id ?? "secrets.ruleset",
|
||||
manifest.Version ?? "unknown",
|
||||
ct);
|
||||
|
||||
// Verify integrity if digest is available
|
||||
if (!string.IsNullOrEmpty(manifest.Integrity?.RulesSha256))
|
||||
{
|
||||
rulesStream.Position = 0;
|
||||
var actualDigest = await ComputeSha256Async(rulesStream, ct);
|
||||
if (!string.Equals(actualDigest, manifest.Integrity.RulesSha256, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Rules file integrity check failed. Expected: {manifest.Integrity.RulesSha256}, Actual: {actualDigest}");
|
||||
}
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Loaded secrets ruleset '{BundleId}' version {Version} with {RuleCount} rules ({EnabledCount} enabled)",
|
||||
ruleset.Id,
|
||||
ruleset.Version,
|
||||
ruleset.Rules.Length,
|
||||
ruleset.EnabledRuleCount);
|
||||
|
||||
return ruleset;
|
||||
}
|
||||
|
||||
public async ValueTask<SecretRuleset> LoadFromJsonlAsync(
|
||||
Stream rulesStream,
|
||||
string bundleId,
|
||||
string bundleVersion,
|
||||
CancellationToken ct = default)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(rulesStream);
|
||||
|
||||
var rules = new List<SecretRule>();
|
||||
using var reader = new StreamReader(rulesStream, Encoding.UTF8, leaveOpen: true);
|
||||
|
||||
int lineNumber = 0;
|
||||
string? line;
|
||||
while ((line = await reader.ReadLineAsync(ct)) is not null)
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
lineNumber++;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var ruleJson = JsonSerializer.Deserialize<RuleJson>(line, JsonOptions);
|
||||
if (ruleJson is null)
|
||||
{
|
||||
_logger.LogWarning("Skipping null rule at line {LineNumber}", lineNumber);
|
||||
continue;
|
||||
}
|
||||
|
||||
var rule = MapToRule(ruleJson);
|
||||
rules.Add(rule);
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "Failed to parse rule at line {LineNumber}", lineNumber);
|
||||
}
|
||||
}
|
||||
|
||||
// Sort rules by ID for deterministic ordering
|
||||
rules.Sort((a, b) => string.Compare(a.Id, b.Id, StringComparison.Ordinal));
|
||||
|
||||
return new SecretRuleset
|
||||
{
|
||||
Id = bundleId,
|
||||
Version = bundleVersion,
|
||||
CreatedAt = _timeProvider.GetUtcNow(),
|
||||
Rules = [.. rules]
|
||||
};
|
||||
}
|
||||
|
||||
private static SecretRule MapToRule(RuleJson json)
|
||||
{
|
||||
return new SecretRule
|
||||
{
|
||||
Id = json.Id ?? throw new InvalidOperationException("Rule ID is required"),
|
||||
Version = json.Version ?? "1.0.0",
|
||||
Name = json.Name ?? json.Id ?? "Unknown",
|
||||
Description = json.Description ?? string.Empty,
|
||||
Type = ParseRuleType(json.Type),
|
||||
Pattern = json.Pattern ?? throw new InvalidOperationException("Rule pattern is required"),
|
||||
Severity = ParseSeverity(json.Severity),
|
||||
Confidence = ParseConfidence(json.Confidence),
|
||||
MaskingHint = json.MaskingHint,
|
||||
Keywords = json.Keywords?.ToImmutableArray() ?? [],
|
||||
FilePatterns = json.FilePatterns?.ToImmutableArray() ?? [],
|
||||
Enabled = json.Enabled ?? true,
|
||||
EntropyThreshold = json.EntropyThreshold ?? 4.5,
|
||||
MinLength = json.MinLength ?? 16,
|
||||
MaxLength = json.MaxLength ?? 1000,
|
||||
Metadata = json.Metadata?.ToImmutableDictionary() ?? ImmutableDictionary<string, string>.Empty
|
||||
};
|
||||
}
|
||||
|
||||
private static SecretRuleType ParseRuleType(string? type) => type?.ToLowerInvariant() switch
|
||||
{
|
||||
"regex" => SecretRuleType.Regex,
|
||||
"entropy" => SecretRuleType.Entropy,
|
||||
"composite" => SecretRuleType.Composite,
|
||||
_ => SecretRuleType.Regex
|
||||
};
|
||||
|
||||
private static SecretSeverity ParseSeverity(string? severity) => severity?.ToLowerInvariant() switch
|
||||
{
|
||||
"low" => SecretSeverity.Low,
|
||||
"medium" => SecretSeverity.Medium,
|
||||
"high" => SecretSeverity.High,
|
||||
"critical" => SecretSeverity.Critical,
|
||||
_ => SecretSeverity.Medium
|
||||
};
|
||||
|
||||
private static SecretConfidence ParseConfidence(string? confidence) => confidence?.ToLowerInvariant() switch
|
||||
{
|
||||
"low" => SecretConfidence.Low,
|
||||
"medium" => SecretConfidence.Medium,
|
||||
"high" => SecretConfidence.High,
|
||||
_ => SecretConfidence.Medium
|
||||
};
|
||||
|
||||
private static async Task<string> ComputeSha256Async(Stream stream, CancellationToken ct)
|
||||
{
|
||||
var hash = await SHA256.HashDataAsync(stream, ct);
|
||||
return Convert.ToHexString(hash).ToLowerInvariant();
|
||||
}
|
||||
|
||||
// JSON deserialization models
|
||||
private sealed class BundleManifest
|
||||
{
|
||||
public string? Id { get; set; }
|
||||
public string? Version { get; set; }
|
||||
public IntegrityInfo? Integrity { get; set; }
|
||||
}
|
||||
|
||||
private sealed class IntegrityInfo
|
||||
{
|
||||
public string? RulesSha256 { get; set; }
|
||||
}
|
||||
|
||||
private sealed class RuleJson
|
||||
{
|
||||
public string? Id { get; set; }
|
||||
public string? Version { get; set; }
|
||||
public string? Name { get; set; }
|
||||
public string? Description { get; set; }
|
||||
public string? Type { get; set; }
|
||||
public string? Pattern { get; set; }
|
||||
public string? Severity { get; set; }
|
||||
public string? Confidence { get; set; }
|
||||
public string? MaskingHint { get; set; }
|
||||
public List<string>? Keywords { get; set; }
|
||||
public List<string>? FilePatterns { get; set; }
|
||||
public bool? Enabled { get; set; }
|
||||
public double? EntropyThreshold { get; set; }
|
||||
public int? MinLength { get; set; }
|
||||
public int? MaxLength { get; set; }
|
||||
public Dictionary<string, string>? Metadata { get; set; }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Confidence level for a secret detection finding.
|
||||
/// </summary>
|
||||
public enum SecretConfidence
|
||||
{
|
||||
/// <summary>
|
||||
/// Low confidence - may be a false positive.
|
||||
/// </summary>
|
||||
Low = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Medium confidence - likely a real secret but requires verification.
|
||||
/// </summary>
|
||||
Medium = 1,
|
||||
|
||||
/// <summary>
|
||||
/// High confidence - almost certainly a real secret.
|
||||
/// </summary>
|
||||
High = 2
|
||||
}
|
||||
@@ -0,0 +1,191 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// A single secret detection rule defining patterns and metadata for identifying secrets.
|
||||
/// </summary>
|
||||
public sealed record SecretRule
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique rule identifier (e.g., "stellaops.secrets.aws-access-key").
|
||||
/// </summary>
|
||||
public required string Id { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Rule version in SemVer format (e.g., "1.0.0").
|
||||
/// </summary>
|
||||
public required string Version { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Human-readable rule name.
|
||||
/// </summary>
|
||||
public required string Name { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Detailed description of what this rule detects.
|
||||
/// </summary>
|
||||
public required string Description { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The detection strategy type.
|
||||
/// </summary>
|
||||
public required SecretRuleType Type { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The detection pattern (regex pattern for Regex type, entropy config for Entropy type).
|
||||
/// </summary>
|
||||
public required string Pattern { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Default severity for findings from this rule.
|
||||
/// </summary>
|
||||
public required SecretSeverity Severity { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Default confidence level for findings from this rule.
|
||||
/// </summary>
|
||||
public required SecretConfidence Confidence { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional masking hint (e.g., "prefix:4,suffix:2") for payload masking.
|
||||
/// </summary>
|
||||
public string? MaskingHint { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Pre-filter keywords for fast rejection of non-matching content.
|
||||
/// </summary>
|
||||
public ImmutableArray<string> Keywords { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Glob patterns for files this rule should be applied to.
|
||||
/// Empty means all text files.
|
||||
/// </summary>
|
||||
public ImmutableArray<string> FilePatterns { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Whether this rule is enabled.
|
||||
/// </summary>
|
||||
public bool Enabled { get; init; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum entropy threshold for entropy-based detection.
|
||||
/// Only used when Type is Entropy or Composite.
|
||||
/// </summary>
|
||||
public double EntropyThreshold { get; init; } = 4.5;
|
||||
|
||||
/// <summary>
|
||||
/// Minimum string length for entropy-based detection.
|
||||
/// </summary>
|
||||
public int MinLength { get; init; } = 16;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum string length for detection (prevents matching entire files).
|
||||
/// </summary>
|
||||
public int MaxLength { get; init; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Optional metadata for the rule.
|
||||
/// </summary>
|
||||
public ImmutableDictionary<string, string> Metadata { get; init; } =
|
||||
ImmutableDictionary<string, string>.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The compiled regex pattern, created lazily.
|
||||
/// </summary>
|
||||
private Regex? _compiledPattern;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the compiled regex for this rule. Returns null if the pattern is invalid.
|
||||
/// </summary>
|
||||
public Regex? GetCompiledPattern()
|
||||
{
|
||||
if (Type == SecretRuleType.Entropy)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
if (_compiledPattern is not null)
|
||||
{
|
||||
return _compiledPattern;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_compiledPattern = new Regex(
|
||||
Pattern,
|
||||
RegexOptions.Compiled | RegexOptions.CultureInvariant,
|
||||
TimeSpan.FromSeconds(5));
|
||||
return _compiledPattern;
|
||||
}
|
||||
catch (ArgumentException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if the content might match this rule based on keywords.
|
||||
/// Returns true if no keywords are defined or if any keyword is found.
|
||||
/// </summary>
|
||||
public bool MightMatch(ReadOnlySpan<char> content)
|
||||
{
|
||||
if (Keywords.IsDefaultOrEmpty)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
foreach (var keyword in Keywords)
|
||||
{
|
||||
if (content.Contains(keyword.AsSpan(), StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if this rule should be applied to the given file path.
|
||||
/// </summary>
|
||||
public bool AppliesToFile(string filePath)
|
||||
{
|
||||
if (FilePatterns.IsDefaultOrEmpty)
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
var fileName = Path.GetFileName(filePath);
|
||||
foreach (var pattern in FilePatterns)
|
||||
{
|
||||
if (MatchesGlob(fileName, pattern) || MatchesGlob(filePath, pattern))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool MatchesGlob(string path, string pattern)
|
||||
{
|
||||
// Simple glob matching for common patterns
|
||||
if (pattern.StartsWith("**", StringComparison.Ordinal))
|
||||
{
|
||||
var suffix = pattern[2..].TrimStart('/').TrimStart('\\');
|
||||
if (suffix.StartsWith("*.", StringComparison.Ordinal))
|
||||
{
|
||||
var extension = suffix[1..];
|
||||
return path.EndsWith(extension, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
return path.Contains(suffix, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
if (pattern.StartsWith("*.", StringComparison.Ordinal))
|
||||
{
|
||||
var extension = pattern[1..];
|
||||
return path.EndsWith(extension, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
return path.Equals(pattern, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// The type of detection strategy used by a secret rule.
|
||||
/// </summary>
|
||||
public enum SecretRuleType
|
||||
{
|
||||
/// <summary>
|
||||
/// Regex-based pattern matching.
|
||||
/// </summary>
|
||||
Regex = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Shannon entropy-based detection for high-entropy strings.
|
||||
/// </summary>
|
||||
Entropy = 1,
|
||||
|
||||
/// <summary>
|
||||
/// Combined regex and entropy detection.
|
||||
/// </summary>
|
||||
Composite = 2
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// A versioned collection of secret detection rules.
|
||||
/// </summary>
|
||||
public sealed record SecretRuleset
|
||||
{
|
||||
/// <summary>
|
||||
/// Bundle identifier (e.g., "secrets.ruleset").
|
||||
/// </summary>
|
||||
public required string Id { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Bundle version in YYYY.MM format (e.g., "2026.01").
|
||||
/// </summary>
|
||||
public required string Version { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// When this bundle was created.
|
||||
/// </summary>
|
||||
public required DateTimeOffset CreatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The rules in this bundle.
|
||||
/// </summary>
|
||||
public required ImmutableArray<SecretRule> Rules { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// SHA-256 digest of the rules file for integrity verification.
|
||||
/// </summary>
|
||||
public string? Sha256Digest { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Optional description of this bundle.
|
||||
/// </summary>
|
||||
public string? Description { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets only the enabled rules from this bundle.
|
||||
/// </summary>
|
||||
public IEnumerable<SecretRule> EnabledRules => Rules.Where(r => r.Enabled);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the count of enabled rules.
|
||||
/// </summary>
|
||||
public int EnabledRuleCount => Rules.Count(r => r.Enabled);
|
||||
|
||||
/// <summary>
|
||||
/// Creates an empty ruleset.
|
||||
/// </summary>
|
||||
public static SecretRuleset Empty { get; } = new()
|
||||
{
|
||||
Id = "empty",
|
||||
Version = "0.0",
|
||||
CreatedAt = DateTimeOffset.MinValue,
|
||||
Rules = []
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Validates that all rules in this bundle have valid patterns.
|
||||
/// </summary>
|
||||
/// <returns>A list of validation errors, empty if valid.</returns>
|
||||
public IReadOnlyList<string> Validate()
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Id))
|
||||
{
|
||||
errors.Add("Bundle ID is required");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(Version))
|
||||
{
|
||||
errors.Add("Bundle version is required");
|
||||
}
|
||||
|
||||
var seenIds = new HashSet<string>(StringComparer.Ordinal);
|
||||
foreach (var rule in Rules)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rule.Id))
|
||||
{
|
||||
errors.Add("Rule ID is required");
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!seenIds.Add(rule.Id))
|
||||
{
|
||||
errors.Add($"Duplicate rule ID: {rule.Id}");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(rule.Pattern))
|
||||
{
|
||||
errors.Add($"Rule '{rule.Id}' has no pattern");
|
||||
}
|
||||
|
||||
if (rule.Type is SecretRuleType.Regex or SecretRuleType.Composite)
|
||||
{
|
||||
if (rule.GetCompiledPattern() is null)
|
||||
{
|
||||
errors.Add($"Rule '{rule.Id}' has invalid regex pattern");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return errors;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets rules that apply to the specified file.
|
||||
/// </summary>
|
||||
public IEnumerable<SecretRule> GetRulesForFile(string filePath)
|
||||
{
|
||||
return EnabledRules.Where(r => r.AppliesToFile(filePath));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Severity level for a secret detection rule.
|
||||
/// </summary>
|
||||
public enum SecretSeverity
|
||||
{
|
||||
/// <summary>
|
||||
/// Low severity - informational or low-risk credentials.
|
||||
/// </summary>
|
||||
Low = 0,
|
||||
|
||||
/// <summary>
|
||||
/// Medium severity - credentials with limited scope or short lifespan.
|
||||
/// </summary>
|
||||
Medium = 1,
|
||||
|
||||
/// <summary>
|
||||
/// High severity - production credentials with broad access.
|
||||
/// </summary>
|
||||
High = 2,
|
||||
|
||||
/// <summary>
|
||||
/// Critical severity - highly privileged credentials requiring immediate action.
|
||||
/// </summary>
|
||||
Critical = 3
|
||||
}
|
||||
@@ -0,0 +1,234 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Analyzer that detects accidentally committed secrets in container layers.
|
||||
/// </summary>
|
||||
public sealed class SecretsAnalyzer : ILanguageAnalyzer
|
||||
{
|
||||
private readonly IOptions<SecretsAnalyzerOptions> _options;
|
||||
private readonly CompositeSecretDetector _detector;
|
||||
private readonly IPayloadMasker _masker;
|
||||
private readonly ILogger<SecretsAnalyzer> _logger;
|
||||
private readonly TimeProvider _timeProvider;
|
||||
|
||||
private SecretRuleset? _ruleset;
|
||||
|
||||
public SecretsAnalyzer(
|
||||
IOptions<SecretsAnalyzerOptions> options,
|
||||
CompositeSecretDetector detector,
|
||||
IPayloadMasker masker,
|
||||
ILogger<SecretsAnalyzer> logger,
|
||||
TimeProvider timeProvider)
|
||||
{
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_detector = detector ?? throw new ArgumentNullException(nameof(detector));
|
||||
_masker = masker ?? throw new ArgumentNullException(nameof(masker));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
_timeProvider = timeProvider ?? throw new ArgumentNullException(nameof(timeProvider));
|
||||
}
|
||||
|
||||
public string Id => "secrets";
|
||||
public string DisplayName => "Secret Leak Detector";
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether the analyzer is enabled and has a valid ruleset.
|
||||
/// </summary>
|
||||
public bool IsEnabled => _options.Value.Enabled && _ruleset is not null;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the currently loaded ruleset.
|
||||
/// </summary>
|
||||
public SecretRuleset? Ruleset => _ruleset;
|
||||
|
||||
/// <summary>
|
||||
/// Sets the ruleset to use for detection.
|
||||
/// Called by SecretsAnalyzerHost after loading the bundle.
|
||||
/// </summary>
|
||||
internal void SetRuleset(SecretRuleset ruleset)
|
||||
{
|
||||
_ruleset = ruleset ?? throw new ArgumentNullException(nameof(ruleset));
|
||||
}
|
||||
|
||||
public async ValueTask AnalyzeAsync(
|
||||
LanguageAnalyzerContext context,
|
||||
LanguageComponentWriter writer,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (!IsEnabled)
|
||||
{
|
||||
_logger.LogDebug("Secrets analyzer is disabled or has no ruleset");
|
||||
return;
|
||||
}
|
||||
|
||||
var options = _options.Value;
|
||||
var findings = new List<SecretLeakEvidence>();
|
||||
var filesScanned = 0;
|
||||
|
||||
// Scan all text files in the root
|
||||
foreach (var filePath in EnumerateTextFiles(context.RootPath, options))
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (findings.Count >= options.MaxFindingsPerScan)
|
||||
{
|
||||
_logger.LogWarning(
|
||||
"Maximum findings limit ({MaxFindings}) reached, stopping scan",
|
||||
options.MaxFindingsPerScan);
|
||||
break;
|
||||
}
|
||||
|
||||
var fileFindings = await ScanFileAsync(context, filePath, options, cancellationToken);
|
||||
findings.AddRange(fileFindings);
|
||||
filesScanned++;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"Secrets scan complete: {FileCount} files scanned, {FindingCount} findings",
|
||||
filesScanned,
|
||||
findings.Count);
|
||||
|
||||
// Store findings in analysis store if available
|
||||
if (context.AnalysisStore is not null && findings.Count > 0)
|
||||
{
|
||||
await StoreFindings(context.AnalysisStore, findings, cancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
private async ValueTask<List<SecretLeakEvidence>> ScanFileAsync(
|
||||
LanguageAnalyzerContext context,
|
||||
string filePath,
|
||||
SecretsAnalyzerOptions options,
|
||||
CancellationToken ct)
|
||||
{
|
||||
var findings = new List<SecretLeakEvidence>();
|
||||
|
||||
try
|
||||
{
|
||||
var fileInfo = new FileInfo(filePath);
|
||||
if (fileInfo.Length > options.MaxFileSizeBytes)
|
||||
{
|
||||
_logger.LogDebug("Skipping large file: {FilePath} ({Size} bytes)", filePath, fileInfo.Length);
|
||||
return findings;
|
||||
}
|
||||
|
||||
var content = await File.ReadAllBytesAsync(filePath, ct);
|
||||
var relativePath = context.GetRelativePath(filePath);
|
||||
|
||||
foreach (var rule in _ruleset!.GetRulesForFile(relativePath))
|
||||
{
|
||||
ct.ThrowIfCancellationRequested();
|
||||
|
||||
var matches = await _detector.DetectAsync(content, relativePath, rule, ct);
|
||||
|
||||
foreach (var match in matches)
|
||||
{
|
||||
// Check confidence threshold
|
||||
var confidence = MapScoreToConfidence(match.ConfidenceScore);
|
||||
if (confidence < options.MinConfidence)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var evidence = SecretLeakEvidence.FromMatch(match, _masker, _ruleset, _timeProvider);
|
||||
findings.Add(evidence);
|
||||
|
||||
_logger.LogDebug(
|
||||
"Found secret: Rule={RuleId}, File={FilePath}:{Line}, Mask={Mask}",
|
||||
rule.Id,
|
||||
relativePath,
|
||||
match.LineNumber,
|
||||
evidence.Mask);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
_logger.LogWarning(ex, "Error scanning file: {FilePath}", filePath);
|
||||
}
|
||||
|
||||
return findings;
|
||||
}
|
||||
|
||||
private static IEnumerable<string> EnumerateTextFiles(string rootPath, SecretsAnalyzerOptions options)
|
||||
{
|
||||
var searchOptions = new EnumerationOptions
|
||||
{
|
||||
RecurseSubdirectories = true,
|
||||
IgnoreInaccessible = true,
|
||||
AttributesToSkip = FileAttributes.System | FileAttributes.Hidden
|
||||
};
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(rootPath, "*", searchOptions))
|
||||
{
|
||||
var extension = Path.GetExtension(file).ToLowerInvariant();
|
||||
|
||||
// Check exclusions
|
||||
if (options.ExcludeExtensions.Contains(extension))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if directory is excluded
|
||||
var relativePath = Path.GetRelativePath(rootPath, file).Replace('\\', '/');
|
||||
if (IsExcludedDirectory(relativePath, options.ExcludeDirectories))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check inclusions if specified
|
||||
if (options.IncludeExtensions.Count > 0 && !options.IncludeExtensions.Contains(extension))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
yield return file;
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsExcludedDirectory(string relativePath, HashSet<string> patterns)
|
||||
{
|
||||
foreach (var pattern in patterns)
|
||||
{
|
||||
if (MatchesGlobPattern(relativePath, pattern))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private static bool MatchesGlobPattern(string path, string pattern)
|
||||
{
|
||||
if (pattern.StartsWith("**/", StringComparison.Ordinal))
|
||||
{
|
||||
var suffix = pattern[3..];
|
||||
if (suffix.EndsWith("/**", StringComparison.Ordinal))
|
||||
{
|
||||
var middle = suffix[..^3];
|
||||
return path.Contains(middle, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
return path.EndsWith(suffix, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
return path.StartsWith(pattern, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static SecretConfidence MapScoreToConfidence(double score) => score switch
|
||||
{
|
||||
>= 0.9 => SecretConfidence.High,
|
||||
>= 0.7 => SecretConfidence.Medium,
|
||||
_ => SecretConfidence.Low
|
||||
};
|
||||
|
||||
private async ValueTask StoreFindings(
|
||||
object analysisStore,
|
||||
List<SecretLeakEvidence> findings,
|
||||
CancellationToken ct)
|
||||
{
|
||||
// TODO: Store findings in ScanAnalysisStore when interface is defined
|
||||
// For now, just log that we would store them
|
||||
_logger.LogDebug("Would store {Count} secret findings in analysis store", findings.Count);
|
||||
await ValueTask.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
using Microsoft.Extensions.Hosting;
|
||||
using StellaOps.Scanner.Analyzers.Secrets.Bundles;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Hosted service that manages the lifecycle of the secrets analyzer.
|
||||
/// Loads and validates the rule bundle on startup with optional signature verification.
|
||||
/// Sprint: SPRINT_20260104_003_SCANNER - Secret Rule Bundles
|
||||
/// </summary>
|
||||
public sealed class SecretsAnalyzerHost : IHostedService
|
||||
{
|
||||
private readonly SecretsAnalyzer _analyzer;
|
||||
private readonly IRulesetLoader _rulesetLoader;
|
||||
private readonly IBundleVerifier? _bundleVerifier;
|
||||
private readonly IOptions<SecretsAnalyzerOptions> _options;
|
||||
private readonly ILogger<SecretsAnalyzerHost> _logger;
|
||||
|
||||
public SecretsAnalyzerHost(
|
||||
SecretsAnalyzer analyzer,
|
||||
IRulesetLoader rulesetLoader,
|
||||
IOptions<SecretsAnalyzerOptions> options,
|
||||
ILogger<SecretsAnalyzerHost> logger,
|
||||
IBundleVerifier? bundleVerifier = null)
|
||||
{
|
||||
_analyzer = analyzer ?? throw new ArgumentNullException(nameof(analyzer));
|
||||
_rulesetLoader = rulesetLoader ?? throw new ArgumentNullException(nameof(rulesetLoader));
|
||||
_bundleVerifier = bundleVerifier;
|
||||
_options = options ?? throw new ArgumentNullException(nameof(options));
|
||||
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gets the bundle verification result from the last startup, if available.
|
||||
/// </summary>
|
||||
public BundleVerificationResult? LastVerificationResult { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Gets whether the analyzer is enabled and has loaded successfully.
|
||||
/// </summary>
|
||||
public bool IsEnabled => _analyzer.IsEnabled;
|
||||
|
||||
/// <summary>
|
||||
/// Gets the loaded bundle version, if available.
|
||||
/// </summary>
|
||||
public string? BundleVersion => _analyzer.Ruleset?.Version;
|
||||
|
||||
public async Task StartAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
var options = _options.Value;
|
||||
|
||||
if (!options.Enabled)
|
||||
{
|
||||
_logger.LogInformation("SecretsAnalyzerHost: Secret leak detection is disabled");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation("SecretsAnalyzerHost: Loading secrets rule bundle from {Path}", options.RulesetPath);
|
||||
|
||||
try
|
||||
{
|
||||
// Verify bundle signature if required
|
||||
if (options.RequireSignatureVerification || _bundleVerifier is not null)
|
||||
{
|
||||
await VerifyBundleAsync(options, cancellationToken);
|
||||
}
|
||||
|
||||
var ruleset = await _rulesetLoader.LoadAsync(options.RulesetPath, cancellationToken);
|
||||
|
||||
// Validate the ruleset
|
||||
var errors = ruleset.Validate();
|
||||
if (errors.Count > 0)
|
||||
{
|
||||
_logger.LogError(
|
||||
"SecretsAnalyzerHost: Bundle validation failed with {ErrorCount} errors: {Errors}",
|
||||
errors.Count,
|
||||
string.Join(", ", errors));
|
||||
|
||||
if (options.FailOnInvalidBundle)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Secret detection bundle validation failed: {string.Join(", ", errors)}");
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
// Set the ruleset on the analyzer
|
||||
_analyzer.SetRuleset(ruleset);
|
||||
|
||||
_logger.LogInformation(
|
||||
"SecretsAnalyzerHost: Loaded bundle '{BundleId}' version {Version} with {RuleCount} rules ({EnabledCount} enabled)",
|
||||
ruleset.Id,
|
||||
ruleset.Version,
|
||||
ruleset.Rules.Length,
|
||||
ruleset.EnabledRuleCount);
|
||||
}
|
||||
catch (DirectoryNotFoundException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "SecretsAnalyzerHost: Bundle directory not found, analyzer disabled");
|
||||
|
||||
if (options.FailOnInvalidBundle)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
catch (FileNotFoundException ex)
|
||||
{
|
||||
_logger.LogWarning(ex, "SecretsAnalyzerHost: Bundle file not found, analyzer disabled");
|
||||
|
||||
if (options.FailOnInvalidBundle)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
catch (Exception ex) when (ex is not OperationCanceledException)
|
||||
{
|
||||
_logger.LogError(ex, "SecretsAnalyzerHost: Failed to load bundle");
|
||||
|
||||
if (options.FailOnInvalidBundle)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken)
|
||||
{
|
||||
_logger.LogInformation("SecretsAnalyzerHost: Shutting down");
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private async Task VerifyBundleAsync(SecretsAnalyzerOptions options, CancellationToken ct)
|
||||
{
|
||||
if (_bundleVerifier is null)
|
||||
{
|
||||
if (options.RequireSignatureVerification)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
"Signature verification is required but no IBundleVerifier is registered.");
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
var verificationOptions = new BundleVerificationOptions
|
||||
{
|
||||
RequireRekorProof = options.RequireRekorProof,
|
||||
TrustedKeyIds = options.TrustedKeyIds.Count > 0 ? [.. options.TrustedKeyIds] : null,
|
||||
SharedSecret = options.SignatureSecret,
|
||||
SharedSecretFile = options.SignatureSecretFile,
|
||||
VerifyIntegrity = true,
|
||||
SkipSignatureVerification = !options.RequireSignatureVerification
|
||||
};
|
||||
|
||||
_logger.LogDebug("SecretsAnalyzerHost: Verifying bundle signature");
|
||||
|
||||
var result = await _bundleVerifier.VerifyAsync(
|
||||
options.RulesetPath,
|
||||
verificationOptions,
|
||||
ct);
|
||||
|
||||
LastVerificationResult = result;
|
||||
|
||||
if (!result.IsValid)
|
||||
{
|
||||
var errorMessage = $"Bundle verification failed: {string.Join("; ", result.ValidationErrors)}";
|
||||
_logger.LogError("SecretsAnalyzerHost: {Error}", errorMessage);
|
||||
|
||||
if (options.FailOnInvalidBundle)
|
||||
{
|
||||
throw new InvalidOperationException(errorMessage);
|
||||
}
|
||||
|
||||
// Allow loading but log prominently
|
||||
_logger.LogWarning(
|
||||
"SecretsAnalyzerHost: Continuing with unverified bundle. " +
|
||||
"Set RequireSignatureVerification=true to enforce verification.");
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogInformation(
|
||||
"SecretsAnalyzerHost: Bundle verified - signed by {KeyId} at {SignedAt}",
|
||||
result.SignerKeyId ?? "unknown",
|
||||
result.SignedAt?.ToString("o") ?? "unknown");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,118 @@
|
||||
using System.ComponentModel.DataAnnotations;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Configuration options for the secrets analyzer.
|
||||
/// </summary>
|
||||
public sealed class SecretsAnalyzerOptions
|
||||
{
|
||||
/// <summary>
|
||||
/// Configuration section name.
|
||||
/// </summary>
|
||||
public const string SectionName = "Scanner:Analyzers:Secrets";
|
||||
|
||||
/// <summary>
|
||||
/// Enable secret leak detection (experimental feature).
|
||||
/// Default: false (opt-in).
|
||||
/// </summary>
|
||||
public bool Enabled { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Path to the ruleset bundle directory.
|
||||
/// </summary>
|
||||
[Required]
|
||||
public string RulesetPath { get; set; } = "/opt/stellaops/plugins/scanner/analyzers/secrets";
|
||||
|
||||
/// <summary>
|
||||
/// Minimum confidence level to report findings.
|
||||
/// </summary>
|
||||
public SecretConfidence MinConfidence { get; set; } = SecretConfidence.Low;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum findings per scan (circuit breaker).
|
||||
/// </summary>
|
||||
[Range(1, 10000)]
|
||||
public int MaxFindingsPerScan { get; set; } = 1000;
|
||||
|
||||
/// <summary>
|
||||
/// Maximum file size to scan in bytes.
|
||||
/// Files larger than this are skipped.
|
||||
/// </summary>
|
||||
[Range(1, 100 * 1024 * 1024)]
|
||||
public long MaxFileSizeBytes { get; set; } = 10 * 1024 * 1024; // 10MB
|
||||
|
||||
/// <summary>
|
||||
/// Enable entropy-based detection.
|
||||
/// </summary>
|
||||
public bool EnableEntropyDetection { get; set; } = true;
|
||||
|
||||
/// <summary>
|
||||
/// Default entropy threshold (bits per character).
|
||||
/// </summary>
|
||||
[Range(3.0, 8.0)]
|
||||
public double EntropyThreshold { get; set; } = 4.5;
|
||||
|
||||
/// <summary>
|
||||
/// File extensions to scan. Empty means all text files.
|
||||
/// </summary>
|
||||
public HashSet<string> IncludeExtensions { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// File extensions to exclude from scanning.
|
||||
/// </summary>
|
||||
public HashSet<string> ExcludeExtensions { get; set; } =
|
||||
[
|
||||
".png", ".jpg", ".jpeg", ".gif", ".ico", ".webp",
|
||||
".zip", ".tar", ".gz", ".bz2", ".xz",
|
||||
".exe", ".dll", ".so", ".dylib",
|
||||
".pdf", ".doc", ".docx", ".xls", ".xlsx",
|
||||
".mp3", ".mp4", ".avi", ".mov", ".mkv",
|
||||
".ttf", ".woff", ".woff2", ".eot",
|
||||
".min.js", ".min.css"
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Directories to exclude from scanning (glob patterns).
|
||||
/// </summary>
|
||||
public HashSet<string> ExcludeDirectories { get; set; } =
|
||||
[
|
||||
"**/node_modules/**",
|
||||
"**/.git/**",
|
||||
"**/vendor/**",
|
||||
"**/__pycache__/**",
|
||||
"**/bin/**",
|
||||
"**/obj/**"
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Whether to fail the scan if the bundle cannot be loaded.
|
||||
/// </summary>
|
||||
public bool FailOnInvalidBundle { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Whether to require DSSE signature verification for bundles.
|
||||
/// </summary>
|
||||
public bool RequireSignatureVerification { get; set; } = false;
|
||||
|
||||
/// <summary>
|
||||
/// Shared secret for HMAC signature verification (base64 or hex).
|
||||
/// </summary>
|
||||
public string? SignatureSecret { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Path to file containing the shared secret.
|
||||
/// </summary>
|
||||
public string? SignatureSecretFile { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// List of trusted key IDs for signature verification.
|
||||
/// If empty, any key is accepted.
|
||||
/// </summary>
|
||||
public HashSet<string> TrustedKeyIds { get; set; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Whether to require Rekor transparency log proof.
|
||||
/// </summary>
|
||||
public bool RequireRekorProof { get; set; } = false;
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
using StellaOps.Scanner.Analyzers.Secrets.Bundles;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Secrets;
|
||||
|
||||
/// <summary>
|
||||
/// Extension methods for registering secrets analyzer services.
|
||||
/// </summary>
|
||||
public static class ServiceCollectionExtensions
|
||||
{
|
||||
/// <summary>
|
||||
/// Adds the secrets analyzer services to the service collection.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <param name="configuration">The configuration.</param>
|
||||
/// <returns>The service collection for chaining.</returns>
|
||||
public static IServiceCollection AddSecretsAnalyzer(
|
||||
this IServiceCollection services,
|
||||
IConfiguration configuration)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configuration);
|
||||
|
||||
// Register options
|
||||
services.AddOptions<SecretsAnalyzerOptions>()
|
||||
.Bind(configuration.GetSection(SecretsAnalyzerOptions.SectionName))
|
||||
.ValidateDataAnnotations()
|
||||
.ValidateOnStart();
|
||||
|
||||
// Register TimeProvider if not already registered
|
||||
services.TryAddSingleton(TimeProvider.System);
|
||||
|
||||
RegisterCoreServices(services);
|
||||
return services;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds the secrets analyzer services with custom options.
|
||||
/// </summary>
|
||||
/// <param name="services">The service collection.</param>
|
||||
/// <param name="configureOptions">Action to configure options.</param>
|
||||
/// <returns>The service collection for chaining.</returns>
|
||||
public static IServiceCollection AddSecretsAnalyzer(
|
||||
this IServiceCollection services,
|
||||
Action<SecretsAnalyzerOptions> configureOptions)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(services);
|
||||
ArgumentNullException.ThrowIfNull(configureOptions);
|
||||
|
||||
// Register options
|
||||
services.AddOptions<SecretsAnalyzerOptions>()
|
||||
.Configure(configureOptions)
|
||||
.ValidateDataAnnotations()
|
||||
.ValidateOnStart();
|
||||
|
||||
RegisterCoreServices(services);
|
||||
return services;
|
||||
}
|
||||
|
||||
private static void RegisterCoreServices(IServiceCollection services)
|
||||
{
|
||||
// Register TimeProvider if not already registered
|
||||
services.TryAddSingleton(TimeProvider.System);
|
||||
|
||||
// Register core services
|
||||
services.AddSingleton<IPayloadMasker, PayloadMasker>();
|
||||
services.AddSingleton<IRulesetLoader, RulesetLoader>();
|
||||
|
||||
// Register detectors
|
||||
services.AddSingleton<RegexDetector>();
|
||||
services.AddSingleton<EntropyDetector>();
|
||||
services.AddSingleton<CompositeSecretDetector>();
|
||||
|
||||
// Register bundle infrastructure (Sprint: SPRINT_20260104_003_SCANNER)
|
||||
services.AddSingleton<IRuleValidator, RuleValidator>();
|
||||
services.AddSingleton<IBundleBuilder, BundleBuilder>();
|
||||
services.AddSingleton<IBundleSigner, BundleSigner>();
|
||||
services.AddSingleton<IBundleVerifier, BundleVerifier>();
|
||||
|
||||
// Register analyzer
|
||||
services.AddSingleton<SecretsAnalyzer>();
|
||||
services.AddSingleton<ILanguageAnalyzer>(sp => sp.GetRequiredService<SecretsAnalyzer>());
|
||||
|
||||
// Register hosted service
|
||||
services.AddSingleton<SecretsAnalyzerHost>();
|
||||
services.AddHostedService(sp => sp.GetRequiredService<SecretsAnalyzerHost>());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
<?xml version='1.0' encoding='utf-8'?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<EnableDefaultItems>false</EnableDefaultItems>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Include="**\*.cs" Exclude="obj\**;bin\**" />
|
||||
<EmbeddedResource Include="**\*.json" Exclude="obj\**;bin\**" />
|
||||
<None Include="**\*" Exclude="**\*.cs;**\*.json;bin\**;obj\**" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../StellaOps.Scanner.Core/StellaOps.Scanner.Core.csproj" />
|
||||
<ProjectReference Include="../StellaOps.Scanner.Analyzers.Lang/StellaOps.Scanner.Analyzers.Lang.csproj" />
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.Evidence.Core/StellaOps.Evidence.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.ConfigurationExtensions" />
|
||||
<PackageReference Include="Microsoft.Extensions.Options.DataAnnotations" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -6,6 +6,7 @@
|
||||
|
||||
using Microsoft.Extensions.Configuration;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Scanner.CallGraph.Binary;
|
||||
using StellaOps.Scanner.CallGraph.Caching;
|
||||
using StellaOps.Scanner.CallGraph.DotNet;
|
||||
using StellaOps.Scanner.CallGraph.Go;
|
||||
@@ -40,6 +41,7 @@ public static class CallGraphServiceCollectionExtensions
|
||||
services.AddSingleton<ICallGraphExtractor, NodeCallGraphExtractor>(); // Node.js/JavaScript via Babel
|
||||
services.AddSingleton<ICallGraphExtractor, PythonCallGraphExtractor>(); // Python via AST analysis
|
||||
services.AddSingleton<ICallGraphExtractor, GoCallGraphExtractor>(); // Go via SSA analysis
|
||||
services.AddSingleton<ICallGraphExtractor, BinaryCallGraphExtractor>(); // Native ELF/PE/Mach-O binaries
|
||||
|
||||
// Register the extractor registry for language-based lookup
|
||||
services.AddSingleton<ICallGraphExtractorRegistry, CallGraphExtractorRegistry>();
|
||||
|
||||
@@ -50,4 +50,8 @@ public static class ScanAnalysisKeys
|
||||
public const string VulnerabilityMatches = "analysis.poe.vulnerability.matches";
|
||||
public const string PoEResults = "analysis.poe.results";
|
||||
public const string PoEConfiguration = "analysis.poe.configuration";
|
||||
|
||||
// Sprint: SPRINT_20251229_046_BE - Secrets Leak Detection
|
||||
public const string SecretFindings = "analysis.secrets.findings";
|
||||
public const string SecretRulesetVersion = "analysis.secrets.ruleset.version";
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user