save checkpoint: save features

This commit is contained in:
master
2026-02-12 10:27:23 +02:00
parent dca86e1248
commit 5bca406787
8837 changed files with 1796879 additions and 5294 deletions

View File

@@ -4,5 +4,6 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
| Task ID | Status | Notes |
| --- | --- | --- |
| QA-SCANNER-VERIFY-004 | DONE | SPRINT_20260212_002 run-001: `ai-ml-supply-chain-security-analysis-module` passed Tier 0/1/2; feature moved to `docs/features/checked/scanner/`. |
| REMED-05 | TODO | Remediation checklist: docs/implplan/audits/csproj-standards/remediation/checklists/src/Scanner/__Libraries/StellaOps.Scanner.AiMlSecurity/StellaOps.Scanner.AiMlSecurity.md. |
| REMED-06 | DONE | SOLID review notes captured for SPRINT_20260130_002. |

View File

@@ -9,3 +9,6 @@ Source of truth: `docs-archived/implplan/2025-12-29-csproj-audit/SPRINT_20251229
| AUDIT-0765-T | DONE | Revalidated 2026-01-07. |
| AUDIT-0765-A | DONE | Already compliant (revalidated 2026-01-07). |
| REMED-06 | DONE | SOLID review notes captured for SPRINT_20260130_002. |
| QA-SCANNER-VERIFY-002 | BLOCKED | SPRINT_20260212_002 run-001: verify `secret-detection-and-credential-leak-guard` across Tier 0/1/2 evidence and terminalize dossier state. |

View File

@@ -0,0 +1,108 @@
{
"runtimeTarget": {
"name": ".NETCoreApp,Version=v10.0",
"signature": ""
},
"compilationOptions": {},
"targets": {
".NETCoreApp,Version=v10.0": {
"StellaOps.Scanner.ChangeTrace/1.0.0": {
"dependencies": {
"Microsoft.Extensions.Logging.Abstractions": "10.0.1",
"Microsoft.Extensions.Options": "10.0.1",
"StellaOps.Canonical.Json": "1.0.0"
},
"runtime": {
"StellaOps.Scanner.ChangeTrace.dll": {}
}
},
"Microsoft.Extensions.DependencyInjection.Abstractions/10.0.1": {
"runtime": {
"lib/net10.0/Microsoft.Extensions.DependencyInjection.Abstractions.dll": {
"assemblyVersion": "10.0.0.0",
"fileVersion": "10.0.125.57005"
}
}
},
"Microsoft.Extensions.Logging.Abstractions/10.0.1": {
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.1"
},
"runtime": {
"lib/net10.0/Microsoft.Extensions.Logging.Abstractions.dll": {
"assemblyVersion": "10.0.0.0",
"fileVersion": "10.0.125.57005"
}
}
},
"Microsoft.Extensions.Options/10.0.1": {
"dependencies": {
"Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.1",
"Microsoft.Extensions.Primitives": "10.0.1"
},
"runtime": {
"lib/net10.0/Microsoft.Extensions.Options.dll": {
"assemblyVersion": "10.0.0.0",
"fileVersion": "10.0.125.57005"
}
}
},
"Microsoft.Extensions.Primitives/10.0.1": {
"runtime": {
"lib/net10.0/Microsoft.Extensions.Primitives.dll": {
"assemblyVersion": "10.0.0.0",
"fileVersion": "10.0.125.57005"
}
}
},
"StellaOps.Canonical.Json/1.0.0": {
"runtime": {
"StellaOps.Canonical.Json.dll": {
"assemblyVersion": "1.0.0.0",
"fileVersion": "1.0.0.0"
}
}
}
}
},
"libraries": {
"StellaOps.Scanner.ChangeTrace/1.0.0": {
"type": "project",
"serviceable": false,
"sha512": ""
},
"Microsoft.Extensions.DependencyInjection.Abstractions/10.0.1": {
"type": "package",
"serviceable": true,
"sha512": "sha512-oIy8fQxxbUsSrrOvgBqlVgOeCtDmrcynnTG+FQufcUWBrwyPfwlUkCDB2vaiBeYPyT+20u9/HeuHeBf+H4F/8g==",
"path": "microsoft.extensions.dependencyinjection.abstractions/10.0.1",
"hashPath": "microsoft.extensions.dependencyinjection.abstractions.10.0.1.nupkg.sha512"
},
"Microsoft.Extensions.Logging.Abstractions/10.0.1": {
"type": "package",
"serviceable": true,
"sha512": "sha512-YkmyiPIWAXVb+lPIrM0LE5bbtLOJkCiRTFiHpkVOvhI7uTvCfoOHLEN0LcsY56GpSD7NqX3gJNpsaDe87/B3zg==",
"path": "microsoft.extensions.logging.abstractions/10.0.1",
"hashPath": "microsoft.extensions.logging.abstractions.10.0.1.nupkg.sha512"
},
"Microsoft.Extensions.Options/10.0.1": {
"type": "package",
"serviceable": true,
"sha512": "sha512-G6VVwywpJI4XIobetGHwg7wDOYC2L2XBYdtskxLaKF/Ynb5QBwLl7Q//wxAR2aVCLkMpoQrjSP9VoORkyddsNQ==",
"path": "microsoft.extensions.options/10.0.1",
"hashPath": "microsoft.extensions.options.10.0.1.nupkg.sha512"
},
"Microsoft.Extensions.Primitives/10.0.1": {
"type": "package",
"serviceable": true,
"sha512": "sha512-DO8XrJkp5x4PddDuc/CH37yDBCs9BYN6ijlKyR3vMb55BP1Vwh90vOX8bNfnKxr5B2qEI3D8bvbY1fFbDveDHQ==",
"path": "microsoft.extensions.primitives/10.0.1",
"hashPath": "microsoft.extensions.primitives.10.0.1.nupkg.sha512"
},
"StellaOps.Canonical.Json/1.0.0": {
"type": "project",
"serviceable": false,
"sha512": ""
}
}
}

View File

@@ -5,9 +5,8 @@
// Description: Service for matching secret findings against exception patterns.
// -----------------------------------------------------------------------------
using Microsoft.Extensions.FileSystemGlobbing;
using System.Text.RegularExpressions;
using System.Text;
namespace StellaOps.Scanner.Core.Secrets.Configuration;
@@ -115,17 +114,14 @@ public sealed class SecretExceptionMatcher
{
try
{
var matcher = new Matcher();
matcher.AddInclude(globPattern);
// Normalize path separators to forward slashes
var normalizedPath = filePath.Replace('\\', '/').TrimStart('/');
// For patterns like **/test/**, we need to match against the path
// Matcher.Match needs both a directory base and files, but we can
// work around this by matching the path itself
var result = matcher.Match(normalizedPath);
return result.HasMatches;
var normalizedPattern = globPattern.Replace('\\', '/').TrimStart('/');
var regexPattern = ConvertGlobToRegex(normalizedPattern);
return Regex.IsMatch(
normalizedPath,
regexPattern,
RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
}
catch
{
@@ -134,6 +130,52 @@ public sealed class SecretExceptionMatcher
}
}
private static string ConvertGlobToRegex(string pattern)
{
var sb = new StringBuilder(pattern.Length * 2);
sb.Append('^');
for (var i = 0; i < pattern.Length; i++)
{
var current = pattern[i];
var next = i + 1 < pattern.Length ? pattern[i + 1] : '\0';
if (current == '*' && next == '*')
{
var following = i + 2 < pattern.Length ? pattern[i + 2] : '\0';
if (following == '/')
{
sb.Append("(?:.*/)?");
i += 2;
continue;
}
sb.Append(".*");
i++;
continue;
}
switch (current)
{
case '*':
sb.Append("[^/]*");
break;
case '?':
sb.Append("[^/]");
break;
case '/':
sb.Append('/');
break;
default:
sb.Append(Regex.Escape(current.ToString()));
break;
}
}
sb.Append('$');
return sb.ToString();
}
private sealed record CompiledExceptionPattern(
SecretExceptionPattern Pattern,
Regex ValueRegex);

View File

@@ -0,0 +1,17 @@
# AGENTS - Scanner Manifest Library
## Scope
- Owns OCI manifest snapshot, layer diffID resolution, base-image attribution, and layer-reuse logic.
- Keep behavior deterministic and offline-friendly.
## Required Reading
- `src/Scanner/AGENTS.md`
- `docs/modules/scanner/architecture.md`
- `docs/code-of-conduct/CODE_OF_CONDUCT.md`
- `docs/code-of-conduct/TESTING_PRACTICES.md`
## Working Agreement
1. Keep base-image detection deterministic (stable ordering, no random tie-breakers).
2. Do not add external network dependencies beyond existing registry clients.
3. When changing contracts or matching semantics, add/adjust focused behavioral tests.
4. Keep logs structured and free of sensitive payload data.

View File

@@ -1,7 +1,5 @@
using Microsoft.Extensions.Logging;
using Npgsql;
using NpgsqlTypes;
using System.Collections.Concurrent;
using System.Collections.Immutable;
@@ -41,15 +39,25 @@ public sealed class BaseImageDetector : IBaseImageDetector
if (_diffIdIndex.TryGetValue(diffId, out var matches))
{
// Prefer exact layer index match
var exactMatch = matches.FirstOrDefault(m => m.LayerIndex == layerIndex);
(string BaseImage, int LayerIndex)[] snapshot;
lock (matches)
{
snapshot = [.. matches];
}
// Prefer exact layer index match.
var exactMatch = snapshot.FirstOrDefault(m => m.LayerIndex == layerIndex);
if (!string.IsNullOrEmpty(exactMatch.BaseImage))
{
return exactMatch.BaseImage;
}
// Return any matching base image
return matches.FirstOrDefault().BaseImage;
// Fuzzy fallback: closest index first, then lexical for deterministic tie-break.
return snapshot
.OrderBy(m => Math.Abs(m.LayerIndex - layerIndex))
.ThenBy(m => m.BaseImage, StringComparer.OrdinalIgnoreCase)
.Select(m => m.BaseImage)
.FirstOrDefault();
}
return null;
@@ -139,9 +147,12 @@ public sealed class BaseImageDetector : IBaseImageDetector
_diffIdIndex[diffId] = list;
}
// Remove existing entry for this base image and add new one
list.RemoveAll(e => e.BaseImage.Equals(baseImageRef, StringComparison.OrdinalIgnoreCase));
list.Add((baseImageRef, i));
// Remove existing entry for this base image and add new one.
lock (list)
{
list.RemoveAll(e => e.BaseImage.Equals(baseImageRef, StringComparison.OrdinalIgnoreCase));
list.Add((baseImageRef, i));
}
}
_logger.LogInformation("Registered base image {BaseImage}", baseImageRef);
@@ -160,6 +171,38 @@ public sealed class BaseImageDetector : IBaseImageDetector
return [.. _knownBaseImages.Keys];
}
public async Task<IReadOnlyList<BaseImageMatch>> GetRecommendationsAsync(
IEnumerable<string> layerDiffIds,
int maxRecommendations = 3,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(layerDiffIds);
await EnsureIndexLoadedAsync(cancellationToken).ConfigureAwait(false);
return BaseImageMatchEngine.RankMatches(layerDiffIds, BuildKnownBaseImageInfos(), maxRecommendations);
}
public async Task<IReadOnlyDictionary<string, IReadOnlyList<BaseImageMatch>>> GetRecommendationsBulkAsync(
IReadOnlyDictionary<string, IReadOnlyList<string>> imagesByReference,
int maxRecommendations = 3,
CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(imagesByReference);
await EnsureIndexLoadedAsync(cancellationToken).ConfigureAwait(false);
var known = BuildKnownBaseImageInfos();
var results = new SortedDictionary<string, IReadOnlyList<BaseImageMatch>>(StringComparer.OrdinalIgnoreCase);
foreach (var (imageReference, layers) in imagesByReference
.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase))
{
results[imageReference] = BaseImageMatchEngine.RankMatches(layers, known, maxRecommendations);
}
return results;
}
private async Task EnsureIndexLoadedAsync(CancellationToken cancellationToken)
{
if (_indexLoaded)
@@ -228,7 +271,10 @@ public sealed class BaseImageDetector : IBaseImageDetector
_diffIdIndex[diffId] = list;
}
list.Add((imageRef, layerIndex));
lock (list)
{
list.Add((imageRef, layerIndex));
}
}
// Don't forget the last image
@@ -255,4 +301,17 @@ public sealed class BaseImageDetector : IBaseImageDetector
// This is a placeholder to show the pattern.
_logger.LogDebug("Built-in base image index ready for population");
}
private IReadOnlyList<BaseImageInfo> BuildKnownBaseImageInfos()
{
return [.. _knownBaseImages
.OrderBy(static kvp => kvp.Key, StringComparer.OrdinalIgnoreCase)
.Select(static kvp => new BaseImageInfo
{
ImageReference = kvp.Key,
LayerDiffIds = [.. kvp.Value],
RegisteredAt = DateTimeOffset.MinValue,
DetectionCount = 0
})];
}
}

View File

@@ -0,0 +1,143 @@
using System.Collections.Immutable;
using System.Globalization;
namespace StellaOps.Scanner.Manifest.Resolution;
/// <summary>
/// Deterministic base-image match scoring for exact and fuzzy recommendations.
/// </summary>
public static class BaseImageMatchEngine
{
/// <summary>
/// Ranks known base-image fingerprints against the candidate image layers.
/// </summary>
public static IReadOnlyList<BaseImageMatch> RankMatches(
IEnumerable<string> candidateLayerDiffIds,
IEnumerable<BaseImageInfo> knownBaseImages,
int maxRecommendations = 3)
{
ArgumentNullException.ThrowIfNull(candidateLayerDiffIds);
ArgumentNullException.ThrowIfNull(knownBaseImages);
if (maxRecommendations <= 0)
{
return [];
}
var candidate = Normalize(candidateLayerDiffIds);
if (candidate.Length == 0)
{
return [];
}
var candidateLayerSet = new HashSet<string>(candidate, StringComparer.OrdinalIgnoreCase);
var matches = new List<BaseImageMatch>();
foreach (var baseImage in knownBaseImages.OrderBy(static x => x.ImageReference, StringComparer.OrdinalIgnoreCase))
{
var baseLayers = Normalize(baseImage.LayerDiffIds);
if (baseLayers.Length == 0)
{
continue;
}
var prefixMatches = CountPrefixMatches(candidate, baseLayers);
var overlap = baseLayers.Count(candidateLayerSet.Contains);
if (overlap == 0)
{
continue;
}
var exact = prefixMatches == baseLayers.Length && baseLayers.Length <= candidate.Length;
var confidence = exact
? 1.0
: ComputeFuzzyConfidence(prefixMatches, overlap, candidate.Length, baseLayers.Length);
// Keep only meaningful fuzzy recommendations.
if (!exact && confidence < 0.55)
{
continue;
}
var rationale = exact
? "Exact ordered prefix match for base-image layers."
: FormattableString.Invariant(
$"Fuzzy overlap: prefix={prefixMatches}/{baseLayers.Length}, overlap={overlap}/{baseLayers.Length}.");
matches.Add(new BaseImageMatch
{
ImageReference = baseImage.ImageReference,
MatchType = exact ? BaseImageMatchType.Exact : BaseImageMatchType.Fuzzy,
Confidence = exact ? 1.0 : Math.Round(confidence, 4),
MatchedLayerCount = overlap,
CandidateLayerCount = candidate.Length,
BaseLayerCount = baseLayers.Length,
Rationale = rationale
});
}
return [.. matches
.OrderByDescending(static m => m.Confidence)
.ThenByDescending(static m => m.MatchedLayerCount)
.ThenBy(static m => m.ImageReference, StringComparer.OrdinalIgnoreCase)
.Take(maxRecommendations)];
}
private static double ComputeFuzzyConfidence(
int prefixMatches,
int overlap,
int candidateCount,
int baseCount)
{
var prefixCoverage = baseCount == 0 ? 0 : (double)prefixMatches / baseCount;
var baseCoverage = baseCount == 0 ? 0 : (double)overlap / baseCount;
var candidateCoverage = candidateCount == 0 ? 0 : (double)overlap / candidateCount;
var sizePenalty = 0.0;
var maxCount = Math.Max(candidateCount, baseCount);
if (maxCount > 0)
{
sizePenalty = Math.Abs(candidateCount - baseCount) / (double)maxCount;
}
var score = (0.55 * prefixCoverage) + (0.30 * baseCoverage) + (0.15 * candidateCoverage) - (0.10 * sizePenalty);
return Math.Clamp(score, 0.0, 0.99);
}
private static int CountPrefixMatches(IReadOnlyList<string> candidate, IReadOnlyList<string> baseLayers)
{
var max = Math.Min(candidate.Count, baseLayers.Count);
var matches = 0;
for (var i = 0; i < max; i++)
{
if (!string.Equals(candidate[i], baseLayers[i], StringComparison.OrdinalIgnoreCase))
{
break;
}
matches++;
}
return matches;
}
private static ImmutableArray<string> Normalize(IEnumerable<string> layerDiffIds)
{
return [.. layerDiffIds
.Where(static x => !string.IsNullOrWhiteSpace(x))
.Select(static x => NormalizeDiffId(x))
.Distinct(StringComparer.OrdinalIgnoreCase)];
}
private static string NormalizeDiffId(string diffId)
{
var value = diffId.Trim();
if (value.StartsWith("sha256:", StringComparison.OrdinalIgnoreCase))
{
return "sha256:" + value["sha256:".Length..].ToLower(CultureInfo.InvariantCulture);
}
return value.ToLower(CultureInfo.InvariantCulture);
}
}

View File

@@ -39,6 +39,32 @@ public interface IBaseImageDetector
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>List of registered base image references.</returns>
Task<IReadOnlyList<string>> GetRegisteredBaseImagesAsync(CancellationToken cancellationToken = default);
/// <summary>
/// Produces ranked base-image recommendations for ordered layer diffIDs.
/// </summary>
/// <param name="layerDiffIds">Ordered layer diffIDs for the image being evaluated.</param>
/// <param name="maxRecommendations">Maximum number of recommendations to return.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Ranked recommendations ordered by confidence.</returns>
Task<IReadOnlyList<BaseImageMatch>> GetRecommendationsAsync(
IEnumerable<string> layerDiffIds,
int maxRecommendations = 3,
CancellationToken cancellationToken = default);
/// <summary>
/// Produces ranked base-image recommendations for multiple images in one call.
/// </summary>
/// <param name="imagesByReference">
/// Image references mapped to ordered layer diffIDs for each image.
/// </param>
/// <param name="maxRecommendations">Maximum recommendations per image.</param>
/// <param name="cancellationToken">Cancellation token.</param>
/// <returns>Recommendation sets per image reference.</returns>
Task<IReadOnlyDictionary<string, IReadOnlyList<BaseImageMatch>>> GetRecommendationsBulkAsync(
IReadOnlyDictionary<string, IReadOnlyList<string>> imagesByReference,
int maxRecommendations = 3,
CancellationToken cancellationToken = default);
}
/// <summary>
@@ -66,3 +92,53 @@ public sealed record BaseImageInfo
/// </summary>
public long DetectionCount { get; init; }
}
/// <summary>
/// Match semantics for a base-image recommendation.
/// </summary>
public enum BaseImageMatchType
{
Exact = 0,
Fuzzy = 1
}
/// <summary>
/// Ranked base-image recommendation for a candidate image.
/// </summary>
public sealed record BaseImageMatch
{
/// <summary>
/// Base image reference (e.g., "alpine:3.19").
/// </summary>
public required string ImageReference { get; init; }
/// <summary>
/// Exact or fuzzy match type.
/// </summary>
public BaseImageMatchType MatchType { get; init; }
/// <summary>
/// Confidence score in [0,1].
/// </summary>
public double Confidence { get; init; }
/// <summary>
/// Number of overlapping layers between candidate and base image.
/// </summary>
public int MatchedLayerCount { get; init; }
/// <summary>
/// Number of layers in the candidate image.
/// </summary>
public int CandidateLayerCount { get; init; }
/// <summary>
/// Number of layers in the matched base image fingerprint.
/// </summary>
public int BaseLayerCount { get; init; }
/// <summary>
/// Deterministic rationale for why this recommendation was returned.
/// </summary>
public required string Rationale { get; init; }
}

View File

@@ -4,5 +4,9 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
| Task ID | Status | Notes |
| --- | --- | --- |
| QA-SCANNER-VERIFY-005 | DONE | SPRINT_20260212_002 run-001: `api-gateway-boundary-extractor` passed Tier 0/1/2 and moved to `docs/features/checked/scanner/`. |
| REMED-05 | TODO | Remediation checklist: docs/implplan/audits/csproj-standards/remediation/checklists/src/Scanner/__Libraries/StellaOps.Scanner.Reachability/StellaOps.Scanner.Reachability.md. |
| REMED-06 | DONE | SOLID review notes captured for SPRINT_20260130_002. |
| QA-SCANNER-VERIFY-001 | BLOCKED | SPRINT_20260212_002 run-001: blocked verification for `3-bit-reachability-gate` due missing AGENTS in required Scanner reachability/smart-diff test modules. |

View File

@@ -6,3 +6,6 @@ Source of truth: `docs/implplan/SPRINT_20260130_002_Tools_csproj_remediation_sol
| --- | --- | --- |
| REMED-05 | TODO | Remediation checklist: docs/implplan/audits/csproj-standards/remediation/checklists/src/Scanner/__Libraries/StellaOps.Scanner.SmartDiff/StellaOps.Scanner.SmartDiff.md. |
| REMED-06 | DONE | SOLID review notes captured for SPRINT_20260130_002. |
| QA-SCANNER-VERIFY-001 | BLOCKED | SPRINT_20260212_002 run-001: blocked verification for `3-bit-reachability-gate` due missing AGENTS in required Scanner reachability/smart-diff test modules. |