test fixes and new product advisories work

2026-01-28 02:30:48 +02:00
parent 82caceba56
commit 644887997c
288 changed files with 69101 additions and 375 deletions
--- a/src/__Libraries/StellaOps.TestKit/Analysis/IntentCoverageReport.cs
+++ b/src/__Libraries/StellaOps.TestKit/Analysis/IntentCoverageReport.cs
@@ -0,0 +1,396 @@
+using System.Reflection;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using StellaOps.TestKit.Traits;
+
+namespace StellaOps.TestKit.Analysis;
+
+/// <summary>
+/// Generates intent coverage reports from test assemblies.
+/// </summary>
+/// <remarks>
+/// The report generator scans assemblies for tests with Intent traits and produces
+/// coverage matrices showing distribution of intents across modules. This helps
+/// identify coverage gaps (e.g., 90% Operational, 2% Safety) and drive testing investment.
+///
+/// Usage:
+/// <code>
+/// var generator = new IntentCoverageReportGenerator();
+/// generator.AddAssembly(typeof(MyTests).Assembly);
+/// var report = generator.Generate();
+/// await report.WriteJsonAsync("intent-coverage.json");
+/// </code>
+/// </remarks>
+public sealed class IntentCoverageReportGenerator
+{
+    private readonly List<Assembly> _assemblies = new();
+    private readonly Dictionary<string, ModuleIntentStats> _moduleStats = new();
+
+    /// <summary>
+    /// Add an assembly to scan for intent-tagged tests.
+    /// </summary>
+    public void AddAssembly(Assembly assembly)
+    {
+        ArgumentNullException.ThrowIfNull(assembly);
+        _assemblies.Add(assembly);
+    }
+
+    /// <summary>
+    /// Add multiple assemblies to scan.
+    /// </summary>
+    public void AddAssemblies(IEnumerable<Assembly> assemblies)
+    {
+        foreach (var assembly in assemblies)
+        {
+            AddAssembly(assembly);
+        }
+    }
+
+    /// <summary>
+    /// Generate the intent coverage report.
+    /// </summary>
+    public IntentCoverageReport Generate()
+    {
+        _moduleStats.Clear();
+
+        foreach (var assembly in _assemblies)
+        {
+            ScanAssembly(assembly);
+        }
+
+        var intents = TestIntents.All.ToDictionary(
+            i => i,
+            i => _moduleStats.Values.Sum(m => m.IntentCounts.GetValueOrDefault(i, 0)));
+
+        var totalTests = _moduleStats.Values.Sum(m => m.TotalTests);
+        var taggedTests = _moduleStats.Values.Sum(m => m.TaggedTests);
+        var untaggedTests = totalTests - taggedTests;
+
+        return new IntentCoverageReport
+        {
+            GeneratedAt = DateTimeOffset.UtcNow,
+            TotalTests = totalTests,
+            TaggedTests = taggedTests,
+            UntaggedTests = untaggedTests,
+            TagCoveragePercent = totalTests > 0 ? (double)taggedTests / totalTests * 100 : 0,
+            IntentDistribution = intents,
+            ModuleStats = _moduleStats.ToDictionary(
+                kvp => kvp.Key,
+                kvp => kvp.Value.ToReadOnly()),
+            Warnings = GenerateWarnings(intents, totalTests, taggedTests)
+        };
+    }
+
+    private void ScanAssembly(Assembly assembly)
+    {
+        var moduleName = ExtractModuleName(assembly);
+
+        if (!_moduleStats.TryGetValue(moduleName, out var stats))
+        {
+            stats = new ModuleIntentStats { ModuleName = moduleName };
+            _moduleStats[moduleName] = stats;
+        }
+
+        var testTypes = assembly.GetTypes()
+            .Where(t => t.IsClass && !t.IsAbstract && HasTestMethods(t));
+
+        foreach (var type in testTypes)
+        {
+            ScanType(type, stats);
+        }
+    }
+
+    private static void ScanType(Type type, ModuleIntentStats stats)
+    {
+        // Check class-level intent attributes
+        var classIntents = type.GetCustomAttributes<IntentAttribute>().ToList();
+
+        var testMethods = type.GetMethods(BindingFlags.Public | BindingFlags.Instance)
+            .Where(m => IsTestMethod(m));
+
+        foreach (var method in testMethods)
+        {
+            stats.TotalTests++;
+
+            var methodIntents = method.GetCustomAttributes<IntentAttribute>().ToList();
+            var allIntents = classIntents.Concat(methodIntents).ToList();
+
+            if (allIntents.Count > 0)
+            {
+                stats.TaggedTests++;
+                foreach (var intent in allIntents)
+                {
+                    stats.IntentCounts.TryGetValue(intent.Intent, out var count);
+                    stats.IntentCounts[intent.Intent] = count + 1;
+                }
+
+                if (allIntents.Any(i => !string.IsNullOrWhiteSpace(i.Rationale)))
+                {
+                    stats.TestsWithRationale++;
+                }
+            }
+            else
+            {
+                // Check for Trait-based intent
+                var traitAttrs = method.GetCustomAttributes()
+                    .Where(a => a.GetType().Name == "TraitAttribute")
+                    .ToList();
+
+                foreach (var attr in traitAttrs)
+                {
+                    var nameProp = attr.GetType().GetProperty("Name");
+                    var valueProp = attr.GetType().GetProperty("Value");
+                    if (nameProp?.GetValue(attr) is string name &&
+                        valueProp?.GetValue(attr) is string value &&
+                        name == "Intent")
+                    {
+                        stats.TaggedTests++;
+                        stats.IntentCounts.TryGetValue(value, out var count);
+                        stats.IntentCounts[value] = count + 1;
+                    }
+                }
+            }
+        }
+    }
+
+    private static bool HasTestMethods(Type type)
+    {
+        return type.GetMethods(BindingFlags.Public | BindingFlags.Instance)
+            .Any(IsTestMethod);
+    }
+
+    private static bool IsTestMethod(MethodInfo method)
+    {
+        var attrs = method.GetCustomAttributes().Select(a => a.GetType().Name).ToHashSet();
+        return attrs.Contains("FactAttribute") ||
+               attrs.Contains("TheoryAttribute") ||
+               attrs.Contains("TestAttribute");
+    }
+
+    private static string ExtractModuleName(Assembly assembly)
+    {
+        var name = assembly.GetName().Name ?? "Unknown";
+
+        // Extract module from assembly name like "StellaOps.Policy.Tests"
+        var parts = name.Split('.');
+        if (parts.Length >= 2 && parts[0] == "StellaOps")
+        {
+            return parts[1];
+        }
+
+        return name;
+    }
+
+    private static List<string> GenerateWarnings(
+        Dictionary<string, int> intents,
+        int totalTests,
+        int taggedTests)
+    {
+        var warnings = new List<string>();
+
+        // Warn if less than 50% of tests are tagged
+        if (totalTests > 0 && (double)taggedTests / totalTests < 0.5)
+        {
+            var percent = (double)taggedTests / totalTests * 100;
+            warnings.Add($"Low intent coverage: only {percent:F1}% of tests have intent tags");
+        }
+
+        // Warn about intent imbalance
+        var totalTagged = intents.Values.Sum();
+        if (totalTagged > 10)
+        {
+            foreach (var (intent, count) in intents)
+            {
+                var percent = (double)count / totalTagged * 100;
+                if (percent > 80)
+                {
+                    warnings.Add($"Intent imbalance: {intent} accounts for {percent:F1}% of tagged tests");
+                }
+                else if (percent < 5 && intent is "Safety" or "Regulatory")
+                {
+                    warnings.Add($"Critical intent underrepresented: {intent} is only {percent:F1}% of tagged tests");
+                }
+            }
+        }
+
+        // Warn if Safety is completely missing
+        if (!intents.TryGetValue(TestIntents.Safety, out var safetyCount) || safetyCount == 0)
+        {
+            warnings.Add("No tests tagged with Safety intent");
+        }
+
+        return warnings;
+    }
+
+    private sealed class ModuleIntentStats
+    {
+        public required string ModuleName { get; init; }
+        public int TotalTests { get; set; }
+        public int TaggedTests { get; set; }
+        public int TestsWithRationale { get; set; }
+        public Dictionary<string, int> IntentCounts { get; } = new();
+
+        public ModuleIntentStatsReadOnly ToReadOnly() => new()
+        {
+            ModuleName = ModuleName,
+            TotalTests = TotalTests,
+            TaggedTests = TaggedTests,
+            TestsWithRationale = TestsWithRationale,
+            TagCoveragePercent = TotalTests > 0 ? (double)TaggedTests / TotalTests * 100 : 0,
+            IntentCounts = new Dictionary<string, int>(IntentCounts)
+        };
+    }
+}
+
+/// <summary>
+/// Intent coverage report output format.
+/// </summary>
+public sealed record IntentCoverageReport
+{
+    /// <summary>
+    /// When the report was generated.
+    /// </summary>
+    public required DateTimeOffset GeneratedAt { get; init; }
+
+    /// <summary>
+    /// Total number of test methods scanned.
+    /// </summary>
+    public required int TotalTests { get; init; }
+
+    /// <summary>
+    /// Number of tests with intent tags.
+    /// </summary>
+    public required int TaggedTests { get; init; }
+
+    /// <summary>
+    /// Number of tests without intent tags.
+    /// </summary>
+    public required int UntaggedTests { get; init; }
+
+    /// <summary>
+    /// Percentage of tests with intent tags (0-100).
+    /// </summary>
+    public required double TagCoveragePercent { get; init; }
+
+    /// <summary>
+    /// Count of tests per intent category.
+    /// </summary>
+    public required Dictionary<string, int> IntentDistribution { get; init; }
+
+    /// <summary>
+    /// Per-module statistics.
+    /// </summary>
+    public required Dictionary<string, ModuleIntentStatsReadOnly> ModuleStats { get; init; }
+
+    /// <summary>
+    /// Generated warnings about coverage gaps or imbalances.
+    /// </summary>
+    public required List<string> Warnings { get; init; }
+
+    /// <summary>
+    /// Write the report as JSON to a file.
+    /// </summary>
+    public async Task WriteJsonAsync(string filePath, CancellationToken ct = default)
+    {
+        var options = new JsonSerializerOptions
+        {
+            WriteIndented = true,
+            PropertyNamingPolicy = JsonNamingPolicy.CamelCase
+        };
+
+        await using var stream = File.Create(filePath);
+        await JsonSerializer.SerializeAsync(stream, this, options, ct);
+    }
+
+    /// <summary>
+    /// Generate a markdown summary of the report.
+    /// </summary>
+    public string ToMarkdown()
+    {
+        var sb = new System.Text.StringBuilder();
+        sb.AppendLine("# Intent Coverage Report");
+        sb.AppendLine();
+        sb.AppendLine($"Generated: {GeneratedAt:yyyy-MM-dd HH:mm:ss} UTC");
+        sb.AppendLine();
+        sb.AppendLine("## Summary");
+        sb.AppendLine();
+        sb.AppendLine($"- Total tests: {TotalTests}");
+        sb.AppendLine($"- Tagged: {TaggedTests} ({TagCoveragePercent:F1}%)");
+        sb.AppendLine($"- Untagged: {UntaggedTests}");
+        sb.AppendLine();
+        sb.AppendLine("## Intent Distribution");
+        sb.AppendLine();
+        sb.AppendLine("| Intent | Count | Percent |");
+        sb.AppendLine("|--------|------:|--------:|");
+
+        var total = IntentDistribution.Values.Sum();
+        foreach (var (intent, count) in IntentDistribution.OrderByDescending(kvp => kvp.Value))
+        {
+            var percent = total > 0 ? (double)count / total * 100 : 0;
+            sb.AppendLine($"| {intent} | {count} | {percent:F1}% |");
+        }
+
+        if (ModuleStats.Count > 0)
+        {
+            sb.AppendLine();
+            sb.AppendLine("## Per-Module Coverage");
+            sb.AppendLine();
+            sb.AppendLine("| Module | Total | Tagged | Coverage |");
+            sb.AppendLine("|--------|------:|-------:|---------:|");
+
+            foreach (var (module, stats) in ModuleStats.OrderBy(kvp => kvp.Key))
+            {
+                sb.AppendLine($"| {module} | {stats.TotalTests} | {stats.TaggedTests} | {stats.TagCoveragePercent:F1}% |");
+            }
+        }
+
+        if (Warnings.Count > 0)
+        {
+            sb.AppendLine();
+            sb.AppendLine("## Warnings");
+            sb.AppendLine();
+            foreach (var warning in Warnings)
+            {
+                sb.AppendLine($"- {warning}");
+            }
+        }
+
+        return sb.ToString();
+    }
+}
+
+/// <summary>
+/// Read-only module intent statistics for report output.
+/// </summary>
+public sealed record ModuleIntentStatsReadOnly
+{
+    /// <summary>
+    /// Module name extracted from assembly.
+    /// </summary>
+    public required string ModuleName { get; init; }
+
+    /// <summary>
+    /// Total test count in module.
+    /// </summary>
+    public required int TotalTests { get; init; }
+
+    /// <summary>
+    /// Tests with intent tags.
+    /// </summary>
+    public required int TaggedTests { get; init; }
+
+    /// <summary>
+    /// Tests with rationale in their intent attribute.
+    /// </summary>
+    public required int TestsWithRationale { get; init; }
+
+    /// <summary>
+    /// Tag coverage percentage (0-100).
+    /// </summary>
+    public required double TagCoveragePercent { get; init; }
+
+    /// <summary>
+    /// Intent counts for this module.
+    /// </summary>
+    public required Dictionary<string, int> IntentCounts { get; init; }
+}