using StellaOps.TestKit.Traits; using System.Reflection; using System.Text.Json; using System.Text.Json.Serialization; namespace StellaOps.TestKit.Analysis; /// /// Generates intent coverage reports from test assemblies. /// /// /// The report generator scans assemblies for tests with Intent traits and produces /// coverage matrices showing distribution of intents across modules. This helps /// identify coverage gaps (e.g., 90% Operational, 2% Safety) and drive testing investment. /// /// Usage: /// /// var generator = new IntentCoverageReportGenerator(); /// generator.AddAssembly(typeof(MyTests).Assembly); /// var report = generator.Generate(); /// await report.WriteJsonAsync("intent-coverage.json"); /// /// public sealed class IntentCoverageReportGenerator { private readonly List _assemblies = new(); private readonly Dictionary _moduleStats = new(); /// /// Add an assembly to scan for intent-tagged tests. /// public void AddAssembly(Assembly assembly) { ArgumentNullException.ThrowIfNull(assembly); _assemblies.Add(assembly); } /// /// Add multiple assemblies to scan. /// public void AddAssemblies(IEnumerable assemblies) { foreach (var assembly in assemblies) { AddAssembly(assembly); } } /// /// Generate the intent coverage report. /// public IntentCoverageReport Generate() { _moduleStats.Clear(); foreach (var assembly in _assemblies) { ScanAssembly(assembly); } var intents = TestIntents.All.ToDictionary( i => i, i => _moduleStats.Values.Sum(m => m.IntentCounts.GetValueOrDefault(i, 0))); var totalTests = _moduleStats.Values.Sum(m => m.TotalTests); var taggedTests = _moduleStats.Values.Sum(m => m.TaggedTests); var untaggedTests = totalTests - taggedTests; return new IntentCoverageReport { GeneratedAt = DateTimeOffset.UtcNow, TotalTests = totalTests, TaggedTests = taggedTests, UntaggedTests = untaggedTests, TagCoveragePercent = totalTests > 0 ? (double)taggedTests / totalTests * 100 : 0, IntentDistribution = intents, ModuleStats = _moduleStats.ToDictionary( kvp => kvp.Key, kvp => kvp.Value.ToReadOnly()), Warnings = GenerateWarnings(intents, totalTests, taggedTests) }; } private void ScanAssembly(Assembly assembly) { var moduleName = ExtractModuleName(assembly); if (!_moduleStats.TryGetValue(moduleName, out var stats)) { stats = new ModuleIntentStats { ModuleName = moduleName }; _moduleStats[moduleName] = stats; } var testTypes = assembly.GetTypes() .Where(t => t.IsClass && !t.IsAbstract && HasTestMethods(t)); foreach (var type in testTypes) { ScanType(type, stats); } } private static void ScanType(Type type, ModuleIntentStats stats) { // Check class-level intent attributes var classIntents = type.GetCustomAttributes().ToList(); var testMethods = type.GetMethods(BindingFlags.Public | BindingFlags.Instance) .Where(m => IsTestMethod(m)); foreach (var method in testMethods) { stats.TotalTests++; var methodIntents = method.GetCustomAttributes().ToList(); var allIntents = classIntents.Concat(methodIntents).ToList(); if (allIntents.Count > 0) { stats.TaggedTests++; foreach (var intent in allIntents) { stats.IntentCounts.TryGetValue(intent.Intent, out var count); stats.IntentCounts[intent.Intent] = count + 1; } if (allIntents.Any(i => !string.IsNullOrWhiteSpace(i.Rationale))) { stats.TestsWithRationale++; } } else { // Check for Trait-based intent var traitAttrs = method.GetCustomAttributes() .Where(a => a.GetType().Name == "TraitAttribute") .ToList(); foreach (var attr in traitAttrs) { var nameProp = attr.GetType().GetProperty("Name"); var valueProp = attr.GetType().GetProperty("Value"); if (nameProp?.GetValue(attr) is string name && valueProp?.GetValue(attr) is string value && name == "Intent") { stats.TaggedTests++; stats.IntentCounts.TryGetValue(value, out var count); stats.IntentCounts[value] = count + 1; } } } } } private static bool HasTestMethods(Type type) { return type.GetMethods(BindingFlags.Public | BindingFlags.Instance) .Any(IsTestMethod); } private static bool IsTestMethod(MethodInfo method) { var attrs = method.GetCustomAttributes().Select(a => a.GetType().Name).ToHashSet(); return attrs.Contains("FactAttribute") || attrs.Contains("TheoryAttribute") || attrs.Contains("TestAttribute"); } private static string ExtractModuleName(Assembly assembly) { var name = assembly.GetName().Name ?? "Unknown"; // Extract module from assembly name like "StellaOps.Policy.Tests" var parts = name.Split('.'); if (parts.Length >= 2 && parts[0] == "StellaOps") { return parts[1]; } return name; } private static List GenerateWarnings( Dictionary intents, int totalTests, int taggedTests) { var warnings = new List(); // Warn if less than 50% of tests are tagged if (totalTests > 0 && (double)taggedTests / totalTests < 0.5) { var percent = (double)taggedTests / totalTests * 100; warnings.Add($"Low intent coverage: only {percent:F1}% of tests have intent tags"); } // Warn about intent imbalance var totalTagged = intents.Values.Sum(); if (totalTagged > 10) { foreach (var (intent, count) in intents) { var percent = (double)count / totalTagged * 100; if (percent > 80) { warnings.Add($"Intent imbalance: {intent} accounts for {percent:F1}% of tagged tests"); } else if (percent < 5 && intent is "Safety" or "Regulatory") { warnings.Add($"Critical intent underrepresented: {intent} is only {percent:F1}% of tagged tests"); } } } // Warn if Safety is completely missing if (!intents.TryGetValue(TestIntents.Safety, out var safetyCount) || safetyCount == 0) { warnings.Add("No tests tagged with Safety intent"); } return warnings; } private sealed class ModuleIntentStats { public required string ModuleName { get; init; } public int TotalTests { get; set; } public int TaggedTests { get; set; } public int TestsWithRationale { get; set; } public Dictionary IntentCounts { get; } = new(); public ModuleIntentStatsReadOnly ToReadOnly() => new() { ModuleName = ModuleName, TotalTests = TotalTests, TaggedTests = TaggedTests, TestsWithRationale = TestsWithRationale, TagCoveragePercent = TotalTests > 0 ? (double)TaggedTests / TotalTests * 100 : 0, IntentCounts = new Dictionary(IntentCounts) }; } } /// /// Intent coverage report output format. /// public sealed record IntentCoverageReport { /// /// When the report was generated. /// public required DateTimeOffset GeneratedAt { get; init; } /// /// Total number of test methods scanned. /// public required int TotalTests { get; init; } /// /// Number of tests with intent tags. /// public required int TaggedTests { get; init; } /// /// Number of tests without intent tags. /// public required int UntaggedTests { get; init; } /// /// Percentage of tests with intent tags (0-100). /// public required double TagCoveragePercent { get; init; } /// /// Count of tests per intent category. /// public required Dictionary IntentDistribution { get; init; } /// /// Per-module statistics. /// public required Dictionary ModuleStats { get; init; } /// /// Generated warnings about coverage gaps or imbalances. /// public required List Warnings { get; init; } /// /// Write the report as JSON to a file. /// public async Task WriteJsonAsync(string filePath, CancellationToken ct = default) { var options = new JsonSerializerOptions { WriteIndented = true, PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; await using var stream = File.Create(filePath); await JsonSerializer.SerializeAsync(stream, this, options, ct); } /// /// Generate a markdown summary of the report. /// public string ToMarkdown() { var sb = new System.Text.StringBuilder(); sb.AppendLine("# Intent Coverage Report"); sb.AppendLine(); sb.AppendLine($"Generated: {GeneratedAt:yyyy-MM-dd HH:mm:ss} UTC"); sb.AppendLine(); sb.AppendLine("## Summary"); sb.AppendLine(); sb.AppendLine($"- Total tests: {TotalTests}"); sb.AppendLine($"- Tagged: {TaggedTests} ({TagCoveragePercent:F1}%)"); sb.AppendLine($"- Untagged: {UntaggedTests}"); sb.AppendLine(); sb.AppendLine("## Intent Distribution"); sb.AppendLine(); sb.AppendLine("| Intent | Count | Percent |"); sb.AppendLine("|--------|------:|--------:|"); var total = IntentDistribution.Values.Sum(); foreach (var (intent, count) in IntentDistribution.OrderByDescending(kvp => kvp.Value)) { var percent = total > 0 ? (double)count / total * 100 : 0; sb.AppendLine($"| {intent} | {count} | {percent:F1}% |"); } if (ModuleStats.Count > 0) { sb.AppendLine(); sb.AppendLine("## Per-Module Coverage"); sb.AppendLine(); sb.AppendLine("| Module | Total | Tagged | Coverage |"); sb.AppendLine("|--------|------:|-------:|---------:|"); foreach (var (module, stats) in ModuleStats.OrderBy(kvp => kvp.Key)) { sb.AppendLine($"| {module} | {stats.TotalTests} | {stats.TaggedTests} | {stats.TagCoveragePercent:F1}% |"); } } if (Warnings.Count > 0) { sb.AppendLine(); sb.AppendLine("## Warnings"); sb.AppendLine(); foreach (var warning in Warnings) { sb.AppendLine($"- {warning}"); } } return sb.ToString(); } } /// /// Read-only module intent statistics for report output. /// public sealed record ModuleIntentStatsReadOnly { /// /// Module name extracted from assembly. /// public required string ModuleName { get; init; } /// /// Total test count in module. /// public required int TotalTests { get; init; } /// /// Tests with intent tags. /// public required int TaggedTests { get; init; } /// /// Tests with rationale in their intent attribute. /// public required int TestsWithRationale { get; init; } /// /// Tag coverage percentage (0-100). /// public required double TagCoveragePercent { get; init; } /// /// Intent counts for this module. /// public required Dictionary IntentCounts { get; init; } }