398 lines
13 KiB
C#
398 lines
13 KiB
C#
|
|
using StellaOps.TestKit.Traits;
|
|
using System.Reflection;
|
|
using System.Text.Json;
|
|
using System.Text.Json.Serialization;
|
|
|
|
namespace StellaOps.TestKit.Analysis;
|
|
|
|
/// <summary>
|
|
/// Generates intent coverage reports from test assemblies.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// The report generator scans assemblies for tests with Intent traits and produces
|
|
/// coverage matrices showing distribution of intents across modules. This helps
|
|
/// identify coverage gaps (e.g., 90% Operational, 2% Safety) and drive testing investment.
|
|
///
|
|
/// Usage:
|
|
/// <code>
|
|
/// var generator = new IntentCoverageReportGenerator();
|
|
/// generator.AddAssembly(typeof(MyTests).Assembly);
|
|
/// var report = generator.Generate();
|
|
/// await report.WriteJsonAsync("intent-coverage.json");
|
|
/// </code>
|
|
/// </remarks>
|
|
public sealed class IntentCoverageReportGenerator
|
|
{
|
|
private readonly List<Assembly> _assemblies = new();
|
|
private readonly Dictionary<string, ModuleIntentStats> _moduleStats = new();
|
|
|
|
/// <summary>
|
|
/// Add an assembly to scan for intent-tagged tests.
|
|
/// </summary>
|
|
public void AddAssembly(Assembly assembly)
|
|
{
|
|
ArgumentNullException.ThrowIfNull(assembly);
|
|
_assemblies.Add(assembly);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Add multiple assemblies to scan.
|
|
/// </summary>
|
|
public void AddAssemblies(IEnumerable<Assembly> assemblies)
|
|
{
|
|
foreach (var assembly in assemblies)
|
|
{
|
|
AddAssembly(assembly);
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Generate the intent coverage report.
|
|
/// </summary>
|
|
public IntentCoverageReport Generate()
|
|
{
|
|
_moduleStats.Clear();
|
|
|
|
foreach (var assembly in _assemblies)
|
|
{
|
|
ScanAssembly(assembly);
|
|
}
|
|
|
|
var intents = TestIntents.All.ToDictionary(
|
|
i => i,
|
|
i => _moduleStats.Values.Sum(m => m.IntentCounts.GetValueOrDefault(i, 0)));
|
|
|
|
var totalTests = _moduleStats.Values.Sum(m => m.TotalTests);
|
|
var taggedTests = _moduleStats.Values.Sum(m => m.TaggedTests);
|
|
var untaggedTests = totalTests - taggedTests;
|
|
|
|
return new IntentCoverageReport
|
|
{
|
|
GeneratedAt = DateTimeOffset.UtcNow,
|
|
TotalTests = totalTests,
|
|
TaggedTests = taggedTests,
|
|
UntaggedTests = untaggedTests,
|
|
TagCoveragePercent = totalTests > 0 ? (double)taggedTests / totalTests * 100 : 0,
|
|
IntentDistribution = intents,
|
|
ModuleStats = _moduleStats.ToDictionary(
|
|
kvp => kvp.Key,
|
|
kvp => kvp.Value.ToReadOnly()),
|
|
Warnings = GenerateWarnings(intents, totalTests, taggedTests)
|
|
};
|
|
}
|
|
|
|
private void ScanAssembly(Assembly assembly)
|
|
{
|
|
var moduleName = ExtractModuleName(assembly);
|
|
|
|
if (!_moduleStats.TryGetValue(moduleName, out var stats))
|
|
{
|
|
stats = new ModuleIntentStats { ModuleName = moduleName };
|
|
_moduleStats[moduleName] = stats;
|
|
}
|
|
|
|
var testTypes = assembly.GetTypes()
|
|
.Where(t => t.IsClass && !t.IsAbstract && HasTestMethods(t));
|
|
|
|
foreach (var type in testTypes)
|
|
{
|
|
ScanType(type, stats);
|
|
}
|
|
}
|
|
|
|
private static void ScanType(Type type, ModuleIntentStats stats)
|
|
{
|
|
// Check class-level intent attributes
|
|
var classIntents = type.GetCustomAttributes<IntentAttribute>().ToList();
|
|
|
|
var testMethods = type.GetMethods(BindingFlags.Public | BindingFlags.Instance)
|
|
.Where(m => IsTestMethod(m));
|
|
|
|
foreach (var method in testMethods)
|
|
{
|
|
stats.TotalTests++;
|
|
|
|
var methodIntents = method.GetCustomAttributes<IntentAttribute>().ToList();
|
|
var allIntents = classIntents.Concat(methodIntents).ToList();
|
|
|
|
if (allIntents.Count > 0)
|
|
{
|
|
stats.TaggedTests++;
|
|
foreach (var intent in allIntents)
|
|
{
|
|
stats.IntentCounts.TryGetValue(intent.Intent, out var count);
|
|
stats.IntentCounts[intent.Intent] = count + 1;
|
|
}
|
|
|
|
if (allIntents.Any(i => !string.IsNullOrWhiteSpace(i.Rationale)))
|
|
{
|
|
stats.TestsWithRationale++;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// Check for Trait-based intent
|
|
var traitAttrs = method.GetCustomAttributes()
|
|
.Where(a => a.GetType().Name == "TraitAttribute")
|
|
.ToList();
|
|
|
|
foreach (var attr in traitAttrs)
|
|
{
|
|
var nameProp = attr.GetType().GetProperty("Name");
|
|
var valueProp = attr.GetType().GetProperty("Value");
|
|
if (nameProp?.GetValue(attr) is string name &&
|
|
valueProp?.GetValue(attr) is string value &&
|
|
name == "Intent")
|
|
{
|
|
stats.TaggedTests++;
|
|
stats.IntentCounts.TryGetValue(value, out var count);
|
|
stats.IntentCounts[value] = count + 1;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
private static bool HasTestMethods(Type type)
|
|
{
|
|
return type.GetMethods(BindingFlags.Public | BindingFlags.Instance)
|
|
.Any(IsTestMethod);
|
|
}
|
|
|
|
private static bool IsTestMethod(MethodInfo method)
|
|
{
|
|
var attrs = method.GetCustomAttributes().Select(a => a.GetType().Name).ToHashSet();
|
|
return attrs.Contains("FactAttribute") ||
|
|
attrs.Contains("TheoryAttribute") ||
|
|
attrs.Contains("TestAttribute");
|
|
}
|
|
|
|
private static string ExtractModuleName(Assembly assembly)
|
|
{
|
|
var name = assembly.GetName().Name ?? "Unknown";
|
|
|
|
// Extract module from assembly name like "StellaOps.Policy.Tests"
|
|
var parts = name.Split('.');
|
|
if (parts.Length >= 2 && parts[0] == "StellaOps")
|
|
{
|
|
return parts[1];
|
|
}
|
|
|
|
return name;
|
|
}
|
|
|
|
private static List<string> GenerateWarnings(
|
|
Dictionary<string, int> intents,
|
|
int totalTests,
|
|
int taggedTests)
|
|
{
|
|
var warnings = new List<string>();
|
|
|
|
// Warn if less than 50% of tests are tagged
|
|
if (totalTests > 0 && (double)taggedTests / totalTests < 0.5)
|
|
{
|
|
var percent = (double)taggedTests / totalTests * 100;
|
|
warnings.Add($"Low intent coverage: only {percent:F1}% of tests have intent tags");
|
|
}
|
|
|
|
// Warn about intent imbalance
|
|
var totalTagged = intents.Values.Sum();
|
|
if (totalTagged > 10)
|
|
{
|
|
foreach (var (intent, count) in intents)
|
|
{
|
|
var percent = (double)count / totalTagged * 100;
|
|
if (percent > 80)
|
|
{
|
|
warnings.Add($"Intent imbalance: {intent} accounts for {percent:F1}% of tagged tests");
|
|
}
|
|
else if (percent < 5 && intent is "Safety" or "Regulatory")
|
|
{
|
|
warnings.Add($"Critical intent underrepresented: {intent} is only {percent:F1}% of tagged tests");
|
|
}
|
|
}
|
|
}
|
|
|
|
// Warn if Safety is completely missing
|
|
if (!intents.TryGetValue(TestIntents.Safety, out var safetyCount) || safetyCount == 0)
|
|
{
|
|
warnings.Add("No tests tagged with Safety intent");
|
|
}
|
|
|
|
return warnings;
|
|
}
|
|
|
|
private sealed class ModuleIntentStats
|
|
{
|
|
public required string ModuleName { get; init; }
|
|
public int TotalTests { get; set; }
|
|
public int TaggedTests { get; set; }
|
|
public int TestsWithRationale { get; set; }
|
|
public Dictionary<string, int> IntentCounts { get; } = new();
|
|
|
|
public ModuleIntentStatsReadOnly ToReadOnly() => new()
|
|
{
|
|
ModuleName = ModuleName,
|
|
TotalTests = TotalTests,
|
|
TaggedTests = TaggedTests,
|
|
TestsWithRationale = TestsWithRationale,
|
|
TagCoveragePercent = TotalTests > 0 ? (double)TaggedTests / TotalTests * 100 : 0,
|
|
IntentCounts = new Dictionary<string, int>(IntentCounts)
|
|
};
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Intent coverage report output format.
|
|
/// </summary>
|
|
public sealed record IntentCoverageReport
|
|
{
|
|
/// <summary>
|
|
/// When the report was generated.
|
|
/// </summary>
|
|
public required DateTimeOffset GeneratedAt { get; init; }
|
|
|
|
/// <summary>
|
|
/// Total number of test methods scanned.
|
|
/// </summary>
|
|
public required int TotalTests { get; init; }
|
|
|
|
/// <summary>
|
|
/// Number of tests with intent tags.
|
|
/// </summary>
|
|
public required int TaggedTests { get; init; }
|
|
|
|
/// <summary>
|
|
/// Number of tests without intent tags.
|
|
/// </summary>
|
|
public required int UntaggedTests { get; init; }
|
|
|
|
/// <summary>
|
|
/// Percentage of tests with intent tags (0-100).
|
|
/// </summary>
|
|
public required double TagCoveragePercent { get; init; }
|
|
|
|
/// <summary>
|
|
/// Count of tests per intent category.
|
|
/// </summary>
|
|
public required Dictionary<string, int> IntentDistribution { get; init; }
|
|
|
|
/// <summary>
|
|
/// Per-module statistics.
|
|
/// </summary>
|
|
public required Dictionary<string, ModuleIntentStatsReadOnly> ModuleStats { get; init; }
|
|
|
|
/// <summary>
|
|
/// Generated warnings about coverage gaps or imbalances.
|
|
/// </summary>
|
|
public required List<string> Warnings { get; init; }
|
|
|
|
/// <summary>
|
|
/// Write the report as JSON to a file.
|
|
/// </summary>
|
|
public async Task WriteJsonAsync(string filePath, CancellationToken ct = default)
|
|
{
|
|
var options = new JsonSerializerOptions
|
|
{
|
|
WriteIndented = true,
|
|
PropertyNamingPolicy = JsonNamingPolicy.CamelCase
|
|
};
|
|
|
|
await using var stream = File.Create(filePath);
|
|
await JsonSerializer.SerializeAsync(stream, this, options, ct);
|
|
}
|
|
|
|
/// <summary>
|
|
/// Generate a markdown summary of the report.
|
|
/// </summary>
|
|
public string ToMarkdown()
|
|
{
|
|
var sb = new System.Text.StringBuilder();
|
|
sb.AppendLine("# Intent Coverage Report");
|
|
sb.AppendLine();
|
|
sb.AppendLine($"Generated: {GeneratedAt:yyyy-MM-dd HH:mm:ss} UTC");
|
|
sb.AppendLine();
|
|
sb.AppendLine("## Summary");
|
|
sb.AppendLine();
|
|
sb.AppendLine($"- Total tests: {TotalTests}");
|
|
sb.AppendLine($"- Tagged: {TaggedTests} ({TagCoveragePercent:F1}%)");
|
|
sb.AppendLine($"- Untagged: {UntaggedTests}");
|
|
sb.AppendLine();
|
|
sb.AppendLine("## Intent Distribution");
|
|
sb.AppendLine();
|
|
sb.AppendLine("| Intent | Count | Percent |");
|
|
sb.AppendLine("|--------|------:|--------:|");
|
|
|
|
var total = IntentDistribution.Values.Sum();
|
|
foreach (var (intent, count) in IntentDistribution.OrderByDescending(kvp => kvp.Value))
|
|
{
|
|
var percent = total > 0 ? (double)count / total * 100 : 0;
|
|
sb.AppendLine($"| {intent} | {count} | {percent:F1}% |");
|
|
}
|
|
|
|
if (ModuleStats.Count > 0)
|
|
{
|
|
sb.AppendLine();
|
|
sb.AppendLine("## Per-Module Coverage");
|
|
sb.AppendLine();
|
|
sb.AppendLine("| Module | Total | Tagged | Coverage |");
|
|
sb.AppendLine("|--------|------:|-------:|---------:|");
|
|
|
|
foreach (var (module, stats) in ModuleStats.OrderBy(kvp => kvp.Key))
|
|
{
|
|
sb.AppendLine($"| {module} | {stats.TotalTests} | {stats.TaggedTests} | {stats.TagCoveragePercent:F1}% |");
|
|
}
|
|
}
|
|
|
|
if (Warnings.Count > 0)
|
|
{
|
|
sb.AppendLine();
|
|
sb.AppendLine("## Warnings");
|
|
sb.AppendLine();
|
|
foreach (var warning in Warnings)
|
|
{
|
|
sb.AppendLine($"- {warning}");
|
|
}
|
|
}
|
|
|
|
return sb.ToString();
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Read-only module intent statistics for report output.
|
|
/// </summary>
|
|
public sealed record ModuleIntentStatsReadOnly
|
|
{
|
|
/// <summary>
|
|
/// Module name extracted from assembly.
|
|
/// </summary>
|
|
public required string ModuleName { get; init; }
|
|
|
|
/// <summary>
|
|
/// Total test count in module.
|
|
/// </summary>
|
|
public required int TotalTests { get; init; }
|
|
|
|
/// <summary>
|
|
/// Tests with intent tags.
|
|
/// </summary>
|
|
public required int TaggedTests { get; init; }
|
|
|
|
/// <summary>
|
|
/// Tests with rationale in their intent attribute.
|
|
/// </summary>
|
|
public required int TestsWithRationale { get; init; }
|
|
|
|
/// <summary>
|
|
/// Tag coverage percentage (0-100).
|
|
/// </summary>
|
|
public required double TagCoveragePercent { get; init; }
|
|
|
|
/// <summary>
|
|
/// Intent counts for this module.
|
|
/// </summary>
|
|
public required Dictionary<string, int> IntentCounts { get; init; }
|
|
}
|