Add Policy DSL Validator, Schema Exporter, and Simulation Smoke tools

- Implemented PolicyDslValidator with command-line options for strict mode and JSON output. - Created PolicySchemaExporter to generate JSON schemas for policy-related models. - Developed PolicySimulationSmoke tool to validate policy simulations against expected outcomes. - Added project files and necessary dependencies for each tool. - Ensured proper error handling and usage instructions across tools.
2025-10-27 08:00:11 +02:00
parent 651b8e0fa3
commit 96d52884e8
712 changed files with 49449 additions and 6124 deletions
--- a/tools/LanguageAnalyzerSmoke/LanguageAnalyzerSmoke.csproj
+++ b/tools/LanguageAnalyzerSmoke/LanguageAnalyzerSmoke.csproj
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net10.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.0-rc.2.25502.107" />
+    <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\StellaOps.Scanner.Analyzers.Lang\StellaOps.Scanner.Analyzers.Lang.csproj" />
+    <ProjectReference Include="..\..\src\StellaOps.Scanner.Core\StellaOps.Scanner.Core.csproj" />
+  </ItemGroup>
+</Project>
--- a/tools/LanguageAnalyzerSmoke/Program.cs
+++ b/tools/LanguageAnalyzerSmoke/Program.cs
@@ -0,0 +1,348 @@
+using System.Collections.Immutable;
+using System.Diagnostics;
+using System.Reflection;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+using StellaOps.Scanner.Analyzers.Lang;
+using StellaOps.Scanner.Analyzers.Lang.Plugin;
+using StellaOps.Scanner.Core.Security;
+
+internal sealed record SmokeScenario(string Name, string[] UsageHintRelatives)
+{
+    public IReadOnlyList<string> ResolveUsageHints(string scenarioRoot)
+        => UsageHintRelatives.Select(relative => Path.GetFullPath(Path.Combine(scenarioRoot, relative))).ToArray();
+}
+
+internal sealed class SmokeOptions
+{
+    public string RepoRoot { get; set; } = Directory.GetCurrentDirectory();
+    public string PluginDirectoryName { get; set; } = "StellaOps.Scanner.Analyzers.Lang.Python";
+    public string FixtureRelativePath { get; set; } = Path.Combine("src", "StellaOps.Scanner.Analyzers.Lang.Python.Tests", "Fixtures", "lang", "python");
+
+    public static SmokeOptions Parse(string[] args)
+    {
+        var options = new SmokeOptions();
+
+        for (var index = 0; index < args.Length; index++)
+        {
+            var current = args[index];
+            switch (current)
+            {
+                case "--repo-root":
+                case "-r":
+                    options.RepoRoot = RequireValue(args, ref index, current);
+                    break;
+                case "--plugin-directory":
+                case "-p":
+                    options.PluginDirectoryName = RequireValue(args, ref index, current);
+                    break;
+                case "--fixture-path":
+                case "-f":
+                    options.FixtureRelativePath = RequireValue(args, ref index, current);
+                    break;
+                case "--help":
+                case "-h":
+                    PrintUsage();
+                    Environment.Exit(0);
+                    break;
+                default:
+                    throw new ArgumentException($"Unknown argument '{current}'. Use --help for usage.");
+            }
+        }
+
+        options.RepoRoot = Path.GetFullPath(options.RepoRoot);
+        return options;
+    }
+
+    private static string RequireValue(string[] args, ref int index, string switchName)
+    {
+        if (index + 1 >= args.Length)
+        {
+            throw new ArgumentException($"Missing value for '{switchName}'.");
+        }
+
+        index++;
+        var value = args[index];
+        if (string.IsNullOrWhiteSpace(value))
+        {
+            throw new ArgumentException($"Value for '{switchName}' cannot be empty.");
+        }
+
+        return value;
+    }
+
+    private static void PrintUsage()
+    {
+        Console.WriteLine("Language Analyzer Smoke Harness");
+        Console.WriteLine("Usage: dotnet run --project tools/LanguageAnalyzerSmoke -- [options]");
+        Console.WriteLine();
+        Console.WriteLine("Options:");
+        Console.WriteLine("  -r, --repo-root <path>        Repository root (defaults to current working directory)");
+        Console.WriteLine("  -p, --plugin-directory <name> Analyzer plug-in directory under plugins/scanner/analyzers/lang (defaults to StellaOps.Scanner.Analyzers.Lang.Python)");
+        Console.WriteLine("  -f, --fixture-path <path>     Relative path to fixtures root (defaults to src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python)");
+        Console.WriteLine("  -h, --help                    Show usage information");
+    }
+}
+
+internal sealed record PluginManifest
+{
+    [JsonPropertyName("schemaVersion")]
+    public string SchemaVersion { get; init; } = string.Empty;
+
+    [JsonPropertyName("id")]
+    public string Id { get; init; } = string.Empty;
+
+    [JsonPropertyName("displayName")]
+    public string DisplayName { get; init; } = string.Empty;
+
+    [JsonPropertyName("version")]
+    public string Version { get; init; } = string.Empty;
+
+    [JsonPropertyName("requiresRestart")]
+    public bool RequiresRestart { get; init; }
+
+    [JsonPropertyName("entryPoint")]
+    public PluginEntryPoint EntryPoint { get; init; } = new();
+
+    [JsonPropertyName("capabilities")]
+    public IReadOnlyList<string> Capabilities { get; init; } = Array.Empty<string>();
+
+    [JsonPropertyName("metadata")]
+    public IReadOnlyDictionary<string, string> Metadata { get; init; } = ImmutableDictionary<string, string>.Empty;
+}
+
+internal sealed record PluginEntryPoint
+{
+    [JsonPropertyName("type")]
+    public string Type { get; init; } = string.Empty;
+
+    [JsonPropertyName("assembly")]
+    public string Assembly { get; init; } = string.Empty;
+
+    [JsonPropertyName("typeName")]
+    public string TypeName { get; init; } = string.Empty;
+}
+
+file static class Program
+{
+    private static readonly SmokeScenario[] PythonScenarios =
+    {
+        new("simple-venv", new[] { Path.Combine("bin", "simple-tool") }),
+        new("pip-cache", new[] { Path.Combine("lib", "python3.11", "site-packages", "cache_pkg-1.2.3.data", "scripts", "cache-tool") }),
+        new("layered-editable", new[] { Path.Combine("layer1", "usr", "bin", "layered-cli") })
+    };
+
+    public static async Task<int> Main(string[] args)
+    {
+        try
+        {
+            var options = SmokeOptions.Parse(args);
+            await RunAsync(options).ConfigureAwait(false);
+            Console.WriteLine("✅ Python analyzer smoke checks passed");
+            return 0;
+        }
+        catch (Exception ex)
+        {
+            Console.Error.WriteLine($"❌ {ex.Message}");
+            return 1;
+        }
+    }
+
+    private static async Task RunAsync(SmokeOptions options)
+    {
+        ValidateOptions(options);
+
+        var pluginRoot = Path.Combine(options.RepoRoot, "plugins", "scanner", "analyzers", "lang", options.PluginDirectoryName);
+        var manifestPath = Path.Combine(pluginRoot, "manifest.json");
+        if (!File.Exists(manifestPath))
+        {
+            throw new FileNotFoundException($"Plug-in manifest not found at '{manifestPath}'.", manifestPath);
+        }
+
+        using var manifestStream = File.OpenRead(manifestPath);
+        var manifest = JsonSerializer.Deserialize<PluginManifest>(manifestStream, new JsonSerializerOptions
+        {
+            PropertyNameCaseInsensitive = true,
+            ReadCommentHandling = JsonCommentHandling.Skip
+        }) ?? throw new InvalidOperationException($"Unable to parse manifest '{manifestPath}'.");
+
+        ValidateManifest(manifest, options.PluginDirectoryName);
+
+        var pluginAssemblyPath = Path.Combine(pluginRoot, manifest.EntryPoint.Assembly);
+        if (!File.Exists(pluginAssemblyPath))
+        {
+            throw new FileNotFoundException($"Plug-in assembly '{manifest.EntryPoint.Assembly}' not found under '{pluginRoot}'.", pluginAssemblyPath);
+        }
+
+        var sha256 = ComputeSha256(pluginAssemblyPath);
+        Console.WriteLine($"→ Plug-in assembly SHA-256: {sha256}");
+
+        using var serviceProvider = BuildServiceProvider();
+        var catalog = new LanguageAnalyzerPluginCatalog(new RestartOnlyPluginGuard(), NullLogger<LanguageAnalyzerPluginCatalog>.Instance);
+        catalog.LoadFromDirectory(pluginRoot, seal: true);
+
+        if (catalog.Plugins.Count == 0)
+        {
+            throw new InvalidOperationException($"No analyzer plug-ins were loaded from '{pluginRoot}'.");
+        }
+
+        var analyzerSet = catalog.CreateAnalyzers(serviceProvider);
+        if (analyzerSet.Count == 0)
+        {
+            throw new InvalidOperationException("Language analyzer plug-ins reported no analyzers.");
+        }
+
+        var analyzerIds = analyzerSet.Select(analyzer => analyzer.Id).ToArray();
+        Console.WriteLine($"→ Loaded analyzers: {string.Join(", ", analyzerIds)}");
+
+        if (!analyzerIds.Contains("python", StringComparer.OrdinalIgnoreCase))
+        {
+            throw new InvalidOperationException("Python analyzer was not created by the plug-in.");
+        }
+
+        var fixtureRoot = Path.GetFullPath(Path.Combine(options.RepoRoot, options.FixtureRelativePath));
+        if (!Directory.Exists(fixtureRoot))
+        {
+            throw new DirectoryNotFoundException($"Fixture directory '{fixtureRoot}' does not exist.");
+        }
+
+        foreach (var scenario in PythonScenarios)
+        {
+            await RunScenarioAsync(scenario, fixtureRoot, catalog, serviceProvider).ConfigureAwait(false);
+        }
+    }
+
+    private static ServiceProvider BuildServiceProvider()
+    {
+        var services = new ServiceCollection();
+        services.AddLogging();
+        return services.BuildServiceProvider();
+    }
+
+    private static async Task RunScenarioAsync(SmokeScenario scenario, string fixtureRoot, ILanguageAnalyzerPluginCatalog catalog, IServiceProvider services)
+    {
+        var scenarioRoot = Path.Combine(fixtureRoot, scenario.Name);
+        if (!Directory.Exists(scenarioRoot))
+        {
+            throw new DirectoryNotFoundException($"Scenario '{scenario.Name}' directory missing at '{scenarioRoot}'.");
+        }
+
+        var goldenPath = Path.Combine(scenarioRoot, "expected.json");
+        string? goldenNormalized = null;
+        if (File.Exists(goldenPath))
+        {
+            goldenNormalized = NormalizeJson(await File.ReadAllTextAsync(goldenPath).ConfigureAwait(false));
+        }
+
+        var usageHints = new LanguageUsageHints(scenario.ResolveUsageHints(scenarioRoot));
+        var context = new LanguageAnalyzerContext(scenarioRoot, TimeProvider.System, usageHints, services);
+
+        var coldEngine = new LanguageAnalyzerEngine(catalog.CreateAnalyzers(services));
+        var coldStopwatch = Stopwatch.StartNew();
+        var coldResult = await coldEngine.AnalyzeAsync(context, CancellationToken.None).ConfigureAwait(false);
+        coldStopwatch.Stop();
+
+        if (coldResult.Components.Count == 0)
+        {
+            throw new InvalidOperationException($"Scenario '{scenario.Name}' produced no components during cold run.");
+        }
+
+        var coldJson = NormalizeJson(coldResult.ToJson(indent: true));
+        if (goldenNormalized is string expected && !string.Equals(coldJson, expected, StringComparison.Ordinal))
+        {
+            Console.WriteLine($"⚠️ Scenario '{scenario.Name}' output deviates from repository golden snapshot.");
+        }
+
+        var warmEngine = new LanguageAnalyzerEngine(catalog.CreateAnalyzers(services));
+        var warmStopwatch = Stopwatch.StartNew();
+        var warmResult = await warmEngine.AnalyzeAsync(context, CancellationToken.None).ConfigureAwait(false);
+        warmStopwatch.Stop();
+
+        var warmJson = NormalizeJson(warmResult.ToJson(indent: true));
+        if (!string.Equals(coldJson, warmJson, StringComparison.Ordinal))
+        {
+            throw new InvalidOperationException($"Scenario '{scenario.Name}' produced different outputs between cold and warm runs.");
+        }
+
+        EnsureDurationWithinBudget(scenario.Name, coldStopwatch.Elapsed, warmStopwatch.Elapsed);
+
+        Console.WriteLine($"✓ Scenario '{scenario.Name}' — components {coldResult.Components.Count}, cold {coldStopwatch.Elapsed.TotalMilliseconds:F1} ms, warm {warmStopwatch.Elapsed.TotalMilliseconds:F1} ms");
+    }
+
+    private static void EnsureDurationWithinBudget(string scenarioName, TimeSpan coldDuration, TimeSpan warmDuration)
+    {
+        var coldBudget = TimeSpan.FromSeconds(30);
+        var warmBudget = TimeSpan.FromSeconds(5);
+
+        if (coldDuration > coldBudget)
+        {
+            throw new InvalidOperationException($"Scenario '{scenarioName}' cold run exceeded budget ({coldDuration.TotalSeconds:F2}s > {coldBudget.TotalSeconds:F2}s).");
+        }
+
+        if (warmDuration > warmBudget)
+        {
+            throw new InvalidOperationException($"Scenario '{scenarioName}' warm run exceeded budget ({warmDuration.TotalSeconds:F2}s > {warmBudget.TotalSeconds:F2}s).");
+        }
+    }
+
+    private static string NormalizeJson(string json)
+        => json.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd();
+
+    private static void ValidateOptions(SmokeOptions options)
+    {
+        if (!Directory.Exists(options.RepoRoot))
+        {
+            throw new DirectoryNotFoundException($"Repository root '{options.RepoRoot}' does not exist.");
+        }
+    }
+
+    private static void ValidateManifest(PluginManifest manifest, string expectedDirectory)
+    {
+        if (!string.Equals(manifest.SchemaVersion, "1.0", StringComparison.Ordinal))
+        {
+            throw new InvalidOperationException($"Unexpected manifest schema version '{manifest.SchemaVersion}'.");
+        }
+
+        if (!manifest.RequiresRestart)
+        {
+            throw new InvalidOperationException("Language analyzer plug-in must be marked as restart-only.");
+        }
+
+        if (!string.Equals(manifest.EntryPoint.Type, "dotnet", StringComparison.OrdinalIgnoreCase))
+        {
+            throw new InvalidOperationException($"Unsupported entry point type '{manifest.EntryPoint.Type}'.");
+        }
+
+        if (!manifest.Capabilities.Contains("python", StringComparer.OrdinalIgnoreCase))
+        {
+            throw new InvalidOperationException("Manifest capabilities do not include 'python'.");
+        }
+
+        if (!string.Equals(manifest.EntryPoint.TypeName, "StellaOps.Scanner.Analyzers.Lang.Python.PythonAnalyzerPlugin", StringComparison.Ordinal))
+        {
+            throw new InvalidOperationException($"Unexpected entry point type name '{manifest.EntryPoint.TypeName}'.");
+        }
+
+        if (!string.Equals(manifest.Id, "stellaops.analyzer.lang.python", StringComparison.OrdinalIgnoreCase))
+        {
+            throw new InvalidOperationException($"Manifest id '{manifest.Id}' does not match expected plug-in id for directory '{expectedDirectory}'.");
+        }
+    }
+
+    private static string ComputeSha256(string path)
+    {
+        using var hash = SHA256.Create();
+        using var stream = File.OpenRead(path);
+        var digest = hash.ComputeHash(stream);
+        var builder = new StringBuilder(digest.Length * 2);
+        foreach (var b in digest)
+        {
+            builder.Append(b.ToString("x2"));
+        }
+        return builder.ToString();
+    }
+}