Add Policy DSL Validator, Schema Exporter, and Simulation Smoke tools
- Implemented PolicyDslValidator with command-line options for strict mode and JSON output. - Created PolicySchemaExporter to generate JSON schemas for policy-related models. - Developed PolicySimulationSmoke tool to validate policy simulations against expected outcomes. - Added project files and necessary dependencies for each tool. - Ensured proper error handling and usage instructions across tools.
This commit is contained in:
master
18
tools/LanguageAnalyzerSmoke/LanguageAnalyzerSmoke.csproj
Normal file
18
tools/LanguageAnalyzerSmoke/LanguageAnalyzerSmoke.csproj
Normal file
@@ -0,0 +1,18 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\StellaOps.Scanner.Analyzers.Lang\StellaOps.Scanner.Analyzers.Lang.csproj" />
|
||||
<ProjectReference Include="..\..\src\StellaOps.Scanner.Core\StellaOps.Scanner.Core.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
348
tools/LanguageAnalyzerSmoke/Program.cs
Normal file
348
tools/LanguageAnalyzerSmoke/Program.cs
Normal file
@@ -0,0 +1,348 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Plugin;
|
||||
using StellaOps.Scanner.Core.Security;
|
||||
|
||||
internal sealed record SmokeScenario(string Name, string[] UsageHintRelatives)
|
||||
{
|
||||
public IReadOnlyList<string> ResolveUsageHints(string scenarioRoot)
|
||||
=> UsageHintRelatives.Select(relative => Path.GetFullPath(Path.Combine(scenarioRoot, relative))).ToArray();
|
||||
}
|
||||
|
||||
internal sealed class SmokeOptions
|
||||
{
|
||||
public string RepoRoot { get; set; } = Directory.GetCurrentDirectory();
|
||||
public string PluginDirectoryName { get; set; } = "StellaOps.Scanner.Analyzers.Lang.Python";
|
||||
public string FixtureRelativePath { get; set; } = Path.Combine("src", "StellaOps.Scanner.Analyzers.Lang.Python.Tests", "Fixtures", "lang", "python");
|
||||
|
||||
public static SmokeOptions Parse(string[] args)
|
||||
{
|
||||
var options = new SmokeOptions();
|
||||
|
||||
for (var index = 0; index < args.Length; index++)
|
||||
{
|
||||
var current = args[index];
|
||||
switch (current)
|
||||
{
|
||||
case "--repo-root":
|
||||
case "-r":
|
||||
options.RepoRoot = RequireValue(args, ref index, current);
|
||||
break;
|
||||
case "--plugin-directory":
|
||||
case "-p":
|
||||
options.PluginDirectoryName = RequireValue(args, ref index, current);
|
||||
break;
|
||||
case "--fixture-path":
|
||||
case "-f":
|
||||
options.FixtureRelativePath = RequireValue(args, ref index, current);
|
||||
break;
|
||||
case "--help":
|
||||
case "-h":
|
||||
PrintUsage();
|
||||
Environment.Exit(0);
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentException($"Unknown argument '{current}'. Use --help for usage.");
|
||||
}
|
||||
}
|
||||
|
||||
options.RepoRoot = Path.GetFullPath(options.RepoRoot);
|
||||
return options;
|
||||
}
|
||||
|
||||
private static string RequireValue(string[] args, ref int index, string switchName)
|
||||
{
|
||||
if (index + 1 >= args.Length)
|
||||
{
|
||||
throw new ArgumentException($"Missing value for '{switchName}'.");
|
||||
}
|
||||
|
||||
index++;
|
||||
var value = args[index];
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
throw new ArgumentException($"Value for '{switchName}' cannot be empty.");
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
private static void PrintUsage()
|
||||
{
|
||||
Console.WriteLine("Language Analyzer Smoke Harness");
|
||||
Console.WriteLine("Usage: dotnet run --project tools/LanguageAnalyzerSmoke -- [options]");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("Options:");
|
||||
Console.WriteLine(" -r, --repo-root <path> Repository root (defaults to current working directory)");
|
||||
Console.WriteLine(" -p, --plugin-directory <name> Analyzer plug-in directory under plugins/scanner/analyzers/lang (defaults to StellaOps.Scanner.Analyzers.Lang.Python)");
|
||||
Console.WriteLine(" -f, --fixture-path <path> Relative path to fixtures root (defaults to src/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python)");
|
||||
Console.WriteLine(" -h, --help Show usage information");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record PluginManifest
|
||||
{
|
||||
[JsonPropertyName("schemaVersion")]
|
||||
public string SchemaVersion { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("id")]
|
||||
public string Id { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("displayName")]
|
||||
public string DisplayName { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("requiresRestart")]
|
||||
public bool RequiresRestart { get; init; }
|
||||
|
||||
[JsonPropertyName("entryPoint")]
|
||||
public PluginEntryPoint EntryPoint { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("capabilities")]
|
||||
public IReadOnlyList<string> Capabilities { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("metadata")]
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; init; } = ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
internal sealed record PluginEntryPoint
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("assembly")]
|
||||
public string Assembly { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("typeName")]
|
||||
public string TypeName { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
file static class Program
|
||||
{
|
||||
private static readonly SmokeScenario[] PythonScenarios =
|
||||
{
|
||||
new("simple-venv", new[] { Path.Combine("bin", "simple-tool") }),
|
||||
new("pip-cache", new[] { Path.Combine("lib", "python3.11", "site-packages", "cache_pkg-1.2.3.data", "scripts", "cache-tool") }),
|
||||
new("layered-editable", new[] { Path.Combine("layer1", "usr", "bin", "layered-cli") })
|
||||
};
|
||||
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
try
|
||||
{
|
||||
var options = SmokeOptions.Parse(args);
|
||||
await RunAsync(options).ConfigureAwait(false);
|
||||
Console.WriteLine("✅ Python analyzer smoke checks passed");
|
||||
return 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"❌ {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task RunAsync(SmokeOptions options)
|
||||
{
|
||||
ValidateOptions(options);
|
||||
|
||||
var pluginRoot = Path.Combine(options.RepoRoot, "plugins", "scanner", "analyzers", "lang", options.PluginDirectoryName);
|
||||
var manifestPath = Path.Combine(pluginRoot, "manifest.json");
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Plug-in manifest not found at '{manifestPath}'.", manifestPath);
|
||||
}
|
||||
|
||||
using var manifestStream = File.OpenRead(manifestPath);
|
||||
var manifest = JsonSerializer.Deserialize<PluginManifest>(manifestStream, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip
|
||||
}) ?? throw new InvalidOperationException($"Unable to parse manifest '{manifestPath}'.");
|
||||
|
||||
ValidateManifest(manifest, options.PluginDirectoryName);
|
||||
|
||||
var pluginAssemblyPath = Path.Combine(pluginRoot, manifest.EntryPoint.Assembly);
|
||||
if (!File.Exists(pluginAssemblyPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Plug-in assembly '{manifest.EntryPoint.Assembly}' not found under '{pluginRoot}'.", pluginAssemblyPath);
|
||||
}
|
||||
|
||||
var sha256 = ComputeSha256(pluginAssemblyPath);
|
||||
Console.WriteLine($"→ Plug-in assembly SHA-256: {sha256}");
|
||||
|
||||
using var serviceProvider = BuildServiceProvider();
|
||||
var catalog = new LanguageAnalyzerPluginCatalog(new RestartOnlyPluginGuard(), NullLogger<LanguageAnalyzerPluginCatalog>.Instance);
|
||||
catalog.LoadFromDirectory(pluginRoot, seal: true);
|
||||
|
||||
if (catalog.Plugins.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No analyzer plug-ins were loaded from '{pluginRoot}'.");
|
||||
}
|
||||
|
||||
var analyzerSet = catalog.CreateAnalyzers(serviceProvider);
|
||||
if (analyzerSet.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Language analyzer plug-ins reported no analyzers.");
|
||||
}
|
||||
|
||||
var analyzerIds = analyzerSet.Select(analyzer => analyzer.Id).ToArray();
|
||||
Console.WriteLine($"→ Loaded analyzers: {string.Join(", ", analyzerIds)}");
|
||||
|
||||
if (!analyzerIds.Contains("python", StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Python analyzer was not created by the plug-in.");
|
||||
}
|
||||
|
||||
var fixtureRoot = Path.GetFullPath(Path.Combine(options.RepoRoot, options.FixtureRelativePath));
|
||||
if (!Directory.Exists(fixtureRoot))
|
||||
{
|
||||
throw new DirectoryNotFoundException($"Fixture directory '{fixtureRoot}' does not exist.");
|
||||
}
|
||||
|
||||
foreach (var scenario in PythonScenarios)
|
||||
{
|
||||
await RunScenarioAsync(scenario, fixtureRoot, catalog, serviceProvider).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static ServiceProvider BuildServiceProvider()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static async Task RunScenarioAsync(SmokeScenario scenario, string fixtureRoot, ILanguageAnalyzerPluginCatalog catalog, IServiceProvider services)
|
||||
{
|
||||
var scenarioRoot = Path.Combine(fixtureRoot, scenario.Name);
|
||||
if (!Directory.Exists(scenarioRoot))
|
||||
{
|
||||
throw new DirectoryNotFoundException($"Scenario '{scenario.Name}' directory missing at '{scenarioRoot}'.");
|
||||
}
|
||||
|
||||
var goldenPath = Path.Combine(scenarioRoot, "expected.json");
|
||||
string? goldenNormalized = null;
|
||||
if (File.Exists(goldenPath))
|
||||
{
|
||||
goldenNormalized = NormalizeJson(await File.ReadAllTextAsync(goldenPath).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
var usageHints = new LanguageUsageHints(scenario.ResolveUsageHints(scenarioRoot));
|
||||
var context = new LanguageAnalyzerContext(scenarioRoot, TimeProvider.System, usageHints, services);
|
||||
|
||||
var coldEngine = new LanguageAnalyzerEngine(catalog.CreateAnalyzers(services));
|
||||
var coldStopwatch = Stopwatch.StartNew();
|
||||
var coldResult = await coldEngine.AnalyzeAsync(context, CancellationToken.None).ConfigureAwait(false);
|
||||
coldStopwatch.Stop();
|
||||
|
||||
if (coldResult.Components.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Scenario '{scenario.Name}' produced no components during cold run.");
|
||||
}
|
||||
|
||||
var coldJson = NormalizeJson(coldResult.ToJson(indent: true));
|
||||
if (goldenNormalized is string expected && !string.Equals(coldJson, expected, StringComparison.Ordinal))
|
||||
{
|
||||
Console.WriteLine($"⚠️ Scenario '{scenario.Name}' output deviates from repository golden snapshot.");
|
||||
}
|
||||
|
||||
var warmEngine = new LanguageAnalyzerEngine(catalog.CreateAnalyzers(services));
|
||||
var warmStopwatch = Stopwatch.StartNew();
|
||||
var warmResult = await warmEngine.AnalyzeAsync(context, CancellationToken.None).ConfigureAwait(false);
|
||||
warmStopwatch.Stop();
|
||||
|
||||
var warmJson = NormalizeJson(warmResult.ToJson(indent: true));
|
||||
if (!string.Equals(coldJson, warmJson, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"Scenario '{scenario.Name}' produced different outputs between cold and warm runs.");
|
||||
}
|
||||
|
||||
EnsureDurationWithinBudget(scenario.Name, coldStopwatch.Elapsed, warmStopwatch.Elapsed);
|
||||
|
||||
Console.WriteLine($"✓ Scenario '{scenario.Name}' — components {coldResult.Components.Count}, cold {coldStopwatch.Elapsed.TotalMilliseconds:F1} ms, warm {warmStopwatch.Elapsed.TotalMilliseconds:F1} ms");
|
||||
}
|
||||
|
||||
private static void EnsureDurationWithinBudget(string scenarioName, TimeSpan coldDuration, TimeSpan warmDuration)
|
||||
{
|
||||
var coldBudget = TimeSpan.FromSeconds(30);
|
||||
var warmBudget = TimeSpan.FromSeconds(5);
|
||||
|
||||
if (coldDuration > coldBudget)
|
||||
{
|
||||
throw new InvalidOperationException($"Scenario '{scenarioName}' cold run exceeded budget ({coldDuration.TotalSeconds:F2}s > {coldBudget.TotalSeconds:F2}s).");
|
||||
}
|
||||
|
||||
if (warmDuration > warmBudget)
|
||||
{
|
||||
throw new InvalidOperationException($"Scenario '{scenarioName}' warm run exceeded budget ({warmDuration.TotalSeconds:F2}s > {warmBudget.TotalSeconds:F2}s).");
|
||||
}
|
||||
}
|
||||
|
||||
private static string NormalizeJson(string json)
|
||||
=> json.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd();
|
||||
|
||||
private static void ValidateOptions(SmokeOptions options)
|
||||
{
|
||||
if (!Directory.Exists(options.RepoRoot))
|
||||
{
|
||||
throw new DirectoryNotFoundException($"Repository root '{options.RepoRoot}' does not exist.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateManifest(PluginManifest manifest, string expectedDirectory)
|
||||
{
|
||||
if (!string.Equals(manifest.SchemaVersion, "1.0", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"Unexpected manifest schema version '{manifest.SchemaVersion}'.");
|
||||
}
|
||||
|
||||
if (!manifest.RequiresRestart)
|
||||
{
|
||||
throw new InvalidOperationException("Language analyzer plug-in must be marked as restart-only.");
|
||||
}
|
||||
|
||||
if (!string.Equals(manifest.EntryPoint.Type, "dotnet", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Unsupported entry point type '{manifest.EntryPoint.Type}'.");
|
||||
}
|
||||
|
||||
if (!manifest.Capabilities.Contains("python", StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Manifest capabilities do not include 'python'.");
|
||||
}
|
||||
|
||||
if (!string.Equals(manifest.EntryPoint.TypeName, "StellaOps.Scanner.Analyzers.Lang.Python.PythonAnalyzerPlugin", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"Unexpected entry point type name '{manifest.EntryPoint.TypeName}'.");
|
||||
}
|
||||
|
||||
if (!string.Equals(manifest.Id, "stellaops.analyzer.lang.python", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Manifest id '{manifest.Id}' does not match expected plug-in id for directory '{expectedDirectory}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256(string path)
|
||||
{
|
||||
using var hash = SHA256.Create();
|
||||
using var stream = File.OpenRead(path);
|
||||
var digest = hash.ComputeHash(stream);
|
||||
var builder = new StringBuilder(digest.Length * 2);
|
||||
foreach (var b in digest)
|
||||
{
|
||||
builder.Append(b.ToString("x2"));
|
||||
}
|
||||
return builder.ToString();
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user