feat: Add UI benchmark driver and scenarios for graph interactions
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
devportal-offline / build-offline (push) Has been cancelled
- Introduced `ui_bench_driver.mjs` to read scenarios and fixture manifest, generating a deterministic run plan. - Created `ui_bench_plan.md` outlining the purpose, scope, and next steps for the benchmark. - Added `ui_bench_scenarios.json` containing various scenarios for graph UI interactions. - Implemented tests for CLI commands, ensuring bundle verification and telemetry defaults. - Developed schemas for orchestrator components, including replay manifests and event envelopes. - Added mock API for risk management, including listing and statistics functionalities. - Implemented models for risk profiles and query options to support the new API.
This commit is contained in:
@@ -48,17 +48,43 @@ namespace StellaOps.Cli.Commands;
|
||||
internal static class CommandHandlers
|
||||
{
|
||||
private const string KmsPassphraseEnvironmentVariable = "STELLAOPS_KMS_PASSPHRASE";
|
||||
private static readonly JsonSerializerOptions KmsJsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = true
|
||||
};
|
||||
private static readonly JsonSerializerOptions KmsJsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
WriteIndented = true
|
||||
};
|
||||
|
||||
private static async Task VerifyBundleAsync(string path, ILogger logger, CancellationToken cancellationToken)
|
||||
{
|
||||
// Simple SHA256 check using sidecar .sha256 file if present; fail closed on mismatch.
|
||||
var shaPath = path + ".sha256";
|
||||
if (!File.Exists(shaPath))
|
||||
{
|
||||
logger.LogError("Checksum file missing for bundle {Bundle}. Expected sidecar {Sidecar}.", path, shaPath);
|
||||
Environment.ExitCode = 21;
|
||||
throw new InvalidOperationException("Checksum file missing");
|
||||
}
|
||||
|
||||
var expected = (await File.ReadAllTextAsync(shaPath, cancellationToken).ConfigureAwait(false)).Trim();
|
||||
using var stream = File.OpenRead(path);
|
||||
var hash = await SHA256.HashDataAsync(stream, cancellationToken).ConfigureAwait(false);
|
||||
var actual = Convert.ToHexString(hash).ToLowerInvariant();
|
||||
|
||||
if (!string.Equals(expected, actual, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
logger.LogError("Checksum mismatch for {Bundle}. Expected {Expected} but found {Actual}", path, expected, actual);
|
||||
Environment.ExitCode = 22;
|
||||
throw new InvalidOperationException("Checksum verification failed");
|
||||
}
|
||||
|
||||
logger.LogInformation("Checksum verified for {Bundle}", path);
|
||||
}
|
||||
|
||||
public static async Task HandleScannerDownloadAsync(
|
||||
IServiceProvider services,
|
||||
string channel,
|
||||
string? output,
|
||||
bool overwrite,
|
||||
bool install,
|
||||
public static async Task HandleScannerDownloadAsync(
|
||||
IServiceProvider services,
|
||||
string channel,
|
||||
string? output,
|
||||
bool overwrite,
|
||||
bool install,
|
||||
bool verbose,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
@@ -88,24 +114,29 @@ internal static class CommandHandlers
|
||||
|
||||
CliMetrics.RecordScannerDownload(channel, result.FromCache);
|
||||
|
||||
if (install)
|
||||
{
|
||||
var installer = scope.ServiceProvider.GetRequiredService<IScannerInstaller>();
|
||||
await installer.InstallAsync(result.Path, verbose, cancellationToken).ConfigureAwait(false);
|
||||
CliMetrics.RecordScannerInstall(channel);
|
||||
}
|
||||
if (install)
|
||||
{
|
||||
await VerifyBundleAsync(result.Path, logger, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var installer = scope.ServiceProvider.GetRequiredService<IScannerInstaller>();
|
||||
await installer.InstallAsync(result.Path, verbose, cancellationToken).ConfigureAwait(false);
|
||||
CliMetrics.RecordScannerInstall(channel);
|
||||
}
|
||||
|
||||
Environment.ExitCode = 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to download scanner bundle.");
|
||||
Environment.ExitCode = 1;
|
||||
}
|
||||
finally
|
||||
{
|
||||
verbosity.MinimumLevel = previousLevel;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
logger.LogError(ex, "Failed to download scanner bundle.");
|
||||
if (Environment.ExitCode == 0)
|
||||
{
|
||||
Environment.ExitCode = 1;
|
||||
}
|
||||
}
|
||||
finally
|
||||
{
|
||||
verbosity.MinimumLevel = previousLevel;
|
||||
}
|
||||
}
|
||||
|
||||
public static async Task HandleTaskRunnerSimulateAsync(
|
||||
|
||||
@@ -107,9 +107,9 @@ public sealed class CliProfileStore
|
||||
public Dictionary<string, CliProfile> Profiles { get; init; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
/// <summary>
|
||||
/// Default telemetry opt-in status.
|
||||
/// Default telemetry opt-in status. Defaults to false for privacy.
|
||||
/// </summary>
|
||||
public bool? TelemetryEnabled { get; set; }
|
||||
public bool TelemetryEnabled { get; set; } = false;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -225,7 +225,7 @@ public sealed class CliProfileManager
|
||||
public async Task SetTelemetryEnabledAsync(bool? enabled, CancellationToken cancellationToken = default)
|
||||
{
|
||||
var store = await GetStoreAsync(cancellationToken).ConfigureAwait(false);
|
||||
store.TelemetryEnabled = enabled;
|
||||
store.TelemetryEnabled = enabled ?? false;
|
||||
await SaveStoreAsync(store, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
@@ -247,7 +247,18 @@ public sealed class CliProfileManager
|
||||
{
|
||||
await using var stream = File.OpenRead(_profilesFilePath);
|
||||
var store = await JsonSerializer.DeserializeAsync<CliProfileStore>(stream, JsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
return store ?? new CliProfileStore();
|
||||
if (store is null)
|
||||
{
|
||||
return new CliProfileStore();
|
||||
}
|
||||
|
||||
// Ensure default-off if older files had telemetryEnabled missing/null.
|
||||
if (!store.TelemetryEnabled)
|
||||
{
|
||||
store.TelemetryEnabled = false;
|
||||
}
|
||||
|
||||
return store;
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
|
||||
@@ -4512,11 +4512,11 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient
|
||||
}
|
||||
|
||||
// CLI-SDK-64-001: SDK update operations
|
||||
public async Task<SdkUpdateResponse> CheckSdkUpdatesAsync(SdkUpdateRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
EnsureBackendConfigured();
|
||||
public async Task<SdkUpdateResponse> CheckSdkUpdatesAsync(SdkUpdateRequest request, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(request);
|
||||
|
||||
EnsureBackendConfigured();
|
||||
OfflineModeGuard.ThrowIfOffline("sdk update");
|
||||
|
||||
var queryParams = new List<string>();
|
||||
@@ -4554,9 +4554,9 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient
|
||||
};
|
||||
}
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<SdkUpdateResponse>(JsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
return result ?? new SdkUpdateResponse { Success = false, Error = "Empty response" };
|
||||
}
|
||||
var result = await response.Content.ReadFromJsonAsync<SdkUpdateResponse>(JsonOptions, cancellationToken).ConfigureAwait(false);
|
||||
return result ?? new SdkUpdateResponse { Success = false, Error = "Empty response" };
|
||||
}
|
||||
|
||||
public async Task<SdkListResponse> ListInstalledSdksAsync(string? language, string? tenant, CancellationToken cancellationToken)
|
||||
{
|
||||
|
||||
Reference in New Issue
Block a user