audit notes work completed, test fixes work (95% done), new sprints, new data sources setup and configuration
This commit is contained in:
master
1
src/Cli/__Tests/StellaOps.Cli.Tests/.skip-from-solution
Normal file
1
src/Cli/__Tests/StellaOps.Cli.Tests/.skip-from-solution
Normal file
@@ -0,0 +1 @@
|
||||
This project causes MSBuild hang due to deep dependency tree. Build individually with: dotnet build StellaOps.Cli.Tests.csproj
|
||||
@@ -0,0 +1,404 @@
|
||||
// Copyright (c) StellaOps. All rights reserved.
|
||||
// Licensed under the AGPL-3.0-or-later license.
|
||||
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Cli.Commands.Advise;
|
||||
using StellaOps.Cli.Services.Models.Chat;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Cli.Tests.Commands;
|
||||
|
||||
[Trait("Category", "Unit")]
|
||||
public sealed class AdviseChatCommandTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task RenderQueryResponse_Table_RendersCorrectFormat()
|
||||
{
|
||||
// Arrange
|
||||
var response = CreateSampleQueryResponse();
|
||||
var sb = new StringBuilder();
|
||||
await using var writer = new StringWriter(sb);
|
||||
|
||||
// Act
|
||||
await ChatRenderer.RenderQueryResponseAsync(response, ChatOutputFormat.Table, writer, CancellationToken.None);
|
||||
var output = sb.ToString();
|
||||
|
||||
// Assert
|
||||
Assert.Contains("=== Advisory Chat Response ===", output);
|
||||
Assert.Contains("Response ID: resp-123", output);
|
||||
Assert.Contains("Intent:", output);
|
||||
Assert.Contains("vulnerability_query", output);
|
||||
Assert.Contains("This is a test summary response.", output);
|
||||
Assert.Contains("--- Mitigations ---", output);
|
||||
Assert.Contains("[MIT-001] Update Package", output);
|
||||
Assert.Contains("[RECOMMENDED]", output);
|
||||
Assert.Contains("--- Evidence ---", output);
|
||||
Assert.Contains("[sbom] SBOM Reference", output);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RenderQueryResponse_Json_ReturnsValidJson()
|
||||
{
|
||||
// Arrange
|
||||
var response = CreateSampleQueryResponse();
|
||||
var sb = new StringBuilder();
|
||||
await using var writer = new StringWriter(sb);
|
||||
|
||||
// Act
|
||||
await ChatRenderer.RenderQueryResponseAsync(response, ChatOutputFormat.Json, writer, CancellationToken.None);
|
||||
var output = sb.ToString();
|
||||
|
||||
// Assert
|
||||
Assert.Contains("\"responseId\"", output);
|
||||
Assert.Contains("\"resp-123\"", output);
|
||||
Assert.Contains("\"intent\"", output);
|
||||
Assert.Contains("\"vulnerability_query\"", output);
|
||||
Assert.Contains("\"summary\"", output);
|
||||
Assert.Contains("\"mitigations\"", output);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RenderQueryResponse_Markdown_RendersHeadings()
|
||||
{
|
||||
// Arrange
|
||||
var response = CreateSampleQueryResponse();
|
||||
var sb = new StringBuilder();
|
||||
await using var writer = new StringWriter(sb);
|
||||
|
||||
// Act
|
||||
await ChatRenderer.RenderQueryResponseAsync(response, ChatOutputFormat.Markdown, writer, CancellationToken.None);
|
||||
var output = sb.ToString();
|
||||
|
||||
// Assert
|
||||
Assert.Contains("# Advisory Chat Response", output);
|
||||
Assert.Contains("## Summary", output);
|
||||
Assert.Contains("## Mitigations", output);
|
||||
Assert.Contains("## Evidence", output);
|
||||
Assert.Contains("**(Recommended)**", output);
|
||||
Assert.Contains("| Type | Reference | Label |", output);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RenderDoctorResponse_Table_ShowsQuotasAndTools()
|
||||
{
|
||||
// Arrange
|
||||
var response = CreateSampleDoctorResponse();
|
||||
var sb = new StringBuilder();
|
||||
await using var writer = new StringWriter(sb);
|
||||
|
||||
// Act
|
||||
await ChatRenderer.RenderDoctorResponseAsync(response, ChatOutputFormat.Table, writer, CancellationToken.None);
|
||||
var output = sb.ToString();
|
||||
|
||||
// Assert
|
||||
Assert.Contains("=== Advisory Chat Doctor ===", output);
|
||||
Assert.Contains("Tenant: tenant-001", output);
|
||||
Assert.Contains("User: user-001", output);
|
||||
Assert.Contains("--- Quotas ---", output);
|
||||
Assert.Contains("Requests/Minute:", output);
|
||||
Assert.Contains("--- Tool Access ---", output);
|
||||
Assert.Contains("Allow All: No", output);
|
||||
Assert.Contains("SBOM:", output);
|
||||
Assert.Contains("VEX:", output);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RenderDoctorResponse_Json_ReturnsValidJson()
|
||||
{
|
||||
// Arrange
|
||||
var response = CreateSampleDoctorResponse();
|
||||
var sb = new StringBuilder();
|
||||
await using var writer = new StringWriter(sb);
|
||||
|
||||
// Act
|
||||
await ChatRenderer.RenderDoctorResponseAsync(response, ChatOutputFormat.Json, writer, CancellationToken.None);
|
||||
var output = sb.ToString();
|
||||
|
||||
// Assert
|
||||
Assert.Contains("\"tenantId\"", output);
|
||||
Assert.Contains("\"tenant-001\"", output);
|
||||
Assert.Contains("\"quotas\"", output);
|
||||
Assert.Contains("\"requestsPerMinuteLimit\"", output);
|
||||
Assert.Contains("\"tools\"", output);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RenderSettingsResponse_Table_ShowsEffectiveSettings()
|
||||
{
|
||||
// Arrange
|
||||
var response = CreateSampleSettingsResponse();
|
||||
var sb = new StringBuilder();
|
||||
await using var writer = new StringWriter(sb);
|
||||
|
||||
// Act
|
||||
await ChatRenderer.RenderSettingsResponseAsync(response, ChatOutputFormat.Table, writer, CancellationToken.None);
|
||||
var output = sb.ToString();
|
||||
|
||||
// Assert
|
||||
Assert.Contains("=== Advisory Chat Settings ===", output);
|
||||
Assert.Contains("Tenant: tenant-001", output);
|
||||
Assert.Contains("Scope: effective", output);
|
||||
Assert.Contains("--- Effective Settings ---", output);
|
||||
Assert.Contains("Source: environment", output);
|
||||
Assert.Contains("Quotas:", output);
|
||||
Assert.Contains("Requests/Minute:", output);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RenderSettingsResponse_Json_ReturnsValidJson()
|
||||
{
|
||||
// Arrange
|
||||
var response = CreateSampleSettingsResponse();
|
||||
var sb = new StringBuilder();
|
||||
await using var writer = new StringWriter(sb);
|
||||
|
||||
// Act
|
||||
await ChatRenderer.RenderSettingsResponseAsync(response, ChatOutputFormat.Json, writer, CancellationToken.None);
|
||||
var output = sb.ToString();
|
||||
|
||||
// Assert
|
||||
Assert.Contains("\"tenantId\"", output);
|
||||
Assert.Contains("\"tenant-001\"", output);
|
||||
Assert.Contains("\"effective\"", output);
|
||||
Assert.Contains("\"quotas\"", output);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RenderQueryResponse_WithDeniedActions_ShowsDenialReason()
|
||||
{
|
||||
// Arrange
|
||||
var response = new ChatQueryResponse
|
||||
{
|
||||
ResponseId = "resp-denied",
|
||||
Intent = "action_request",
|
||||
GeneratedAt = DateTimeOffset.UtcNow,
|
||||
Summary = "Action was denied.",
|
||||
Confidence = new ChatConfidence { Overall = 0.9, EvidenceQuality = 0.85, ModelCertainty = 0.95 },
|
||||
ProposedActions =
|
||||
[
|
||||
new ChatProposedAction
|
||||
{
|
||||
Id = "ACT-001",
|
||||
Tool = "vex.update",
|
||||
Description = "Update VEX document",
|
||||
Denied = true,
|
||||
DenyReason = "Tool not in allowlist",
|
||||
RequiresConfirmation = false
|
||||
}
|
||||
]
|
||||
};
|
||||
var sb = new StringBuilder();
|
||||
await using var writer = new StringWriter(sb);
|
||||
|
||||
// Act
|
||||
await ChatRenderer.RenderQueryResponseAsync(response, ChatOutputFormat.Table, writer, CancellationToken.None);
|
||||
var output = sb.ToString();
|
||||
|
||||
// Assert
|
||||
Assert.Contains("--- Proposed Actions ---", output);
|
||||
Assert.Contains("[DENIED]", output);
|
||||
Assert.Contains("Reason: Tool not in allowlist", output);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task RenderDoctorResponse_WithLastDenial_ShowsDenialInfo()
|
||||
{
|
||||
// Arrange
|
||||
var response = new ChatDoctorResponse
|
||||
{
|
||||
TenantId = "tenant-001",
|
||||
UserId = "user-001",
|
||||
Quotas = new ChatQuotaStatus
|
||||
{
|
||||
RequestsPerMinuteLimit = 10,
|
||||
RequestsPerMinuteRemaining = 0,
|
||||
RequestsPerMinuteResetsAt = DateTimeOffset.UtcNow.AddMinutes(1),
|
||||
RequestsPerDayLimit = 100,
|
||||
RequestsPerDayRemaining = 50,
|
||||
RequestsPerDayResetsAt = DateTimeOffset.UtcNow.AddHours(12),
|
||||
TokensPerDayLimit = 50000,
|
||||
TokensPerDayRemaining = 25000,
|
||||
TokensPerDayResetsAt = DateTimeOffset.UtcNow.AddHours(12)
|
||||
},
|
||||
Tools = new ChatToolAccess
|
||||
{
|
||||
AllowAll = false,
|
||||
AllowedTools = ["sbom.read", "vex.query"]
|
||||
},
|
||||
LastDenied = new ChatDenialInfo
|
||||
{
|
||||
Timestamp = DateTimeOffset.UtcNow.AddMinutes(-5),
|
||||
Reason = "Quota exceeded",
|
||||
Code = "QUOTA_EXCEEDED",
|
||||
Query = "What vulnerabilities affect my image?"
|
||||
}
|
||||
};
|
||||
var sb = new StringBuilder();
|
||||
await using var writer = new StringWriter(sb);
|
||||
|
||||
// Act
|
||||
await ChatRenderer.RenderDoctorResponseAsync(response, ChatOutputFormat.Table, writer, CancellationToken.None);
|
||||
var output = sb.ToString();
|
||||
|
||||
// Assert
|
||||
Assert.Contains("--- Last Denial ---", output);
|
||||
Assert.Contains("Reason: Quota exceeded", output);
|
||||
Assert.Contains("Code: QUOTA_EXCEEDED", output);
|
||||
Assert.Contains("Query: What vulnerabilities affect my image?", output);
|
||||
}
|
||||
|
||||
private static ChatQueryResponse CreateSampleQueryResponse()
|
||||
{
|
||||
return new ChatQueryResponse
|
||||
{
|
||||
ResponseId = "resp-123",
|
||||
BundleId = "bundle-456",
|
||||
Intent = "vulnerability_query",
|
||||
GeneratedAt = DateTimeOffset.UtcNow,
|
||||
Summary = "This is a test summary response.",
|
||||
Impact = new ChatImpactAssessment
|
||||
{
|
||||
Severity = "High",
|
||||
AffectedComponents = ["component-a", "component-b"],
|
||||
Description = "Critical vulnerability in component-a."
|
||||
},
|
||||
Reachability = new ChatReachabilityAssessment
|
||||
{
|
||||
Reachable = true,
|
||||
Paths = ["/app/main.js -> /lib/vulnerable.js"],
|
||||
Confidence = 0.92
|
||||
},
|
||||
Mitigations =
|
||||
[
|
||||
new ChatMitigationOption
|
||||
{
|
||||
Id = "MIT-001",
|
||||
Title = "Update Package",
|
||||
Description = "Update the vulnerable package to the latest version.",
|
||||
Effort = "Low",
|
||||
Recommended = true
|
||||
},
|
||||
new ChatMitigationOption
|
||||
{
|
||||
Id = "MIT-002",
|
||||
Title = "Apply Workaround",
|
||||
Description = "Disable the affected feature temporarily.",
|
||||
Effort = "Medium",
|
||||
Recommended = false
|
||||
}
|
||||
],
|
||||
EvidenceLinks =
|
||||
[
|
||||
new ChatEvidenceLink
|
||||
{
|
||||
Type = "sbom",
|
||||
Ref = "sbom:sha256:abc123",
|
||||
Label = "SBOM Reference"
|
||||
},
|
||||
new ChatEvidenceLink
|
||||
{
|
||||
Type = "vex",
|
||||
Ref = "vex:sha256:def456",
|
||||
Label = "VEX Document"
|
||||
}
|
||||
],
|
||||
Confidence = new ChatConfidence
|
||||
{
|
||||
Overall = 0.87,
|
||||
EvidenceQuality = 0.9,
|
||||
ModelCertainty = 0.85
|
||||
},
|
||||
ProposedActions =
|
||||
[
|
||||
new ChatProposedAction
|
||||
{
|
||||
Id = "ACT-001",
|
||||
Tool = "sbom.read",
|
||||
Description = "Read SBOM details",
|
||||
RequiresConfirmation = false,
|
||||
Denied = false
|
||||
}
|
||||
],
|
||||
FollowUp = new ChatFollowUp
|
||||
{
|
||||
SuggestedQueries =
|
||||
[
|
||||
"What is the CVE severity?",
|
||||
"Are there any patches available?"
|
||||
],
|
||||
RelatedTopics = ["CVE-2024-1234", "npm:lodash"]
|
||||
},
|
||||
Diagnostics = new ChatDiagnostics
|
||||
{
|
||||
TokensUsed = 1500,
|
||||
ProcessingTimeMs = 250,
|
||||
EvidenceSourcesQueried = 3
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static ChatDoctorResponse CreateSampleDoctorResponse()
|
||||
{
|
||||
return new ChatDoctorResponse
|
||||
{
|
||||
TenantId = "tenant-001",
|
||||
UserId = "user-001",
|
||||
Quotas = new ChatQuotaStatus
|
||||
{
|
||||
RequestsPerMinuteLimit = 10,
|
||||
RequestsPerMinuteRemaining = 8,
|
||||
RequestsPerMinuteResetsAt = DateTimeOffset.UtcNow.AddSeconds(45),
|
||||
RequestsPerDayLimit = 100,
|
||||
RequestsPerDayRemaining = 75,
|
||||
RequestsPerDayResetsAt = DateTimeOffset.UtcNow.AddHours(12),
|
||||
TokensPerDayLimit = 50000,
|
||||
TokensPerDayRemaining = 35000,
|
||||
TokensPerDayResetsAt = DateTimeOffset.UtcNow.AddHours(12)
|
||||
},
|
||||
Tools = new ChatToolAccess
|
||||
{
|
||||
AllowAll = false,
|
||||
AllowedTools = ["sbom.read", "vex.query", "findings.topk"],
|
||||
Providers = new ChatToolProviders
|
||||
{
|
||||
Sbom = true,
|
||||
Vex = true,
|
||||
Reachability = true,
|
||||
Policy = false,
|
||||
Findings = true
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static ChatSettingsResponse CreateSampleSettingsResponse()
|
||||
{
|
||||
return new ChatSettingsResponse
|
||||
{
|
||||
TenantId = "tenant-001",
|
||||
UserId = "user-001",
|
||||
Scope = "effective",
|
||||
Effective = new ChatEffectiveSettings
|
||||
{
|
||||
Quotas = new ChatQuotaSettings
|
||||
{
|
||||
RequestsPerMinute = 10,
|
||||
RequestsPerDay = 100,
|
||||
TokensPerDay = 50000,
|
||||
ToolCallsPerDay = 500
|
||||
},
|
||||
Tools = new ChatToolSettings
|
||||
{
|
||||
AllowAll = false,
|
||||
AllowedTools = ["sbom.read", "vex.query"]
|
||||
},
|
||||
Source = "environment"
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -1,132 +0,0 @@
|
||||
using System.CommandLine;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using StellaOps.Cli.Commands.Scan;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Cli.Tests.Commands;
|
||||
|
||||
public sealed class BinaryDiffCommandTests
|
||||
{
|
||||
private readonly IServiceProvider _services;
|
||||
private readonly Option<bool> _verboseOption;
|
||||
private readonly CancellationToken _cancellationToken;
|
||||
|
||||
public BinaryDiffCommandTests()
|
||||
{
|
||||
_services = new ServiceCollection().BuildServiceProvider();
|
||||
_verboseOption = new Option<bool>("--verbose", new[] { "-v" })
|
||||
{
|
||||
Description = "Enable verbose output"
|
||||
};
|
||||
_cancellationToken = CancellationToken.None;
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildDiffCommand_HasRequiredOptions()
|
||||
{
|
||||
var command = BuildDiffCommand();
|
||||
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--base", "-b"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--target", "-t"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--mode", "-m"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--emit-dsse", "-d"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--signing-key"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--format", "-f"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--platform", "-p"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--include-unchanged"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--sections"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--registry-auth"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--timeout"));
|
||||
Assert.Contains(command.Options, option => HasAlias(option, "--verbose", "-v"));
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void BuildDiffCommand_RequiresBaseAndTarget()
|
||||
{
|
||||
var command = BuildDiffCommand();
|
||||
var baseOption = FindOption(command, "--base");
|
||||
var targetOption = FindOption(command, "--target");
|
||||
|
||||
Assert.NotNull(baseOption);
|
||||
Assert.NotNull(targetOption);
|
||||
Assert.True(baseOption!.IsRequired);
|
||||
Assert.True(targetOption!.IsRequired);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DiffCommand_ParsesMinimalArgs()
|
||||
{
|
||||
var root = BuildRoot(out _);
|
||||
|
||||
var result = root.Parse("scan diff --base registry.example.com/app:1 --target registry.example.com/app:2");
|
||||
|
||||
Assert.Empty(result.Errors);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DiffCommand_FailsWhenBaseMissing()
|
||||
{
|
||||
var root = BuildRoot(out _);
|
||||
|
||||
var result = root.Parse("scan diff --target registry.example.com/app:2");
|
||||
|
||||
Assert.NotEmpty(result.Errors);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public void DiffCommand_ParsesSectionsValues()
|
||||
{
|
||||
var root = BuildRoot(out var diffCommand);
|
||||
|
||||
var result = root.Parse("scan diff --base registry.example.com/app:1 --target registry.example.com/app:2 --sections .text,.rodata --sections .data");
|
||||
|
||||
Assert.Empty(result.Errors);
|
||||
|
||||
var sectionsOption = diffCommand.Options
|
||||
.OfType<Option<string[]>>()
|
||||
.Single(option => HasAlias(option, "--sections"));
|
||||
var values = result.GetValueForOption(sectionsOption);
|
||||
|
||||
Assert.Contains(".text,.rodata", values);
|
||||
Assert.Contains(".data", values);
|
||||
Assert.True(sectionsOption.AllowMultipleArgumentsPerToken);
|
||||
}
|
||||
|
||||
private Command BuildDiffCommand()
|
||||
{
|
||||
return BinaryDiffCommandGroup.BuildDiffCommand(_services, _verboseOption, _cancellationToken);
|
||||
}
|
||||
|
||||
private RootCommand BuildRoot(out Command diffCommand)
|
||||
{
|
||||
diffCommand = BuildDiffCommand();
|
||||
var scan = new Command("scan", "Scanner operations")
|
||||
{
|
||||
diffCommand
|
||||
};
|
||||
return new RootCommand { scan };
|
||||
}
|
||||
|
||||
private static Option? FindOption(Command command, string alias)
|
||||
{
|
||||
return command.Options.FirstOrDefault(option =>
|
||||
option.Name.Equals(alias, StringComparison.OrdinalIgnoreCase) ||
|
||||
option.Name.Equals(alias.TrimStart('-'), StringComparison.OrdinalIgnoreCase) ||
|
||||
option.Aliases.Contains(alias));
|
||||
}
|
||||
|
||||
private static bool HasAlias(Option option, params string[] aliases)
|
||||
{
|
||||
foreach (var alias in aliases)
|
||||
{
|
||||
if (option.Name.Equals(alias, StringComparison.OrdinalIgnoreCase) ||
|
||||
option.Name.Equals(alias.TrimStart('-'), StringComparison.OrdinalIgnoreCase) ||
|
||||
option.Aliases.Contains(alias))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -37,3 +37,4 @@
|
||||
<ProjectReference Include="../../../__Libraries/StellaOps.TestKit/StellaOps.TestKit.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
|
||||
|
||||
Reference in New Issue
Block a user