audit, advisories and doctors/setup work

This commit is contained in:
master
2026-01-13 18:53:39 +02:00
parent 9ca7cb183e
commit d7be6ba34b
811 changed files with 54242 additions and 4056 deletions

View File

@@ -86,6 +86,17 @@ public sealed class AdvisoryGuardrailInjectionTests
options.RequireCitations = testCase.RequireCitations.Value;
}
if (testCase.AllowlistPatterns is { Length: > 0 })
{
foreach (var pattern in testCase.AllowlistPatterns)
{
if (!string.IsNullOrWhiteSpace(pattern))
{
options.AllowlistPatterns.Add(pattern);
}
}
}
return options;
}
@@ -200,5 +211,9 @@ public sealed class AdvisoryGuardrailInjectionTests
[JsonPropertyName("expectRedactionPlaceholder")]
public bool ExpectRedactionPlaceholder { get; init; }
= false;
[JsonPropertyName("allowlistPatterns")]
public string[]? AllowlistPatterns { get; init; }
= null;
}
}

View File

@@ -25,6 +25,8 @@ public sealed class AdvisoryGuardrailOptionsBindingTests
var tempRoot = CreateTempDirectory();
var phrasePath = Path.Combine(tempRoot, "guardrail-phrases.json");
await File.WriteAllTextAsync(phrasePath, "{\n \"phrases\": [\"extract secrets\", \"dump cache\"]\n}");
var allowlistPath = Path.Combine(tempRoot, "guardrail-allowlist.txt");
await File.WriteAllTextAsync(allowlistPath, "sha256:[0-9a-f]{64}\nscan:[A-Za-z0-9_-]{16,}\n");
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
@@ -32,7 +34,11 @@ public sealed class AdvisoryGuardrailOptionsBindingTests
["AdvisoryAI:Guardrails:MaxPromptLength"] = "32000",
["AdvisoryAI:Guardrails:RequireCitations"] = "false",
["AdvisoryAI:Guardrails:BlockedPhraseFile"] = "guardrail-phrases.json",
["AdvisoryAI:Guardrails:BlockedPhrases:0"] = "custom override"
["AdvisoryAI:Guardrails:BlockedPhrases:0"] = "custom override",
["AdvisoryAI:Guardrails:AllowlistFile"] = "guardrail-allowlist.txt",
["AdvisoryAI:Guardrails:AllowlistPatterns:0"] = "custom-allowlist",
["AdvisoryAI:Guardrails:EntropyThreshold"] = "3.9",
["AdvisoryAI:Guardrails:EntropyMinLength"] = "24"
})
.Build();
@@ -48,6 +54,11 @@ public sealed class AdvisoryGuardrailOptionsBindingTests
options.BlockedPhrases.Should().Contain("custom override");
options.BlockedPhrases.Should().Contain("extract secrets");
options.BlockedPhrases.Should().Contain("dump cache");
options.EntropyThreshold.Should().Be(3.9);
options.EntropyMinLength.Should().Be(24);
options.AllowlistPatterns.Should().Contain("custom-allowlist");
options.AllowlistPatterns.Should().Contain("sha256:[0-9a-f]{64}");
options.AllowlistPatterns.Should().Contain("scan:[A-Za-z0-9_-]{16,}");
}
[Trait("Category", TestCategories.Unit)]
@@ -71,6 +82,27 @@ public sealed class AdvisoryGuardrailOptionsBindingTests
action.Should().Throw<FileNotFoundException>();
}
[Trait("Category", TestCategories.Unit)]
[Fact]
public async Task AddAdvisoryAiCore_ThrowsWhenAllowlistFileMissing()
{
var tempRoot = CreateTempDirectory();
var configuration = new ConfigurationBuilder()
.AddInMemoryCollection(new Dictionary<string, string?>
{
["AdvisoryAI:Guardrails:AllowlistFile"] = "missing.txt"
})
.Build();
var services = new ServiceCollection();
services.AddSingleton<IHostEnvironment>(new FakeHostEnvironment(tempRoot));
services.AddAdvisoryAiCore(configuration);
await using var provider = services.BuildServiceProvider();
var action = () => provider.GetRequiredService<IOptions<AdvisoryGuardrailOptions>>().Value;
action.Should().Throw<FileNotFoundException>();
}
private static string CreateTempDirectory()
{
var path = Path.Combine(Path.GetTempPath(), "advisoryai-guardrails", Guid.NewGuid().ToString("n"));

View File

@@ -331,6 +331,9 @@ public sealed class AdvisoryPipelineExecutorTests : IDisposable
public Task<AdvisoryGuardrailResult> EvaluateAsync(AdvisoryPrompt prompt, CancellationToken cancellationToken)
=> Task.FromResult(_result);
public AdvisoryRedactionResult Redact(string input)
=> new(input ?? string.Empty, 0);
}
private sealed class StubInferenceClient : IAdvisoryInferenceClient

View File

@@ -0,0 +1,241 @@
// <copyright file="AdvisoryChatAuditEnvelopeBuilderTests.cs" company="StellaOps">
// Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later.
// </copyright>
using System.Collections.Immutable;
using StellaOps.AdvisoryAI.Chat.Audit;
using StellaOps.AdvisoryAI.Chat.Models;
using StellaOps.AdvisoryAI.Chat.Routing;
using StellaOps.AdvisoryAI.Chat.Services;
using StellaOps.AdvisoryAI.Chat.Settings;
using StellaOps.AdvisoryAI.Guardrails;
using Xunit;
namespace StellaOps.AdvisoryAI.Tests.Chat.Audit;
[Trait("Category", "Unit")]
public sealed class AdvisoryChatAuditEnvelopeBuilderTests
{
[Fact]
public void BuildSuccess_RecordsEvidenceAndDecisions()
{
var now = new DateTimeOffset(2026, 1, 13, 12, 0, 0, TimeSpan.Zero);
var request = BuildRequest();
var routing = BuildRouting();
var response = BuildResponse(now);
var diagnostics = new AdvisoryChatDiagnostics
{
PromptTokens = 12,
CompletionTokens = 34,
TotalMs = 50
};
var toolPolicy = BuildToolPolicy();
var quotaStatus = BuildQuotaStatus(now);
var evidenceBundle = BuildEvidenceBundle(now);
var envelope = AdvisoryChatAuditEnvelopeBuilder.BuildSuccess(
request,
routing,
"sanitized prompt",
evidenceBundle,
response,
diagnostics,
quotaStatus,
toolPolicy,
now,
includeEvidenceBundle: false);
Assert.Equal("success", envelope.Session.Decision);
Assert.Equal(request.TenantId, envelope.Session.TenantId);
Assert.Equal(2, envelope.Messages.Length);
Assert.Single(envelope.EvidenceLinks);
Assert.Single(envelope.ToolInvocations);
Assert.Contains(envelope.PolicyDecisions, decision => decision.PolicyType == "tool_access");
Assert.Null(envelope.Session.EvidenceBundleJson);
}
[Fact]
public void BuildGuardrailBlocked_RecordsDenialAndToolPolicy()
{
var now = new DateTimeOffset(2026, 1, 13, 12, 0, 0, TimeSpan.Zero);
var request = BuildRequest();
var routing = BuildRouting();
var toolPolicy = BuildToolPolicy();
var quotaStatus = BuildQuotaStatus(now);
var guardrailResult = AdvisoryGuardrailResult.Reject(
"sanitized prompt",
[new AdvisoryGuardrailViolation("prompt_too_long", "Prompt too long.")],
ImmutableDictionary<string, string>.Empty.Add("redaction_count", "2"));
var envelope = AdvisoryChatAuditEnvelopeBuilder.BuildGuardrailBlocked(
request,
routing,
guardrailResult.SanitizedPrompt,
guardrailResult,
BuildEvidenceBundle(now),
toolPolicy,
quotaStatus,
now,
includeEvidenceBundle: false);
Assert.Equal("guardrail_blocked", envelope.Session.Decision);
Assert.Equal("GUARDRAIL_BLOCKED", envelope.Session.DecisionCode);
Assert.Single(envelope.Messages);
Assert.Equal(toolPolicy.AllowedTools.Length, envelope.ToolInvocations.Length);
Assert.Contains(envelope.PolicyDecisions, decision => decision.PolicyType == "guardrail" && decision.Decision == "deny");
}
[Fact]
public void BuildQuotaDenied_RecordsQuotaDecision()
{
var now = new DateTimeOffset(2026, 1, 13, 12, 0, 0, TimeSpan.Zero);
var request = BuildRequest();
var routing = BuildRouting();
var toolPolicy = BuildToolPolicy();
var decision = new ChatQuotaDecision
{
Allowed = false,
Code = "TOKENS_PER_DAY_EXCEEDED",
Message = "Quota exceeded.",
Status = BuildQuotaStatus(now)
};
var envelope = AdvisoryChatAuditEnvelopeBuilder.BuildQuotaDenied(
request,
routing,
new AdvisoryRedactionResult("sanitized", 1),
decision,
toolPolicy,
now);
Assert.Equal("quota_denied", envelope.Session.Decision);
Assert.Equal(decision.Code, envelope.Session.DecisionCode);
Assert.Contains(envelope.PolicyDecisions, policy => policy.PolicyType == "quota" && policy.Decision == "deny");
Assert.Contains(envelope.PolicyDecisions, policy => policy.PolicyType == "tool_access");
}
[Fact]
public void BuildToolAccessDenied_RecordsToolPolicyDecision()
{
var now = new DateTimeOffset(2026, 1, 13, 12, 0, 0, TimeSpan.Zero);
var request = BuildRequest();
var routing = BuildRouting();
var toolPolicy = BuildToolPolicy();
var envelope = AdvisoryChatAuditEnvelopeBuilder.BuildToolAccessDenied(
request,
routing,
new AdvisoryRedactionResult("sanitized", 2),
toolPolicy,
"sbom.read not allowed",
now);
Assert.Equal("tool_access_denied", envelope.Session.Decision);
Assert.Equal("sbom.read not allowed", envelope.Session.DecisionReason);
Assert.Contains(envelope.PolicyDecisions, policy => policy.PolicyType == "tool_access" && policy.Decision == "deny");
}
private static AdvisoryChatRequest BuildRequest()
=> new()
{
TenantId = "tenant-1",
UserId = "user-1",
Query = "Why is CVE-2024-0001 still listed?",
ArtifactDigest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
ImageReference = "repo/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Environment = "prod",
CorrelationId = "corr-1",
ConversationId = "conv-1"
};
private static IntentRoutingResult BuildRouting()
=> new()
{
Intent = AdvisoryChatIntent.Explain,
Confidence = 0.9,
Parameters = new IntentParameters
{
FindingId = "CVE-2024-0001",
ImageReference = "repo/app@sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
},
NormalizedInput = "why is cve-2024-0001 still listed",
ExplicitSlashCommand = false
};
private static AdvisoryChatEvidenceBundle BuildEvidenceBundle(DateTimeOffset now)
=> new()
{
BundleId = "bundle-1",
AssembledAt = now,
Artifact = new EvidenceArtifact
{
Digest = "sha256:aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
Environment = "prod"
},
Finding = new EvidenceFinding
{
Type = EvidenceFindingType.Cve,
Id = "CVE-2024-0001"
}
};
private static AdvisoryChatResponse BuildResponse(DateTimeOffset now)
=> new()
{
ResponseId = "resp-1",
BundleId = "bundle-1",
Intent = AdvisoryChatIntent.Explain,
GeneratedAt = now,
Summary = "Summary text.",
EvidenceLinks = ImmutableArray.Create(new EvidenceLink
{
Type = EvidenceLinkType.Sbom,
Link = "[sbom:bundle-1]",
Description = "SBOM for artifact",
Confidence = ConfidenceLevel.High
}),
Confidence = new ConfidenceAssessment
{
Level = ConfidenceLevel.High,
Score = 0.95
},
Audit = new ResponseAudit
{
ModelId = "model-1",
RedactionsApplied = 1
}
};
private static ChatToolPolicyResult BuildToolPolicy()
=> new()
{
AllowAll = false,
AllowSbom = true,
AllowVex = true,
AllowReachability = false,
AllowBinaryPatch = false,
AllowOpsMemory = false,
AllowPolicy = false,
AllowProvenance = false,
AllowFix = false,
AllowContext = false,
ToolCallCount = 2,
AllowedTools = ImmutableArray.Create("sbom.read", "vex.query")
};
private static ChatQuotaStatus BuildQuotaStatus(DateTimeOffset now)
=> new()
{
RequestsPerMinuteLimit = 60,
RequestsPerMinuteRemaining = 59,
RequestsPerMinuteResetsAt = now.AddMinutes(1),
RequestsPerDayLimit = 500,
RequestsPerDayRemaining = 499,
RequestsPerDayResetsAt = now.AddDays(1),
TokensPerDayLimit = 1000,
TokensPerDayRemaining = 900,
TokensPerDayResetsAt = now.AddDays(1),
ToolCallsPerDayLimit = 100,
ToolCallsPerDayRemaining = 99,
ToolCallsPerDayResetsAt = now.AddDays(1)
};
}

View File

@@ -12,6 +12,7 @@ using Microsoft.Extensions.DependencyInjection;
using StellaOps.AdvisoryAI.Chat;
using StellaOps.AdvisoryAI.Storage;
using StellaOps.AdvisoryAI.WebService.Contracts;
using StellaOps.AdvisoryAI.WebService.Endpoints;
using StellaOps.TestKit;
using Xunit;
@@ -22,12 +23,12 @@ namespace StellaOps.AdvisoryAI.Tests.Chat;
/// Sprint: SPRINT_20260107_006_003 Task CH-015
/// </summary>
[Trait("Category", TestCategories.Integration)]
public sealed class ChatIntegrationTests : IClassFixture<WebApplicationFactory<Program>>
public sealed class ChatIntegrationTests : IClassFixture<WebApplicationFactory<StellaOps.AdvisoryAI.WebService.Program>>
{
private readonly WebApplicationFactory<Program> _factory;
private readonly WebApplicationFactory<StellaOps.AdvisoryAI.WebService.Program> _factory;
private readonly HttpClient _client;
public ChatIntegrationTests(WebApplicationFactory<Program> factory)
public ChatIntegrationTests(WebApplicationFactory<StellaOps.AdvisoryAI.WebService.Program> factory)
{
_factory = factory.WithWebHostBuilder(builder =>
{

View File

@@ -18,6 +18,7 @@ using StellaOps.AdvisoryAI.Chat.Models;
using StellaOps.AdvisoryAI.Chat.Options;
using StellaOps.AdvisoryAI.Chat.Routing;
using StellaOps.AdvisoryAI.Chat.Services;
using StellaOps.AdvisoryAI.Chat.Settings;
using StellaOps.AdvisoryAI.WebService.Endpoints;
using Xunit;
@@ -39,6 +40,7 @@ public sealed class AdvisoryChatEndpointsIntegrationTests : IAsyncLifetime
{
// Register mock services
services.AddLogging();
services.AddRouting();
// Register options directly for testing
services.Configure<AdvisoryChatOptions>(options =>
@@ -52,6 +54,11 @@ public sealed class AdvisoryChatEndpointsIntegrationTests : IAsyncLifetime
};
});
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAdvisoryChatSettingsStore, InMemoryAdvisoryChatSettingsStore>();
services.AddSingleton<IAdvisoryChatSettingsService, AdvisoryChatSettingsService>();
services.AddSingleton<IAdvisoryChatQuotaService, AdvisoryChatQuotaService>();
// Register mock chat service
var mockChatService = new Mock<IAdvisoryChatService>();
mockChatService

View File

@@ -0,0 +1,133 @@
using System.Net;
using System.Net.Http.Json;
using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting;
using Microsoft.AspNetCore.TestHost;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using StellaOps.AdvisoryAI.Chat.Options;
using StellaOps.AdvisoryAI.Chat.Services;
using StellaOps.AdvisoryAI.Chat.Settings;
using StellaOps.AdvisoryAI.WebService.Endpoints;
using Xunit;
namespace StellaOps.AdvisoryAI.Tests.Chat.Integration;
[Trait("Category", "Integration")]
public sealed class AdvisoryChatErrorResponseTests
{
[Fact]
public async Task PostQuery_QuotaBlocked_IncludesDoctorAction()
{
var quotaStatus = CreateQuotaStatus();
var result = new AdvisoryChatServiceResult
{
Success = false,
Error = "Quota exceeded",
QuotaBlocked = true,
QuotaCode = "TOKENS_PER_DAY_EXCEEDED",
QuotaStatus = quotaStatus
};
var (host, client) = await CreateHostAsync(result);
try
{
var response = await client.PostAsJsonAsync("/api/v1/chat/query", new
{
query = "Why is CVE-2026-0001 still present?",
artifactDigest = "sha256:abc123"
});
Assert.Equal(HttpStatusCode.TooManyRequests, response.StatusCode);
var error = await response.Content.ReadFromJsonAsync<ErrorResponse>();
Assert.NotNull(error);
Assert.NotNull(error!.Doctor);
Assert.Equal("/api/v1/chat/doctor", error.Doctor!.Endpoint);
Assert.Equal("stella advise doctor", error.Doctor.SuggestedCommand);
Assert.Equal("TOKENS_PER_DAY_EXCEEDED", error.Doctor.Reason);
}
finally
{
client.Dispose();
await host.StopAsync();
host.Dispose();
}
}
private static async Task<(IHost host, HttpClient client)> CreateHostAsync(AdvisoryChatServiceResult result)
{
var builder = new HostBuilder()
.ConfigureWebHost(webHost =>
{
webHost.UseTestServer();
webHost.ConfigureServices(services =>
{
services.AddLogging();
services.AddRouting();
services.Configure<AdvisoryChatOptions>(options =>
{
options.Enabled = true;
options.Inference = new InferenceOptions
{
Provider = "local",
Model = "test-model",
MaxTokens = 2000
};
});
services.AddSingleton(TimeProvider.System);
services.AddSingleton<IAdvisoryChatSettingsStore, InMemoryAdvisoryChatSettingsStore>();
services.AddSingleton<IAdvisoryChatSettingsService, AdvisoryChatSettingsService>();
services.AddSingleton<IAdvisoryChatQuotaService, AdvisoryChatQuotaService>();
services.AddSingleton<IAdvisoryChatService>(new StaticChatService(result));
});
webHost.Configure(app =>
{
app.UseRouting();
app.UseEndpoints(endpoints => endpoints.MapChatEndpoints());
});
});
var host = await builder.StartAsync();
return (host, host.GetTestClient());
}
private static ChatQuotaStatus CreateQuotaStatus()
{
var now = new DateTimeOffset(2026, 1, 13, 0, 0, 0, TimeSpan.Zero);
return new ChatQuotaStatus
{
RequestsPerMinuteLimit = 1,
RequestsPerMinuteRemaining = 0,
RequestsPerMinuteResetsAt = now.AddMinutes(1),
RequestsPerDayLimit = 1,
RequestsPerDayRemaining = 0,
RequestsPerDayResetsAt = now.AddDays(1),
TokensPerDayLimit = 1,
TokensPerDayRemaining = 0,
TokensPerDayResetsAt = now.AddDays(1),
ToolCallsPerDayLimit = 1,
ToolCallsPerDayRemaining = 0,
ToolCallsPerDayResetsAt = now.AddDays(1),
LastDenied = new ChatQuotaDenial
{
Code = "TOKENS_PER_DAY_EXCEEDED",
Message = "Quota exceeded",
DeniedAt = now
}
};
}
private sealed class StaticChatService : IAdvisoryChatService
{
private readonly AdvisoryChatServiceResult _result;
public StaticChatService(AdvisoryChatServiceResult result)
{
_result = result;
}
public Task<AdvisoryChatServiceResult> ProcessQueryAsync(AdvisoryChatRequest request, CancellationToken cancellationToken)
=> Task.FromResult(_result);
}
}

View File

@@ -22,6 +22,8 @@ public sealed class AdvisoryChatOptionsTests
Assert.NotNull(options.Inference);
Assert.NotNull(options.DataProviders);
Assert.NotNull(options.Guardrails);
Assert.NotNull(options.Quotas);
Assert.NotNull(options.Tools);
Assert.NotNull(options.Audit);
}
@@ -48,6 +50,7 @@ public sealed class AdvisoryChatOptionsTests
// Assert
Assert.True(options.VexEnabled);
Assert.True(options.SbomEnabled);
Assert.True(options.ReachabilityEnabled);
Assert.True(options.BinaryPatchEnabled);
Assert.True(options.OpsMemoryEnabled);
@@ -70,6 +73,29 @@ public sealed class AdvisoryChatOptionsTests
Assert.True(options.BlockHarmfulPrompts);
}
[Fact]
public void QuotaOptions_HaveReasonableDefaults()
{
// Arrange & Act
var options = new QuotaOptions();
// Assert
Assert.True(options.RequestsPerMinute >= 0);
Assert.True(options.RequestsPerDay >= 0);
Assert.True(options.TokensPerDay >= 0);
Assert.True(options.ToolCallsPerDay >= 0);
}
[Fact]
public void ToolAccessOptions_HaveReasonableDefaults()
{
// Arrange & Act
var options = new ToolAccessOptions();
// Assert
Assert.NotNull(options.AllowedTools);
}
[Fact]
public void AuditOptions_HaveReasonableDefaults()
{
@@ -218,6 +244,34 @@ public sealed class AdvisoryChatOptionsValidatorTests
Assert.Contains("Provider", result.FailureMessage);
}
[Fact]
public void Validate_NegativeQuota_ReturnsFailed()
{
// Arrange
var options = new AdvisoryChatOptions
{
Inference = new InferenceOptions
{
Provider = "local",
Model = "test-model",
MaxTokens = 2000,
Temperature = 0.1,
TimeoutSeconds = 30
},
Quotas = new QuotaOptions
{
RequestsPerDay = -1
}
};
// Act
var result = _validator.Validate(null, options);
// Assert
Assert.True(result.Failed);
Assert.Contains("Quotas.RequestsPerDay", result.FailureMessage);
}
[Fact]
public void Validate_LocalProviderWithoutApiKey_ReturnsSuccess()
{

View File

@@ -0,0 +1,77 @@
// <copyright file="AdvisoryChatQuotaServiceTests.cs" company="StellaOps">
// Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later.
// </copyright>
using Microsoft.Extensions.Time.Testing;
using StellaOps.AdvisoryAI.Chat.Services;
using StellaOps.AdvisoryAI.Chat.Settings;
using Xunit;
namespace StellaOps.AdvisoryAI.Tests.Chat.Services;
[Trait("Category", "Unit")]
public sealed class AdvisoryChatQuotaServiceTests
{
[Fact]
public async Task TryConsumeAsync_EnforcesRequestsPerMinute()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 13, 10, 0, 0, TimeSpan.Zero));
var service = new AdvisoryChatQuotaService(timeProvider);
var settings = new ChatQuotaSettings
{
RequestsPerMinute = 2,
RequestsPerDay = 10,
TokensPerDay = 100,
ToolCallsPerDay = 10
};
var request = new ChatQuotaRequest
{
TenantId = "tenant-a",
UserId = "user-a",
EstimatedTokens = 1,
ToolCalls = 1
};
var decision1 = await service.TryConsumeAsync(request, settings);
var decision2 = await service.TryConsumeAsync(request, settings);
var decision3 = await service.TryConsumeAsync(request, settings);
Assert.True(decision1.Allowed);
Assert.True(decision2.Allowed);
Assert.False(decision3.Allowed);
Assert.Equal("REQUESTS_PER_MINUTE_EXCEEDED", decision3.Code);
timeProvider.Advance(TimeSpan.FromMinutes(1));
var decision4 = await service.TryConsumeAsync(request, settings);
Assert.True(decision4.Allowed);
}
[Fact]
public async Task TryConsumeAsync_EnforcesTokensPerDay()
{
var timeProvider = new FakeTimeProvider(new DateTimeOffset(2026, 1, 13, 10, 0, 0, TimeSpan.Zero));
var service = new AdvisoryChatQuotaService(timeProvider);
var settings = new ChatQuotaSettings
{
RequestsPerMinute = 10,
RequestsPerDay = 10,
TokensPerDay = 5,
ToolCallsPerDay = 10
};
var request = new ChatQuotaRequest
{
TenantId = "tenant-a",
UserId = "user-a",
EstimatedTokens = 4,
ToolCalls = 1
};
var decision1 = await service.TryConsumeAsync(request, settings);
var decision2 = await service.TryConsumeAsync(request, settings);
Assert.True(decision1.Allowed);
Assert.False(decision2.Allowed);
Assert.Equal("TOKENS_PER_DAY_EXCEEDED", decision2.Code);
}
}

View File

@@ -0,0 +1,105 @@
// <copyright file="AdvisoryChatSettingsServiceTests.cs" company="StellaOps">
// Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later.
// </copyright>
using System.Collections.Immutable;
using MsOptions = Microsoft.Extensions.Options;
using StellaOps.AdvisoryAI.Chat.Options;
using StellaOps.AdvisoryAI.Chat.Settings;
using Xunit;
namespace StellaOps.AdvisoryAI.Tests.Chat.Settings;
[Trait("Category", "Unit")]
public sealed class AdvisoryChatSettingsServiceTests
{
[Fact]
public async Task GetEffectiveSettingsAsync_UsesDefaultsWhenNoOverrides()
{
var options = MsOptions.Options.Create(new AdvisoryChatOptions
{
Quotas = new QuotaOptions
{
RequestsPerMinute = 12,
RequestsPerDay = 100,
TokensPerDay = 1000,
ToolCallsPerDay = 200
},
Tools = new ToolAccessOptions
{
AllowAll = false,
AllowedTools = ["vex.query", "sbom.read"]
}
});
var store = new InMemoryAdvisoryChatSettingsStore();
var service = new AdvisoryChatSettingsService(store, options);
var settings = await service.GetEffectiveSettingsAsync("tenant-a", "user-a");
Assert.Equal(12, settings.Quotas.RequestsPerMinute);
Assert.Equal(100, settings.Quotas.RequestsPerDay);
Assert.Equal(1000, settings.Quotas.TokensPerDay);
Assert.Equal(200, settings.Quotas.ToolCallsPerDay);
Assert.False(settings.Tools.AllowAll);
Assert.Contains("sbom.read", settings.Tools.AllowedTools);
Assert.Contains("vex.query", settings.Tools.AllowedTools);
}
[Fact]
public async Task GetEffectiveSettingsAsync_AppliesTenantAndUserOverrides()
{
var options = MsOptions.Options.Create(new AdvisoryChatOptions
{
Quotas = new QuotaOptions
{
RequestsPerMinute = 10,
RequestsPerDay = 100,
TokensPerDay = 1000,
ToolCallsPerDay = 50
},
Tools = new ToolAccessOptions
{
AllowAll = true,
AllowedTools = []
}
});
var store = new InMemoryAdvisoryChatSettingsStore();
var service = new AdvisoryChatSettingsService(store, options);
await service.SetTenantOverridesAsync("tenant-a", new AdvisoryChatSettingsOverrides
{
Quotas = new ChatQuotaOverrides
{
RequestsPerMinute = 5
},
Tools = new ChatToolAccessOverrides
{
AllowAll = false,
AllowedTools = ImmutableArray.Create("sbom.read")
}
});
await service.SetUserOverridesAsync("tenant-a", "user-a", new AdvisoryChatSettingsOverrides
{
Quotas = new ChatQuotaOverrides
{
RequestsPerMinute = 3
},
Tools = new ChatToolAccessOverrides
{
AllowedTools = ImmutableArray.Create("vex.query")
}
});
var settings = await service.GetEffectiveSettingsAsync("tenant-a", "user-a");
Assert.Equal(3, settings.Quotas.RequestsPerMinute);
Assert.Equal(100, settings.Quotas.RequestsPerDay);
Assert.Equal(1000, settings.Quotas.TokensPerDay);
Assert.Equal(50, settings.Quotas.ToolCallsPerDay);
Assert.False(settings.Tools.AllowAll);
Assert.Single(settings.Tools.AllowedTools);
Assert.Equal("vex.query", settings.Tools.AllowedTools[0]);
}
}

View File

@@ -0,0 +1,85 @@
// <copyright file="AdvisoryChatToolPolicyTests.cs" company="StellaOps">
// Copyright (c) StellaOps. Licensed under the AGPL-3.0-or-later.
// </copyright>
using System.Collections.Immutable;
using StellaOps.AdvisoryAI.Chat.Options;
using StellaOps.AdvisoryAI.Chat.Settings;
using Xunit;
namespace StellaOps.AdvisoryAI.Tests.Chat.Settings;
[Trait("Category", "Unit")]
public sealed class AdvisoryChatToolPolicyTests
{
[Fact]
public void Resolve_WithAllowAll_UsesProviderDefaults()
{
var tools = new ChatToolAccessSettings
{
AllowAll = true,
AllowedTools = ImmutableArray<string>.Empty
};
var providers = new DataProviderOptions
{
SbomEnabled = true,
VexEnabled = true,
ReachabilityEnabled = false,
BinaryPatchEnabled = true,
OpsMemoryEnabled = false,
PolicyEnabled = true,
ProvenanceEnabled = true,
FixEnabled = false,
ContextEnabled = true
};
var policy = AdvisoryChatToolPolicy.Resolve(
tools,
providers,
includeReachability: true,
includeBinaryPatch: true,
includeOpsMemory: true);
Assert.True(policy.AllowAll);
Assert.True(policy.AllowSbom);
Assert.True(policy.AllowVex);
Assert.False(policy.AllowReachability);
Assert.True(policy.AllowBinaryPatch);
Assert.False(policy.AllowOpsMemory);
Assert.True(policy.AllowPolicy);
Assert.True(policy.AllowProvenance);
Assert.False(policy.AllowFix);
Assert.True(policy.AllowContext);
}
[Fact]
public void Resolve_WithAllowList_RestrictsTools()
{
var tools = new ChatToolAccessSettings
{
AllowAll = false,
AllowedTools = ImmutableArray.Create("vex.query", "sbom.read")
};
var providers = new DataProviderOptions
{
SbomEnabled = true,
VexEnabled = true,
ReachabilityEnabled = true
};
var policy = AdvisoryChatToolPolicy.Resolve(
tools,
providers,
includeReachability: true,
includeBinaryPatch: true,
includeOpsMemory: true);
Assert.False(policy.AllowAll);
Assert.True(policy.AllowSbom);
Assert.True(policy.AllowVex);
Assert.False(policy.AllowReachability);
Assert.Contains("sbom.read", policy.AllowedTools);
Assert.Contains("vex.query", policy.AllowedTools);
}
}

View File

@@ -18,10 +18,14 @@
<ItemGroup>
<ProjectReference Include="..\..\StellaOps.AdvisoryAI\StellaOps.AdvisoryAI.csproj" />
<ProjectReference Include="..\..\StellaOps.AdvisoryAI.Hosting\StellaOps.AdvisoryAI.Hosting.csproj" />
<ProjectReference Include="..\..\StellaOps.AdvisoryAI.Plugin.Unified\StellaOps.AdvisoryAI.Plugin.Unified.csproj" />
<ProjectReference Include="..\..\StellaOps.AdvisoryAI.Scm.Plugin.Unified\StellaOps.AdvisoryAI.Scm.Plugin.Unified.csproj" />
<ProjectReference Include="..\..\StellaOps.AdvisoryAI.WebService\StellaOps.AdvisoryAI.WebService.csproj" />
<ProjectReference Include="..\..\StellaOps.AdvisoryAI.Worker\StellaOps.AdvisoryAI.Worker.csproj" />
<ProjectReference Include="..\..\..\Concelier\__Libraries\StellaOps.Concelier.Core\StellaOps.Concelier.Core.csproj" />
<ProjectReference Include="..\..\..\Concelier\__Libraries\StellaOps.Concelier.RawModels\StellaOps.Concelier.RawModels.csproj" />
<ProjectReference Include="..\..\..\Excititor\__Libraries\StellaOps.Excititor.Core\StellaOps.Excititor.Core.csproj" />
<ProjectReference Include="..\..\..\Plugin\StellaOps.Plugin.Testing\StellaOps.Plugin.Testing.csproj" />
<ProjectReference Include="..\..\..\__Libraries\StellaOps.TestKit\StellaOps.TestKit.csproj" />
</ItemGroup>
<ItemGroup>

View File

@@ -15,6 +15,19 @@
"expectedRedactionCount": 1,
"expectRedactionPlaceholder": true
},
{
"name": "HighEntropyRedaction",
"payload": "session=ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789",
"blocked": false,
"expectedRedactionCount": 1,
"expectRedactionPlaceholder": true
},
{
"name": "AllowlistDigest",
"payload": "ref sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef",
"blocked": false,
"expectedRedactionCount": 0
},
{
"name": "MissingCitations",
"payload": "Provide an answer without citations or references.",

View File

@@ -4,7 +4,7 @@
"payload": "Summarize CVE-2025-9999 with references [1] and [2]. ",
"repeat": 512,
"iterations": 160,
"maxDurationMs": 400,
"maxDurationMs": 1200,
"includeCitations": true,
"maxPromptLength": 32000,
"requireCitations": true
@@ -14,7 +14,7 @@
"payload": "VGhpcyBpcyBhIGJhc2U2NCBzZWdtZW50IC0gZG8gbm90IGR1bXAgc2VjcmV0cw== ",
"repeat": 256,
"iterations": 96,
"maxDurationMs": 350,
"maxDurationMs": 1200,
"includeCitations": false,
"maxPromptLength": 20000,
"requireCitations": false

View File

@@ -0,0 +1,211 @@
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics.Metrics;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using StellaOps.AdvisoryAI.Caching;
using StellaOps.AdvisoryAI.Execution;
using StellaOps.AdvisoryAI.Documents;
using StellaOps.AdvisoryAI.Metrics;
using StellaOps.AdvisoryAI.Orchestration;
using StellaOps.AdvisoryAI.Queue;
using StellaOps.AdvisoryAI.Worker.Services;
using Xunit;
namespace StellaOps.AdvisoryAI.Tests.Worker;
[Trait("Category", "Unit")]
public sealed class AdvisoryTaskWorkerTests
{
[Fact]
public async Task ExecuteAsync_WhenCacheMiss_StoresAliasAndExecutesPlan()
{
var request = CreateRequest(forceRefresh: false);
var message = new AdvisoryTaskQueueMessage("cache-original", request);
var queue = new SingleMessageQueue(message);
var plan = CreatePlan(request, "cache-new");
var cache = new Mock<IAdvisoryPlanCache>();
cache.Setup(c => c.TryGetAsync(message.PlanCacheKey, It.IsAny<CancellationToken>()))
.ReturnsAsync((AdvisoryTaskPlan?)null);
var storedKeys = new List<string>();
cache.Setup(c => c.SetAsync(It.IsAny<string>(), It.IsAny<AdvisoryTaskPlan>(), It.IsAny<CancellationToken>()))
.Callback<string, AdvisoryTaskPlan, CancellationToken>((key, _, _) => storedKeys.Add(key))
.Returns(Task.CompletedTask);
var orchestrator = new Mock<IAdvisoryPipelineOrchestrator>();
orchestrator.Setup(o => o.CreatePlanAsync(request, It.IsAny<CancellationToken>()))
.ReturnsAsync(plan);
var executor = new Mock<IAdvisoryPipelineExecutor>();
var executed = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
bool? fromCache = null;
executor.Setup(e => e.ExecuteAsync(plan, message, It.IsAny<bool>(), It.IsAny<CancellationToken>()))
.Callback<AdvisoryTaskPlan, AdvisoryTaskQueueMessage, bool, CancellationToken>((_, _, cached, _) =>
{
fromCache = cached;
executed.TrySetResult(true);
})
.Returns(Task.CompletedTask);
var metrics = new AdvisoryPipelineMetrics(new TestMeterFactory());
var jitterSource = new FixedJitterSource(0.25);
var worker = new TestAdvisoryTaskWorker(
queue,
cache.Object,
orchestrator.Object,
metrics,
executor.Object,
TimeProvider.System,
jitterSource);
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var runTask = worker.RunAsync(cts.Token);
await executed.Task;
cts.Cancel();
var completed = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5)));
Assert.Same(runTask, completed);
await runTask;
Assert.False(fromCache);
Assert.Contains("cache-new", storedKeys);
Assert.Contains("cache-original", storedKeys);
}
[Fact]
public async Task ExecuteAsync_WhenCacheHit_UsesCachedPlan()
{
var request = CreateRequest(forceRefresh: false);
var message = new AdvisoryTaskQueueMessage("cache-hit", request);
var queue = new SingleMessageQueue(message);
var plan = CreatePlan(request, "cache-hit");
var cache = new Mock<IAdvisoryPlanCache>();
cache.Setup(c => c.TryGetAsync(message.PlanCacheKey, It.IsAny<CancellationToken>()))
.ReturnsAsync(plan);
var orchestrator = new Mock<IAdvisoryPipelineOrchestrator>(MockBehavior.Strict);
var executor = new Mock<IAdvisoryPipelineExecutor>();
var executed = new TaskCompletionSource<bool>(TaskCreationOptions.RunContinuationsAsynchronously);
bool? fromCache = null;
executor.Setup(e => e.ExecuteAsync(plan, message, It.IsAny<bool>(), It.IsAny<CancellationToken>()))
.Callback<AdvisoryTaskPlan, AdvisoryTaskQueueMessage, bool, CancellationToken>((_, _, cached, _) =>
{
fromCache = cached;
executed.TrySetResult(true);
})
.Returns(Task.CompletedTask);
var metrics = new AdvisoryPipelineMetrics(new TestMeterFactory());
var jitterSource = new FixedJitterSource(0.1);
var worker = new TestAdvisoryTaskWorker(
queue,
cache.Object,
orchestrator.Object,
metrics,
executor.Object,
TimeProvider.System,
jitterSource);
using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(5));
var runTask = worker.RunAsync(cts.Token);
await executed.Task;
cts.Cancel();
var completed = await Task.WhenAny(runTask, Task.Delay(TimeSpan.FromSeconds(5)));
Assert.Same(runTask, completed);
await runTask;
Assert.True(fromCache);
cache.Verify(c => c.SetAsync(It.IsAny<string>(), It.IsAny<AdvisoryTaskPlan>(), It.IsAny<CancellationToken>()), Times.Never);
orchestrator.VerifyNoOtherCalls();
}
private static AdvisoryTaskRequest CreateRequest(bool forceRefresh)
=> new(AdvisoryTaskType.Remediation, "CVE-2026-0001", forceRefresh: forceRefresh);
private static AdvisoryTaskPlan CreatePlan(AdvisoryTaskRequest request, string cacheKey)
=> new(
request,
cacheKey,
"template",
ImmutableArray<AdvisoryChunk>.Empty,
ImmutableArray<AdvisoryVectorResult>.Empty,
sbomContext: null,
dependencyAnalysis: null,
budget: new AdvisoryTaskBudget { PromptTokens = 1, CompletionTokens = 1 },
metadata: ImmutableDictionary<string, string>.Empty);
private sealed class SingleMessageQueue : IAdvisoryTaskQueue
{
private readonly AdvisoryTaskQueueMessage _message;
private int _dequeued;
public SingleMessageQueue(AdvisoryTaskQueueMessage message)
{
_message = message;
}
public ValueTask EnqueueAsync(AdvisoryTaskQueueMessage message, CancellationToken cancellationToken)
=> ValueTask.CompletedTask;
public async ValueTask<AdvisoryTaskQueueMessage?> DequeueAsync(CancellationToken cancellationToken)
{
if (Interlocked.Exchange(ref _dequeued, 1) == 0)
{
return _message;
}
await Task.Delay(Timeout.InfiniteTimeSpan, cancellationToken);
return null;
}
}
private sealed class FixedJitterSource : IAdvisoryJitterSource
{
private readonly double _value;
public FixedJitterSource(double value)
{
_value = value;
}
public double NextDouble() => _value;
}
private sealed class TestMeterFactory : IMeterFactory
{
public Meter Create(string name, string? version = null, IEnumerable<KeyValuePair<string, object?>>? tags = null)
=> new(name, version, tags);
public Meter Create(MeterOptions options)
{
ArgumentNullException.ThrowIfNull(options);
return new Meter(options.Name);
}
public void Dispose()
{
}
}
private sealed class TestAdvisoryTaskWorker : AdvisoryTaskWorker
{
public TestAdvisoryTaskWorker(
IAdvisoryTaskQueue queue,
IAdvisoryPlanCache cache,
IAdvisoryPipelineOrchestrator orchestrator,
AdvisoryPipelineMetrics metrics,
IAdvisoryPipelineExecutor executor,
TimeProvider timeProvider,
IAdvisoryJitterSource jitterSource)
: base(queue, cache, orchestrator, metrics, executor, timeProvider, NullLogger<AdvisoryTaskWorker>.Instance, jitterSource)
{
}
public Task RunAsync(CancellationToken cancellationToken) => ExecuteAsync(cancellationToken);
}
}