save checkpoint
This commit is contained in:
master
@@ -0,0 +1,26 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "advisoryai-orchestrator",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "AdvisoryPipelineOrchestratorTests, AdvisoryPipelineExecutorTests, AdvisoryPipelinePlanResponseTests, ConversationServiceTests, RunServiceTests, InMemoryRunStoreTests, AdvisoryChatIntentRouterTests, AdvisoryTaskWorkerTests, RunServiceIntegrationTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"AdvisoryPipelineOrchestrator creates and executes task plans (6 tests)",
|
||||
"AdvisoryPipelineExecutor executes pipeline stages with guardrail checks (5 tests)",
|
||||
"AdvisoryPipelinePlanResponse returns structured plan responses (1 test)",
|
||||
"ConversationService maintains conversation context across messages (19 tests)",
|
||||
"RunService tracks runs with artifacts and events (18 tests)",
|
||||
"InMemoryRunStore persists run data correctly (15 tests)",
|
||||
"AdvisoryChatIntentRouter routes intents to correct handlers (16 tests)",
|
||||
"AdvisoryTaskWorker processes queued tasks (2 tests)",
|
||||
"RunServiceIntegration end-to-end run lifecycle (13 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -1,33 +1,26 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"module": "advisoryai",
|
||||
"feature": "advisoryai-pipeline-with-guardrails",
|
||||
"runId": "run-001",
|
||||
"capturedAtUtc": "2026-02-11T11:40:39.8348068Z",
|
||||
"command": {
|
||||
"value": "dotnet test src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj --filter \"FullyQualifiedName~AdvisoryGuardrailInjectionTests|FullyQualifiedName~AdvisoryPipelineExecutorTests|FullyQualifiedName~AdvisoryPipelineOrchestratorTests|FullyQualifiedName~AdvisoryStructuredRetrieverTests|FullyQualifiedName~AdvisoryVectorRetrieverTests|FullyQualifiedName~SbomContextRetrieverTests\" -v normal",
|
||||
"exitCode": 0,
|
||||
"failed": 0,
|
||||
"passed": 572,
|
||||
"skipped": 0,
|
||||
"total": 572,
|
||||
"log": "docs\\\\qa\\\\feature-checks\\\\runs\\\\advisoryai\\\\advisoryai-pipeline-with-guardrails\\\\run-001\\\\evidence\\\\tier2-test-pipeline-behavior.txt"
|
||||
},
|
||||
"behaviorVerified": [
|
||||
"Guardrail harness validates blocked and allowed prompt cases, citation enforcement, and secret redaction placeholders in AdvisoryGuardrailInjectionTests and AdvisoryGuardrailPipelineTests.",
|
||||
"Action execution policy paths (allowed, approval-required, denied, and idempotent skip) are validated in ActionExecutorTests.",
|
||||
"Idempotency key determinism, tenant/action scoping, TTL expiration, and record lifecycle checks are validated in IdempotencyHandlerTests.",
|
||||
"Retriever behaviors for structured, vector, and SBOM contexts include deterministic ordering, metadata population, deduplication, and empty-result handling in retriever test suites.",
|
||||
"Pipeline orchestration and executor tests verify deterministic plan composition, provenance/telemetry persistence, and guardrail outcome recording."
|
||||
],
|
||||
"evidence": [
|
||||
"docs\\\\qa\\\\feature-checks\\\\runs\\\\advisoryai\\\\advisoryai-pipeline-with-guardrails\\\\run-001\\\\evidence\\\\tier2-test-pipeline-behavior.txt",
|
||||
"src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/AdvisoryGuardrailInjectionTests.cs",
|
||||
"src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/Actions/ActionExecutorTests.cs",
|
||||
"src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/Actions/IdempotencyHandlerTests.cs"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; tier commands executed full test assembly."
|
||||
],
|
||||
"verdict": "pass"
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "advisoryai-pipeline-with-guardrails",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "AdvisoryGuardrailPipelineTests, AdvisoryGuardrailInjectionTests, AdvisoryGuardrailOptionsBindingTests, AdvisoryGuardrailPerformanceTests, AdvisoryStructuredRetrieverTests, AdvisoryVectorRetrieverTests, SbomContextRetrieverTests, ActionExecutorTests, IdempotencyHandlerTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"AdvisoryGuardrailPipeline filters prohibited content (2 tests)",
|
||||
"AdvisoryGuardrailInjection injects guardrails into pipeline (1 test)",
|
||||
"AdvisoryGuardrailOptionsBinding loads configuration (3 tests)",
|
||||
"AdvisoryGuardrailPerformance validates pipeline performance (2 tests)",
|
||||
"AdvisoryStructuredRetriever returns relevant CVE/advisory data (4 tests)",
|
||||
"AdvisoryVectorRetriever performs semantic search (1 test)",
|
||||
"SbomContextRetriever enriches prompts with SBOM context (4 tests)",
|
||||
"ActionExecutor executes approved actions (8 tests)",
|
||||
"IdempotencyHandler prevents duplicate action execution (11 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "ai-remedy-autopilot-with-multi-scm-pull-request-generation",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "RemediationIntegrationTests, GitHubPullRequestGeneratorTests, ScmPluginAdapterTests, ScmPluginAdapterHealthTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"RemediationIntegration validates AI-driven remediation plan generation, SBOM delta computation, PR template building, and SCM connector routing (28 tests)",
|
||||
"GitHubPullRequestGenerator creates PRs with evidence-populated descriptions (11 tests)",
|
||||
"ScmPluginAdapter creates branches, commits, and opens PRs through unified plugin interface (6 tests)",
|
||||
"ScmPluginAdapterHealth provides health check for SCM integrations (1 test)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "chat-gateway-with-quotas-and-scrubbing",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "AdvisoryChatQuotaServiceTests, AdvisoryChatOptionsTests, ChatPromptAssemblerTests, GroundingValidatorTests, ChatIntegrationTests, AdvisoryChatEndpointsIntegrationTests, AdvisoryChatErrorResponseTests, AdvisoryChatSecurityTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"AdvisoryChatQuotaService enforces per-user quotas (2 tests)",
|
||||
"AdvisoryChatOptions validates configurable chat options including quotas and budgets (16 tests)",
|
||||
"ChatPromptAssembler scrubs sensitive data from prompts before LLM invocation (13 tests)",
|
||||
"GroundingValidator flags responses lacking evidence grounding (20 tests)",
|
||||
"ChatIntegration end-to-end chat session with streaming responses (14 tests)",
|
||||
"AdvisoryChatEndpointsIntegration REST API endpoint behavior (6 tests)",
|
||||
"AdvisoryChatErrorResponse handles error conditions (1 test)",
|
||||
"AdvisoryChatSecurity authorization and scope enforcement (11 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "evidence-first-ai-outputs",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "EvidenceBundleAssemblerTests, VexDataProviderTests, ReachabilityDataProviderTests, EvidenceCardExportIntegrationTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"EvidenceBundleAssembler assembles evidence bundles from multiple data providers (14 tests)",
|
||||
"VexDataProvider includes VEX status for referenced CVEs (5 tests)",
|
||||
"ReachabilityDataProvider includes reachability scores and call-path evidence (4 tests)",
|
||||
"EvidenceCardExportIntegration validates evidence pack export with attestation metadata (7 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "evidence-first-citations-in-chat-responses",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "ExplanationGeneratorIntegrationTests, ExplanationReplayGoldenTests, GroundingValidatorTests, ActionProposalParserTests, ChatPromptAssemblerTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"EvidenceAnchoredExplanationGenerator generates explanations with citation references (11 tests)",
|
||||
"ExplanationReplayGolden validates deterministic replay of explanation outputs (11 tests)",
|
||||
"GroundingValidator rejects explanations lacking evidence citations (20 tests)",
|
||||
"ActionProposalParser extracts cited evidence IDs from LLM-generated action proposals (18 tests)",
|
||||
"ChatPromptAssembler assembles prompts with citation instructions (13 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "immutable-audit-log-for-ai-interactions",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "AdvisoryChatAuditEnvelopeBuilderTests, ChatIntegrationTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"AdvisoryChatAuditEnvelopeBuilder creates DSSE-signed audit envelopes with prompt, response, and model fingerprint (4 tests)",
|
||||
"ChatIntegration validates audit record creation during chat sessions (14 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "llm-inference-response-caching",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "LlmInferenceCacheTests, LlmProviderConfigValidationTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"LlmInferenceCache deduplicates identical prompt+model combinations via content hash (3 tests)",
|
||||
"LlmProviderConfigValidation validates provider configuration including cache TTL and size limits (2 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "llm-provider-plugin-architecture",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "LlmPluginAdapterTests, LlmPluginAdapterHealthTests, LocalInferenceClientTests, SystemPromptLoaderTests, LlmProviderConfigValidationTests, OfflineInferenceIntegrationTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"LlmPluginAdapter provides unified plugin adapter for LLM providers (5 tests)",
|
||||
"LlmPluginAdapterHealth validates health checks for configured providers (2 tests)",
|
||||
"LocalInferenceClient connects to local inference endpoints (9 tests)",
|
||||
"SystemPromptLoader loads system prompts for inference clients (4 tests)",
|
||||
"LlmProviderConfigValidation rejects invalid configurations (2 tests)",
|
||||
"OfflineInferenceIntegration validates offline-first inference pipeline (23 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "natural-language-to-policy-rule-compiler",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "PolicyStudioIntegrationTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"PolicyStudioIntegration validates NL-to-policy intent parsing, lattice rule generation, property-based test synthesis, bundle compilation, iterative refinement, and conflict detection (16 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "opsmemory-chat-integration",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "ChatIntegrationTests, ConversationServiceTests, EvidenceBundleAssemblerTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"OpsMemory integration surfaces past decisions via ConversationContextBuilder (19 conversation tests)",
|
||||
"OpsMemoryDataProvider contributes to evidence bundles (14 evidence assembler tests)",
|
||||
"ChatIntegration validates OpsMemory context enrichment in chat sessions (14 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "sanctioned-tool-registry",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "AdvisoryChatToolPolicyTests, DeterministicToolsetTests, AdvisoryChatSettingsServiceTests, ToolsetServiceCollectionExtensionsTests, SemanticVersionTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"AdvisoryChatToolPolicy defines sanctioned, read-only, and confirmation-gated tools (2 tests)",
|
||||
"DeterministicToolset provides consistent version and dependency analysis (3 tests)",
|
||||
"AdvisoryChatSettingsService persists tool policy changes (2 tests)",
|
||||
"ToolsetServiceCollectionExtensions registers toolset services in DI (2 tests)",
|
||||
"SemanticVersion parsing validates version format and range matching (5 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
{
|
||||
"type": "integration",
|
||||
"capturedAtUtc": "2026-02-13T14:30:00Z",
|
||||
"feature": "sovereign-offline-ai-inference-with-signed-model-bundles",
|
||||
"module": "advisoryai",
|
||||
"testProject": "src/AdvisoryAI/__Tests/StellaOps.AdvisoryAI.Tests/StellaOps.AdvisoryAI.Tests.csproj",
|
||||
"testFilter": "OfflineInferenceIntegrationTests, SignedModelBundleManagerTests, LocalInferenceClientTests, LlmInferenceCacheTests, DeterminismTests",
|
||||
"testsRun": 575,
|
||||
"testsPassed": 575,
|
||||
"testsFailed": 0,
|
||||
"behaviorVerified": [
|
||||
"OfflineInferenceIntegration validates air-gapped inference pipeline with provider routing and caching (23 tests)",
|
||||
"SignedModelBundleManager validates DSSE signature and digest verification at load time (1 test)",
|
||||
"LocalInferenceClient connects to local inference endpoints with deterministic config (9 tests)",
|
||||
"LlmInferenceCache deduplicates deterministic responses (3 tests)",
|
||||
"Determinism validates temperature=0 fixed-seed produces consistent outputs (12 tests)"
|
||||
],
|
||||
"warnings": [
|
||||
"MTP0001: VSTestTestCaseFilter ignored by Microsoft.Testing.Platform; full test assembly executed (575 total)"
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
Reference in New Issue
Block a user