test fixes and new product advisories work
This commit is contained in:
master
@@ -373,7 +373,216 @@ public async Task Test_TracingBehavior()
|
||||
|
||||
---
|
||||
|
||||
### 9. Test Categories
|
||||
### 9. Observability Contract Testing (Turn #6)
|
||||
|
||||
Contract assertions for treating logs, metrics, and traces as APIs:
|
||||
|
||||
**OTel Contract Testing:**
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Observability;
|
||||
|
||||
[Fact, Trait("Category", TestCategories.Contract)]
|
||||
public async Task Test_SpanContracts()
|
||||
{
|
||||
using var capture = new OtelCapture("MyService");
|
||||
|
||||
await service.ProcessRequestAsync();
|
||||
|
||||
// Verify required spans are present
|
||||
OTelContractAssert.HasRequiredSpans(capture, "ProcessRequest", "ValidateInput", "SaveResult");
|
||||
|
||||
// Verify span attributes
|
||||
var span = capture.CapturedActivities.First();
|
||||
OTelContractAssert.SpanHasAttributes(span, "user_id", "tenant_id", "correlation_id");
|
||||
|
||||
// Check attribute cardinality (prevent metric explosion)
|
||||
OTelContractAssert.AttributeCardinality(capture, "http_method", maxCardinality: 10);
|
||||
|
||||
// Detect high-cardinality attributes globally
|
||||
OTelContractAssert.NoHighCardinalityAttributes(capture, threshold: 100);
|
||||
}
|
||||
```
|
||||
|
||||
**Log Contract Testing:**
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Observability;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
[Fact]
|
||||
public async Task Test_LogContracts()
|
||||
{
|
||||
var logCapture = new List<CapturedLogRecord>();
|
||||
// ... capture logs during test execution ...
|
||||
|
||||
// Verify required fields
|
||||
LogContractAssert.HasRequiredFields(logCapture[0], "CorrelationId", "TenantId");
|
||||
|
||||
// Ensure no PII leakage
|
||||
var piiPatterns = new[]
|
||||
{
|
||||
new Regex(@"\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}\b"), // Email
|
||||
new Regex(@"\b\d{3}-\d{2}-\d{4}\b"), // SSN
|
||||
};
|
||||
LogContractAssert.NoSensitiveData(logCapture, piiPatterns);
|
||||
|
||||
// Verify log level appropriateness
|
||||
LogContractAssert.LogLevelAppropriate(logCapture[0], LogLevel.Information, LogLevel.Warning);
|
||||
|
||||
// Ensure error logs have correlation for troubleshooting
|
||||
LogContractAssert.ErrorLogsHaveCorrelation(logCapture, "CorrelationId", "RequestId");
|
||||
}
|
||||
```
|
||||
|
||||
**Metrics Contract Testing:**
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Observability;
|
||||
|
||||
[Fact]
|
||||
public async Task Test_MetricsContracts()
|
||||
{
|
||||
using var capture = new MetricsCapture("MyService");
|
||||
|
||||
await service.ProcessMultipleRequests();
|
||||
|
||||
// Verify required metrics exist
|
||||
MetricsContractAssert.HasRequiredMetrics(capture, "requests_total", "request_duration_seconds");
|
||||
|
||||
// Check label cardinality bounds
|
||||
MetricsContractAssert.LabelCardinalityBounded(capture, "http_requests_total", maxLabels: 50);
|
||||
|
||||
// Verify counter monotonicity
|
||||
MetricsContractAssert.CounterMonotonic(capture, "processed_items_total");
|
||||
|
||||
// Verify gauge bounds
|
||||
MetricsContractAssert.GaugeInBounds(capture, "active_connections", minValue: 0, maxValue: 1000);
|
||||
}
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `OTelContractAssert.HasRequiredSpans(capture, spanNames)` - Verify spans exist
|
||||
- `OTelContractAssert.SpanHasAttributes(span, attrNames)` - Verify attributes
|
||||
- `OTelContractAssert.AttributeCardinality(capture, attr, max)` - Check cardinality
|
||||
- `OTelContractAssert.NoHighCardinalityAttributes(capture, threshold)` - Detect explosion
|
||||
- `LogContractAssert.HasRequiredFields(record, fields)` - Verify log fields
|
||||
- `LogContractAssert.NoSensitiveData(records, patterns)` - Check for PII
|
||||
- `MetricsContractAssert.MetricExists(capture, name)` - Verify metric
|
||||
- `MetricsContractAssert.LabelCardinalityBounded(capture, name, max)` - Check cardinality
|
||||
- `MetricsCapture` - Capture metrics during test execution
|
||||
- `ContractViolationException` - Thrown when contracts are violated
|
||||
|
||||
---
|
||||
|
||||
### 10. Evidence Chain Traceability (Turn #6)
|
||||
|
||||
Link tests to requirements for regulatory compliance and audit trails:
|
||||
|
||||
**Requirement Attribute:**
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Evidence;
|
||||
|
||||
[Fact]
|
||||
[Requirement("REQ-AUTH-001", SprintTaskId = "AUTH-0127-001")]
|
||||
public async Task Test_UserAuthentication()
|
||||
{
|
||||
// Verify authentication works as required
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Requirement("REQ-AUDIT-002", SprintTaskId = "AUDIT-0127-003", ComplianceControl = "SOC2-AU-12")]
|
||||
public void Test_AuditLogImmutability()
|
||||
{
|
||||
// Verify audit logs cannot be modified
|
||||
}
|
||||
```
|
||||
|
||||
**Filtering tests by requirement:**
|
||||
|
||||
```bash
|
||||
# Run tests for a specific requirement
|
||||
dotnet test --filter "Requirement=REQ-AUTH-001"
|
||||
|
||||
# Run tests for a sprint task
|
||||
dotnet test --filter "SprintTask=AUTH-0127-001"
|
||||
|
||||
# Run tests for a compliance control
|
||||
dotnet test --filter "ComplianceControl=SOC2-AU-12"
|
||||
```
|
||||
|
||||
**Evidence Chain Assertions:**
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Evidence;
|
||||
|
||||
[Fact]
|
||||
[Requirement("REQ-EVIDENCE-001")]
|
||||
public void Test_ArtifactHashStability()
|
||||
{
|
||||
var artifact = GenerateEvidence(input);
|
||||
|
||||
// Verify artifact produces expected hash (golden master)
|
||||
EvidenceChainAssert.ArtifactHashStable(artifact, "abc123...expected-sha256...");
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Requirement("REQ-DETERMINISM-001")]
|
||||
public void Test_EvidenceImmutability()
|
||||
{
|
||||
// Verify generator produces identical output across iterations
|
||||
EvidenceChainAssert.ArtifactImmutable(() => GenerateEvidence(fixedInput), iterations: 100);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
[Requirement("REQ-TRACE-001")]
|
||||
public void Test_TraceabilityComplete()
|
||||
{
|
||||
var requirementId = "REQ-EVIDENCE-001";
|
||||
var testId = "MyTests.TestMethod";
|
||||
var artifactHash = EvidenceChainAssert.ComputeSha256(artifact);
|
||||
|
||||
// Verify all traceability components present
|
||||
EvidenceChainAssert.TraceabilityComplete(requirementId, testId, artifactHash);
|
||||
}
|
||||
```
|
||||
|
||||
**Traceability Report Generation:**
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Evidence;
|
||||
|
||||
// Generate traceability matrix from test assemblies
|
||||
var reporter = new EvidenceChainReporter();
|
||||
reporter.AddAssembly(typeof(MyTests).Assembly);
|
||||
var report = reporter.GenerateReport();
|
||||
|
||||
// Output as Markdown
|
||||
Console.WriteLine(report.ToMarkdown());
|
||||
|
||||
// Output as JSON
|
||||
Console.WriteLine(report.ToJson());
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `RequirementAttribute(string requirementId)` - Link test to requirement
|
||||
- `RequirementAttribute.SprintTaskId` - Link to sprint task (optional)
|
||||
- `RequirementAttribute.ComplianceControl` - Link to compliance control (optional)
|
||||
- `EvidenceChainAssert.ArtifactHashStable(artifact, expectedHash)` - Verify hash
|
||||
- `EvidenceChainAssert.ArtifactImmutable(generator, iterations)` - Verify determinism
|
||||
- `EvidenceChainAssert.ComputeSha256(content)` - Compute SHA-256 hash
|
||||
- `EvidenceChainAssert.RequirementLinked(requirementId)` - Marker assertion
|
||||
- `EvidenceChainAssert.TraceabilityComplete(reqId, testId, artifactId)` - Verify chain
|
||||
- `EvidenceChainReporter.AddAssembly(assembly)` - Add assembly to scan
|
||||
- `EvidenceChainReporter.GenerateReport()` - Generate traceability report
|
||||
- `EvidenceChainReport.ToMarkdown()` - Markdown output
|
||||
- `EvidenceChainReport.ToJson()` - JSON output
|
||||
- `EvidenceTraceabilityException` - Thrown when evidence assertions fail
|
||||
|
||||
---
|
||||
|
||||
### 11. Test Categories
|
||||
|
||||
Standardized trait constants for CI lane filtering:
|
||||
|
||||
@@ -412,6 +621,82 @@ dotnet test --filter "Category=Integration|Category=Contract"
|
||||
- `Security` - Cryptographic validation
|
||||
- `Performance` - Benchmarking, load tests
|
||||
- `Live` - Requires external services (disabled in CI by default)
|
||||
- `PostIncident` - Tests derived from production incidents (Turn #6)
|
||||
- `EvidenceChain` - Requirement traceability tests (Turn #6)
|
||||
- `Longevity` - Time-extended stability tests (Turn #6)
|
||||
- `Interop` - Cross-version compatibility tests (Turn #6)
|
||||
|
||||
---
|
||||
|
||||
### 12. Post-Incident Testing (Turn #6)
|
||||
|
||||
Generate regression tests from production incidents:
|
||||
|
||||
**Generate Test Scaffold from Incident:**
|
||||
|
||||
```csharp
|
||||
using StellaOps.TestKit.Incident;
|
||||
|
||||
// Create incident metadata
|
||||
var metadata = new IncidentMetadata
|
||||
{
|
||||
IncidentId = "INC-2026-001",
|
||||
OccurredAt = DateTimeOffset.Parse("2026-01-15T10:30:00Z"),
|
||||
RootCause = "Race condition in concurrent bundle creation",
|
||||
AffectedModules = ["EvidenceLocker", "Policy"],
|
||||
Severity = IncidentSeverity.P1,
|
||||
Title = "Evidence bundle duplication"
|
||||
};
|
||||
|
||||
// Generate test scaffold from replay manifest
|
||||
var generator = new IncidentTestGenerator();
|
||||
var scaffold = generator.GenerateFromManifestJson(manifestJson, metadata);
|
||||
|
||||
// Output generated test code
|
||||
var code = scaffold.GenerateTestCode();
|
||||
File.WriteAllText($"Tests/{scaffold.TestClassName}.cs", code);
|
||||
```
|
||||
|
||||
**Generated Test Structure:**
|
||||
|
||||
```csharp
|
||||
[Trait("Category", TestCategories.PostIncident)]
|
||||
[Trait("Incident", "INC-2026-001")]
|
||||
[Trait("Severity", "P1")]
|
||||
public sealed class Incident_INC_2026_001_Tests
|
||||
{
|
||||
[Fact]
|
||||
public async Task Validates_RaceCondition_Fix()
|
||||
{
|
||||
// Arrange - fixtures from replay manifest
|
||||
// Act - execute the incident scenario
|
||||
// Assert - verify fix prevents recurrence
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
**Filter Post-Incident Tests:**
|
||||
|
||||
```bash
|
||||
# Run all post-incident tests
|
||||
dotnet test --filter "Category=PostIncident"
|
||||
|
||||
# Run only P1/P2 tests (release-gating)
|
||||
dotnet test --filter "Category=PostIncident&(Severity=P1|Severity=P2)"
|
||||
|
||||
# Run tests for a specific incident
|
||||
dotnet test --filter "Incident=INC-2026-001"
|
||||
```
|
||||
|
||||
**API Reference:**
|
||||
- `IncidentMetadata` - Incident context (ID, severity, root cause, modules)
|
||||
- `IncidentSeverity` - P1 (critical) through P4 (low impact)
|
||||
- `IncidentTestGenerator.GenerateFromManifestJson(json, metadata)` - Generate scaffold
|
||||
- `TestScaffold.GenerateTestCode()` - Output C# test code
|
||||
- `TestScaffold.ToJson()` / `FromJson()` - Serialize/deserialize scaffold
|
||||
- `IncidentTestGenerator.GenerateReport()` - Summary of registered incident tests
|
||||
|
||||
See [Post-Incident Testing Guide](post-incident-testing-guide.md) for complete documentation.
|
||||
|
||||
---
|
||||
|
||||
|
||||
Reference in New Issue
Block a user