Complete batch 012 (golden set diff) and 013 (advisory chat), fix build errors
Sprints completed: - SPRINT_20260110_012_* (golden set diff layer - 10 sprints) - SPRINT_20260110_013_* (advisory chat - 4 sprints) Build fixes applied: - Fix namespace conflicts with Microsoft.Extensions.Options.Options.Create - Fix VexDecisionReachabilityIntegrationTests API drift (major rewrite) - Fix VexSchemaValidationTests FluentAssertions method name - Fix FixChainGateIntegrationTests ambiguous type references - Fix AdvisoryAI test files required properties and namespace aliases - Add stub types for CveMappingController (ICveSymbolMappingService) - Fix VerdictBuilderService static context issue Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
master
294
src/__Tests/load/AdvisoryAI/advisory_chat_load_test.k6.js
Normal file
294
src/__Tests/load/AdvisoryAI/advisory_chat_load_test.k6.js
Normal file
@@ -0,0 +1,294 @@
|
||||
// advisory_chat_load_test.k6.js
|
||||
// k6 Load Test for Advisory AI Chat API
|
||||
//
|
||||
// Performance Targets:
|
||||
// | Metric | Target |
|
||||
// |--------|--------|
|
||||
// | Throughput | 50 req/s sustained |
|
||||
// | P95 Latency | < 2s |
|
||||
// | P99 Latency | < 5s |
|
||||
// | Error Rate | < 1% |
|
||||
// | Concurrent Users | 100 |
|
||||
//
|
||||
// Usage:
|
||||
// k6 run --env BASE_URL=http://localhost:5000 --env AUTH_TOKEN=your-token advisory_chat_load_test.k6.js
|
||||
|
||||
import http from 'k6/http';
|
||||
import { check, sleep } from 'k6';
|
||||
import { Rate, Trend, Counter } from 'k6/metrics';
|
||||
|
||||
// Custom metrics
|
||||
const errorRate = new Rate('errors');
|
||||
const chatLatency = new Trend('chat_latency');
|
||||
const intentLatency = new Trend('intent_latency');
|
||||
const evidencePreviewLatency = new Trend('evidence_preview_latency');
|
||||
const successfulQueries = new Counter('successful_queries');
|
||||
const failedQueries = new Counter('failed_queries');
|
||||
|
||||
export const options = {
|
||||
stages: [
|
||||
{ duration: '30s', target: 10 }, // Ramp up to 10 users
|
||||
{ duration: '2m', target: 50 }, // Sustained load at 50 users
|
||||
{ duration: '1m', target: 100 }, // Peak load at 100 users
|
||||
{ duration: '30s', target: 0 }, // Ramp down
|
||||
],
|
||||
thresholds: {
|
||||
http_req_duration: ['p(95)<2000'], // 95% of requests under 2s
|
||||
errors: ['rate<0.01'], // Error rate < 1%
|
||||
chat_latency: ['p(50)<1500', 'p(95)<2000', 'p(99)<5000'],
|
||||
},
|
||||
};
|
||||
|
||||
const BASE_URL = __ENV.BASE_URL || 'http://localhost:5000';
|
||||
|
||||
// Test data
|
||||
const testCves = [
|
||||
'CVE-2024-12345',
|
||||
'CVE-2024-67890',
|
||||
'CVE-2024-11111',
|
||||
'CVE-2024-22222',
|
||||
'CVE-2024-33333',
|
||||
'CVE-2024-44444',
|
||||
];
|
||||
|
||||
const testDigests = [
|
||||
'sha256:abc123456789def0123456789',
|
||||
'sha256:def456789abc0123456789def',
|
||||
'sha256:ghi789abc0123456789abcdef',
|
||||
'sha256:jkl012def3456789abcdefabc',
|
||||
];
|
||||
|
||||
const testEnvironments = ['prod', 'staging', 'dev', 'prod-eu1', 'prod-us1'];
|
||||
|
||||
const queryTypes = [
|
||||
{ template: '/explain {cve}', intent: 'Explain' },
|
||||
{ template: '/is-it-reachable {cve}', intent: 'IsItReachable' },
|
||||
{ template: '/do-we-have-a-backport {cve}', intent: 'DoWeHaveABackport' },
|
||||
{ template: '/propose-fix {cve}', intent: 'ProposeFix' },
|
||||
{ template: 'What is {cve}?', intent: 'Explain' },
|
||||
{ template: 'Is {cve} reachable in my application?', intent: 'IsItReachable' },
|
||||
];
|
||||
|
||||
function randomElement(arr) {
|
||||
return arr[Math.floor(Math.random() * arr.length)];
|
||||
}
|
||||
|
||||
function generateQuery() {
|
||||
const cve = randomElement(testCves);
|
||||
const queryType = randomElement(queryTypes);
|
||||
return {
|
||||
query: queryType.template.replace('{cve}', cve),
|
||||
expectedIntent: queryType.intent,
|
||||
cve: cve,
|
||||
};
|
||||
}
|
||||
|
||||
export default function () {
|
||||
const cve = randomElement(testCves);
|
||||
const digest = randomElement(testDigests);
|
||||
const environment = randomElement(testEnvironments);
|
||||
const queryData = generateQuery();
|
||||
|
||||
const params = {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Authorization': `Bearer ${__ENV.AUTH_TOKEN || 'test-token'}`,
|
||||
'X-Tenant-Id': 'load-test-tenant',
|
||||
},
|
||||
};
|
||||
|
||||
// Test 1: Main chat query endpoint
|
||||
const chatPayload = JSON.stringify({
|
||||
query: queryData.query,
|
||||
artifactDigest: digest,
|
||||
findingId: queryData.cve,
|
||||
environment: environment,
|
||||
});
|
||||
|
||||
const chatStartTime = Date.now();
|
||||
const chatRes = http.post(`${BASE_URL}/api/v1/chat/query`, chatPayload, params);
|
||||
const chatDuration = Date.now() - chatStartTime;
|
||||
|
||||
chatLatency.add(chatDuration);
|
||||
|
||||
const chatSuccess = check(chatRes, {
|
||||
'chat: status is 200': (r) => r.status === 200,
|
||||
'chat: has response': (r) => {
|
||||
try {
|
||||
const body = JSON.parse(r.body);
|
||||
return body.response !== undefined;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
'chat: has bundleId': (r) => {
|
||||
try {
|
||||
const body = JSON.parse(r.body);
|
||||
return body.bundleId !== undefined;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
});
|
||||
|
||||
if (chatSuccess) {
|
||||
successfulQueries.add(1);
|
||||
} else {
|
||||
failedQueries.add(1);
|
||||
}
|
||||
|
||||
errorRate.add(!chatSuccess);
|
||||
|
||||
// Test 2: Intent detection endpoint (lighter weight)
|
||||
if (Math.random() < 0.3) {
|
||||
const intentPayload = JSON.stringify({
|
||||
query: queryData.query,
|
||||
});
|
||||
|
||||
const intentStartTime = Date.now();
|
||||
const intentRes = http.post(`${BASE_URL}/api/v1/chat/intent`, intentPayload, params);
|
||||
const intentDuration = Date.now() - intentStartTime;
|
||||
|
||||
intentLatency.add(intentDuration);
|
||||
|
||||
check(intentRes, {
|
||||
'intent: status is 200': (r) => r.status === 200,
|
||||
'intent: has intent field': (r) => {
|
||||
try {
|
||||
const body = JSON.parse(r.body);
|
||||
return body.intent !== undefined;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Test 3: Evidence preview endpoint (occasional)
|
||||
if (Math.random() < 0.2) {
|
||||
const previewPayload = JSON.stringify({
|
||||
findingId: cve,
|
||||
artifactDigest: digest,
|
||||
});
|
||||
|
||||
const previewStartTime = Date.now();
|
||||
const previewRes = http.post(`${BASE_URL}/api/v1/chat/evidence-preview`, previewPayload, params);
|
||||
const previewDuration = Date.now() - previewStartTime;
|
||||
|
||||
evidencePreviewLatency.add(previewDuration);
|
||||
|
||||
check(previewRes, {
|
||||
'preview: status is 200': (r) => r.status === 200,
|
||||
});
|
||||
}
|
||||
|
||||
// Test 4: Status endpoint (occasional health check)
|
||||
if (Math.random() < 0.1) {
|
||||
const statusRes = http.get(`${BASE_URL}/api/v1/chat/status`, params);
|
||||
|
||||
check(statusRes, {
|
||||
'status: is 200': (r) => r.status === 200,
|
||||
'status: chat enabled': (r) => {
|
||||
try {
|
||||
const body = JSON.parse(r.body);
|
||||
return body.enabled === true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// Think time: 1-3 seconds between requests
|
||||
sleep(Math.random() * 2 + 1);
|
||||
}
|
||||
|
||||
// Teardown function for summary
|
||||
export function handleSummary(data) {
|
||||
const summary = {
|
||||
timestamp: new Date().toISOString(),
|
||||
baseUrl: BASE_URL,
|
||||
metrics: {
|
||||
http_req_duration: {
|
||||
avg: data.metrics.http_req_duration?.values?.avg,
|
||||
p50: data.metrics.http_req_duration?.values?.['p(50)'],
|
||||
p95: data.metrics.http_req_duration?.values?.['p(95)'],
|
||||
p99: data.metrics.http_req_duration?.values?.['p(99)'],
|
||||
},
|
||||
chat_latency: {
|
||||
avg: data.metrics.chat_latency?.values?.avg,
|
||||
p50: data.metrics.chat_latency?.values?.['p(50)'],
|
||||
p95: data.metrics.chat_latency?.values?.['p(95)'],
|
||||
p99: data.metrics.chat_latency?.values?.['p(99)'],
|
||||
},
|
||||
error_rate: data.metrics.errors?.values?.rate,
|
||||
successful_queries: data.metrics.successful_queries?.values?.count,
|
||||
failed_queries: data.metrics.failed_queries?.values?.count,
|
||||
},
|
||||
thresholds_passed: Object.entries(data.thresholds || {}).every(
|
||||
([, v]) => v.ok
|
||||
),
|
||||
};
|
||||
|
||||
return {
|
||||
'stdout': textSummary(data, { indent: ' ', enableColors: true }),
|
||||
'results/advisory_chat_load_test.json': JSON.stringify(summary, null, 2),
|
||||
};
|
||||
}
|
||||
|
||||
// Custom text summary
|
||||
function textSummary(data, options) {
|
||||
const indent = options.indent || ' ';
|
||||
const lines = [];
|
||||
|
||||
lines.push('');
|
||||
lines.push('='.repeat(60));
|
||||
lines.push(' Advisory Chat Load Test Results');
|
||||
lines.push('='.repeat(60));
|
||||
lines.push('');
|
||||
|
||||
// Request summary
|
||||
if (data.metrics.http_reqs) {
|
||||
lines.push(`${indent}Total Requests: ${data.metrics.http_reqs.values.count}`);
|
||||
lines.push(`${indent}Requests/s: ${data.metrics.http_reqs.values.rate?.toFixed(2)}`);
|
||||
}
|
||||
|
||||
// Latency summary
|
||||
if (data.metrics.http_req_duration) {
|
||||
lines.push('');
|
||||
lines.push(`${indent}HTTP Request Duration:`);
|
||||
lines.push(`${indent}${indent}avg: ${data.metrics.http_req_duration.values.avg?.toFixed(2)}ms`);
|
||||
lines.push(`${indent}${indent}p50: ${data.metrics.http_req_duration.values['p(50)']?.toFixed(2)}ms`);
|
||||
lines.push(`${indent}${indent}p95: ${data.metrics.http_req_duration.values['p(95)']?.toFixed(2)}ms`);
|
||||
lines.push(`${indent}${indent}p99: ${data.metrics.http_req_duration.values['p(99)']?.toFixed(2)}ms`);
|
||||
}
|
||||
|
||||
// Chat latency
|
||||
if (data.metrics.chat_latency) {
|
||||
lines.push('');
|
||||
lines.push(`${indent}Chat Query Latency:`);
|
||||
lines.push(`${indent}${indent}avg: ${data.metrics.chat_latency.values.avg?.toFixed(2)}ms`);
|
||||
lines.push(`${indent}${indent}p50: ${data.metrics.chat_latency.values['p(50)']?.toFixed(2)}ms`);
|
||||
lines.push(`${indent}${indent}p95: ${data.metrics.chat_latency.values['p(95)']?.toFixed(2)}ms`);
|
||||
lines.push(`${indent}${indent}p99: ${data.metrics.chat_latency.values['p(99)']?.toFixed(2)}ms`);
|
||||
}
|
||||
|
||||
// Error rate
|
||||
if (data.metrics.errors) {
|
||||
lines.push('');
|
||||
lines.push(`${indent}Error Rate: ${(data.metrics.errors.values.rate * 100).toFixed(2)}%`);
|
||||
}
|
||||
|
||||
// Threshold results
|
||||
lines.push('');
|
||||
lines.push(`${indent}Threshold Results:`);
|
||||
for (const [name, result] of Object.entries(data.thresholds || {})) {
|
||||
const status = result.ok ? 'PASS' : 'FAIL';
|
||||
lines.push(`${indent}${indent}${name}: ${status}`);
|
||||
}
|
||||
|
||||
lines.push('');
|
||||
lines.push('='.repeat(60));
|
||||
|
||||
return lines.join('\n');
|
||||
}
|
||||
Reference in New Issue
Block a user