documentation cleanse, sprints work and planning. remaining non EF DAL migration to EF
This commit is contained in:
master
@@ -0,0 +1,6 @@
|
||||
{
|
||||
"capturedAtUtc": "2026-02-11T10:44:56.9877336+00:00",
|
||||
"approved": true,
|
||||
"reason": "Failure reproduced and root cause validated from Tier 2 logs and benchmark config path/threshold inspection.",
|
||||
"revisedRootCause": "Config path/threshold mismatch, not missing implementation."
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"iterations": 1,
|
||||
"thresholdMs": 60000,
|
||||
"minThroughputPerSecond": 1,
|
||||
"maxAllocatedMb": 4096,
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "policy_eval_baseline",
|
||||
"label": "Policy evaluation (100k components, 1M findings)",
|
||||
"policyPath": "src/Bench/StellaOps.Bench/PolicyEngine/policies/benchmark-default.policy.yaml",
|
||||
"componentCount": 100000,
|
||||
"advisoriesPerComponent": 10,
|
||||
"seed": 20251026,
|
||||
"thresholdMs": 60000,
|
||||
"minThroughputPerSecond": 1,
|
||||
"maxAllocatedMb": 4096
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,78 @@
|
||||
# HELP linknotmerge_bench_total_ms Link-Not-Merge benchmark total duration metrics (milliseconds).
|
||||
# TYPE linknotmerge_bench_total_ms gauge
|
||||
# HELP linknotmerge_bench_correlation_ms Link-Not-Merge benchmark correlation duration metrics (milliseconds).
|
||||
# TYPE linknotmerge_bench_correlation_ms gauge
|
||||
# HELP linknotmerge_bench_insert_ms Link-Not-Merge benchmark insert duration metrics (milliseconds).
|
||||
# TYPE linknotmerge_bench_insert_ms gauge
|
||||
# HELP linknotmerge_bench_throughput_per_sec Link-Not-Merge benchmark throughput metrics (observations per second).
|
||||
# TYPE linknotmerge_bench_throughput_per_sec gauge
|
||||
# HELP linknotmerge_bench_insert_throughput_per_sec Link-Not-Merge benchmark insert throughput metrics (operations per second).
|
||||
# TYPE linknotmerge_bench_insert_throughput_per_sec gauge
|
||||
# HELP linknotmerge_bench_allocated_mb Link-Not-Merge benchmark allocation metrics (megabytes).
|
||||
# TYPE linknotmerge_bench_allocated_mb gauge
|
||||
linknotmerge_bench_mean_total_ms{scenario="lnm_ingest_baseline"} 35.043779999999998
|
||||
linknotmerge_bench_p95_total_ms{scenario="lnm_ingest_baseline"} 51.970559999999999
|
||||
linknotmerge_bench_max_total_ms{scenario="lnm_ingest_baseline"} 55.704700000000003
|
||||
linknotmerge_bench_threshold_ms{scenario="lnm_ingest_baseline"} 900
|
||||
linknotmerge_bench_mean_correlation_ms{scenario="lnm_ingest_baseline"} 33.893740000000001
|
||||
linknotmerge_bench_mean_insert_ms{scenario="lnm_ingest_baseline"} 1.15004
|
||||
linknotmerge_bench_mean_throughput_per_sec{scenario="lnm_ingest_baseline"} 154982.73909637451
|
||||
linknotmerge_bench_min_throughput_per_sec{scenario="lnm_ingest_baseline"} 89759.03290027591
|
||||
linknotmerge_bench_throughput_floor_per_sec{scenario="lnm_ingest_baseline"} 5500
|
||||
linknotmerge_bench_mean_insert_throughput_per_sec{scenario="lnm_ingest_baseline"} 7780752.3724147053
|
||||
linknotmerge_bench_min_insert_throughput_per_sec{scenario="lnm_ingest_baseline"} 1590431.9613206948
|
||||
linknotmerge_bench_insert_throughput_floor_per_sec{scenario="lnm_ingest_baseline"} 8000
|
||||
linknotmerge_bench_max_allocated_mb{scenario="lnm_ingest_baseline"} 10.450103759765625
|
||||
linknotmerge_bench_max_allocated_threshold_mb{scenario="lnm_ingest_baseline"} 160
|
||||
linknotmerge_bench_baseline_max_total_ms{scenario="lnm_ingest_baseline"} 866.62360000000001
|
||||
linknotmerge_bench_baseline_min_throughput_per_sec{scenario="lnm_ingest_baseline"} 5769.5174999999999
|
||||
linknotmerge_bench_baseline_min_insert_throughput_per_sec{scenario="lnm_ingest_baseline"} 8405.1257000000005
|
||||
linknotmerge_bench_duration_regression_ratio{scenario="lnm_ingest_baseline"} 0.064277847960752518
|
||||
linknotmerge_bench_throughput_regression_ratio{scenario="lnm_ingest_baseline"} 0.064277848296450002
|
||||
linknotmerge_bench_insert_throughput_regression_ratio{scenario="lnm_ingest_baseline"} 0.0052848068351320004
|
||||
linknotmerge_bench_regression_limit{scenario="lnm_ingest_baseline"} 1000
|
||||
linknotmerge_bench_regression_breached{scenario="lnm_ingest_baseline"} 0
|
||||
linknotmerge_bench_mean_total_ms{scenario="lnm_ingest_fanout_medium"} 97.098539999999986
|
||||
linknotmerge_bench_p95_total_ms{scenario="lnm_ingest_fanout_medium"} 111.86999999999999
|
||||
linknotmerge_bench_max_total_ms{scenario="lnm_ingest_fanout_medium"} 113.34139999999999
|
||||
linknotmerge_bench_threshold_ms{scenario="lnm_ingest_fanout_medium"} 1300
|
||||
linknotmerge_bench_mean_correlation_ms{scenario="lnm_ingest_fanout_medium"} 96.200180000000003
|
||||
linknotmerge_bench_mean_insert_ms{scenario="lnm_ingest_fanout_medium"} 0.89836000000000005
|
||||
linknotmerge_bench_mean_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 104431.72059520807
|
||||
linknotmerge_bench_min_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 88229.014287806582
|
||||
linknotmerge_bench_throughput_floor_per_sec{scenario="lnm_ingest_fanout_medium"} 8000
|
||||
linknotmerge_bench_mean_insert_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 11766340.198140498
|
||||
linknotmerge_bench_min_insert_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 9242998.4286902677
|
||||
linknotmerge_bench_insert_throughput_floor_per_sec{scenario="lnm_ingest_fanout_medium"} 13000
|
||||
linknotmerge_bench_max_allocated_mb{scenario="lnm_ingest_fanout_medium"} 30.427078247070312
|
||||
linknotmerge_bench_max_allocated_threshold_mb{scenario="lnm_ingest_fanout_medium"} 220
|
||||
linknotmerge_bench_baseline_max_total_ms{scenario="lnm_ingest_fanout_medium"} 842.88149999999996
|
||||
linknotmerge_bench_baseline_min_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 11864.063899999999
|
||||
linknotmerge_bench_baseline_min_insert_throughput_per_sec{scenario="lnm_ingest_fanout_medium"} 20891.0579
|
||||
linknotmerge_bench_duration_regression_ratio{scenario="lnm_ingest_fanout_medium"} 0.13446896153255233
|
||||
linknotmerge_bench_throughput_regression_ratio{scenario="lnm_ingest_fanout_medium"} 0.13446896121154597
|
||||
linknotmerge_bench_insert_throughput_regression_ratio{scenario="lnm_ingest_fanout_medium"} 0.0022602035542009999
|
||||
linknotmerge_bench_regression_limit{scenario="lnm_ingest_fanout_medium"} 1000
|
||||
linknotmerge_bench_regression_breached{scenario="lnm_ingest_fanout_medium"} 0
|
||||
linknotmerge_bench_mean_total_ms{scenario="lnm_ingest_fanout_high"} 145.11756
|
||||
linknotmerge_bench_p95_total_ms{scenario="lnm_ingest_fanout_high"} 170.75353999999999
|
||||
linknotmerge_bench_max_total_ms{scenario="lnm_ingest_fanout_high"} 173.8039
|
||||
linknotmerge_bench_threshold_ms{scenario="lnm_ingest_fanout_high"} 2200
|
||||
linknotmerge_bench_mean_correlation_ms{scenario="lnm_ingest_fanout_high"} 144.03708
|
||||
linknotmerge_bench_mean_insert_ms{scenario="lnm_ingest_fanout_high"} 1.0804800000000001
|
||||
linknotmerge_bench_mean_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 104958.0651416691
|
||||
linknotmerge_bench_min_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 86304.162334677181
|
||||
linknotmerge_bench_throughput_floor_per_sec{scenario="lnm_ingest_fanout_high"} 7000
|
||||
linknotmerge_bench_mean_insert_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 14102031.90643465
|
||||
linknotmerge_bench_min_insert_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 11248593.92575928
|
||||
linknotmerge_bench_insert_throughput_floor_per_sec{scenario="lnm_ingest_fanout_high"} 13000
|
||||
linknotmerge_bench_max_allocated_mb{scenario="lnm_ingest_fanout_high"} 47.022369384765625
|
||||
linknotmerge_bench_max_allocated_threshold_mb{scenario="lnm_ingest_fanout_high"} 300
|
||||
linknotmerge_bench_baseline_max_total_ms{scenario="lnm_ingest_fanout_high"} 1369.943
|
||||
linknotmerge_bench_baseline_min_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 10949.360699999999
|
||||
linknotmerge_bench_baseline_min_insert_throughput_per_sec{scenario="lnm_ingest_fanout_high"} 19781.676200000002
|
||||
linknotmerge_bench_duration_regression_ratio{scenario="lnm_ingest_fanout_high"} 0.12686943909345133
|
||||
linknotmerge_bench_throughput_regression_ratio{scenario="lnm_ingest_fanout_high"} 0.12686943947778201
|
||||
linknotmerge_bench_insert_throughput_regression_ratio{scenario="lnm_ingest_fanout_high"} 0.0017585910141800002
|
||||
linknotmerge_bench_regression_limit{scenario="lnm_ingest_fanout_high"} 1000
|
||||
linknotmerge_bench_regression_breached{scenario="lnm_ingest_fanout_high"} 0
|
||||
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,25 @@
|
||||
{
|
||||
"iterations": 1,
|
||||
"thresholdMs": 60000,
|
||||
"minThroughputPerSecond": 1,
|
||||
"minInsertThroughputPerSecond": 1,
|
||||
"maxAllocatedMb": 4096,
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "lnm_probe",
|
||||
"label": "LNM probe scenario",
|
||||
"observations": 400,
|
||||
"aliasGroups": 40,
|
||||
"purlsPerObservation": 3,
|
||||
"cpesPerObservation": 2,
|
||||
"referencesPerObservation": 2,
|
||||
"tenants": 2,
|
||||
"batchSize": 100,
|
||||
"seed": 111,
|
||||
"thresholdMs": 60000,
|
||||
"minThroughputPerSecond": 1,
|
||||
"minInsertThroughputPerSecond": 1,
|
||||
"maxAllocatedMb": 4096
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
{
|
||||
"schemaVersion": "linknotmerge-bench/1.0",
|
||||
"capturedAt": "2026-02-11T10:43:08.6431332+00:00",
|
||||
"commit": "local",
|
||||
"environment": "qa-flow",
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "lnm_ingest_baseline",
|
||||
"label": "5k observations, 500 aliases",
|
||||
"iterations": 5,
|
||||
"observations": 5000,
|
||||
"aliases": 500,
|
||||
"linksets": 6000,
|
||||
"meanTotalMs": 35.04378,
|
||||
"p95TotalMs": 51.97056,
|
||||
"maxTotalMs": 55.7047,
|
||||
"meanInsertMs": 1.15004,
|
||||
"meanCorrelationMs": 33.89374,
|
||||
"meanThroughputPerSecond": 154982.7390963745,
|
||||
"minThroughputPerSecond": 89759.03290027591,
|
||||
"meanInsertThroughputPerSecond": 7780752.372414705,
|
||||
"minInsertThroughputPerSecond": 1590431.9613206948,
|
||||
"maxAllocatedMb": 10.450103759765625,
|
||||
"thresholdMs": 900,
|
||||
"minThroughputThresholdPerSecond": 5500,
|
||||
"minInsertThroughputThresholdPerSecond": 8000,
|
||||
"maxAllocatedThresholdMb": 160,
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"observations": 5000,
|
||||
"aliases": 500,
|
||||
"linksets": 6000,
|
||||
"meanTotalMs": 555.1984,
|
||||
"p95TotalMs": 823.4957,
|
||||
"maxTotalMs": 866.6236,
|
||||
"meanInsertMs": 366.2635,
|
||||
"meanCorrelationMs": 188.9349,
|
||||
"meanThroughputPerSecond": 9877.7916,
|
||||
"minThroughputPerSecond": 5769.5175,
|
||||
"meanInsertThroughputPerSecond": 15338.0851,
|
||||
"minInsertThroughputPerSecond": 8405.1257,
|
||||
"maxAllocatedMb": 62.4477
|
||||
},
|
||||
"regression": {
|
||||
"durationRatio": 0.06427784796075252,
|
||||
"throughputRatio": 0.06427784829645,
|
||||
"insertThroughputRatio": 0.005284806835132,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "lnm_ingest_fanout_medium",
|
||||
"label": "10k observations, 800 aliases",
|
||||
"iterations": 5,
|
||||
"observations": 10000,
|
||||
"aliases": 800,
|
||||
"linksets": 14800,
|
||||
"meanTotalMs": 97.09853999999999,
|
||||
"p95TotalMs": 111.86999999999999,
|
||||
"maxTotalMs": 113.3414,
|
||||
"meanInsertMs": 0.89836,
|
||||
"meanCorrelationMs": 96.20018,
|
||||
"meanThroughputPerSecond": 104431.72059520807,
|
||||
"minThroughputPerSecond": 88229.01428780658,
|
||||
"meanInsertThroughputPerSecond": 11766340.198140498,
|
||||
"minInsertThroughputPerSecond": 9242998.428690268,
|
||||
"maxAllocatedMb": 30.427078247070312,
|
||||
"thresholdMs": 1300,
|
||||
"minThroughputThresholdPerSecond": 8000,
|
||||
"minInsertThroughputThresholdPerSecond": 13000,
|
||||
"maxAllocatedThresholdMb": 220,
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"observations": 10000,
|
||||
"aliases": 800,
|
||||
"linksets": 14800,
|
||||
"meanTotalMs": 785.8909,
|
||||
"p95TotalMs": 841.6247,
|
||||
"maxTotalMs": 842.8815,
|
||||
"meanInsertMs": 453.5087,
|
||||
"meanCorrelationMs": 332.3822,
|
||||
"meanThroughputPerSecond": 12794.955,
|
||||
"minThroughputPerSecond": 11864.0639,
|
||||
"meanInsertThroughputPerSecond": 22086.032,
|
||||
"minInsertThroughputPerSecond": 20891.0579,
|
||||
"maxAllocatedMb": 145.8328
|
||||
},
|
||||
"regression": {
|
||||
"durationRatio": 0.13446896153255233,
|
||||
"throughputRatio": 0.13446896121154597,
|
||||
"insertThroughputRatio": 0.002260203554201,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "lnm_ingest_fanout_high",
|
||||
"label": "15k observations, 1200 aliases",
|
||||
"iterations": 5,
|
||||
"observations": 15000,
|
||||
"aliases": 1200,
|
||||
"linksets": 17400,
|
||||
"meanTotalMs": 145.11756,
|
||||
"p95TotalMs": 170.75354,
|
||||
"maxTotalMs": 173.8039,
|
||||
"meanInsertMs": 1.08048,
|
||||
"meanCorrelationMs": 144.03708,
|
||||
"meanThroughputPerSecond": 104958.0651416691,
|
||||
"minThroughputPerSecond": 86304.16233467718,
|
||||
"meanInsertThroughputPerSecond": 14102031.90643465,
|
||||
"minInsertThroughputPerSecond": 11248593.92575928,
|
||||
"maxAllocatedMb": 47.022369384765625,
|
||||
"thresholdMs": 2200,
|
||||
"minThroughputThresholdPerSecond": 7000,
|
||||
"minInsertThroughputThresholdPerSecond": 13000,
|
||||
"maxAllocatedThresholdMb": 300,
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"observations": 15000,
|
||||
"aliases": 1200,
|
||||
"linksets": 17400,
|
||||
"meanTotalMs": 1299.3458,
|
||||
"p95TotalMs": 1367.0934,
|
||||
"maxTotalMs": 1369.943,
|
||||
"meanInsertMs": 741.6265,
|
||||
"meanCorrelationMs": 557.7193,
|
||||
"meanThroughputPerSecond": 11571.0991,
|
||||
"minThroughputPerSecond": 10949.3607,
|
||||
"meanInsertThroughputPerSecond": 20232.518,
|
||||
"minInsertThroughputPerSecond": 19781.6762,
|
||||
"maxAllocatedMb": 238.345
|
||||
},
|
||||
"regression": {
|
||||
"durationRatio": 0.12686943909345133,
|
||||
"throughputRatio": 0.126869439477782,
|
||||
"insertThroughputRatio": 0.0017585910141800002,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
scenario,iterations,observations,aliases,linksets,mean_total_ms,p95_total_ms,max_total_ms,mean_insert_ms,mean_correlation_ms,mean_throughput_per_sec,min_throughput_per_sec,mean_insert_throughput_per_sec,min_insert_throughput_per_sec,max_allocated_mb
|
||||
lnm_ingest_baseline,5,5000,500,6000,35.0438,51.9706,55.7047,1.1500,33.8937,154982.7391,89759.0329,7780752.3724,1590431.9613,10.4501
|
||||
lnm_ingest_fanout_medium,5,10000,800,14800,97.0985,111.8700,113.3414,0.8984,96.2002,104431.7206,88229.0143,11766340.1981,9242998.4287,30.4271
|
||||
lnm_ingest_fanout_high,5,15000,1200,17400,145.1176,170.7535,173.8039,1.0805,144.0371,104958.0651,86304.1623,14102031.9064,11248593.9258,47.0224
|
||||
|
@@ -0,0 +1,19 @@
|
||||
{
|
||||
"iterations": 1,
|
||||
"thresholdMs": 60000,
|
||||
"minThroughputPerSecond": 1,
|
||||
"maxAllocatedMb": 4096,
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "policy_probe",
|
||||
"label": "Policy probe scenario",
|
||||
"policyPath": "src/Bench/StellaOps.Bench/PolicyEngine/policies/benchmark-default.yaml",
|
||||
"componentCount": 200,
|
||||
"advisoriesPerComponent": 2,
|
||||
"seed": 222,
|
||||
"thresholdMs": 60000,
|
||||
"minThroughputPerSecond": 1,
|
||||
"maxAllocatedMb": 4096
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
# HELP policy_engine_bench_duration_ms Policy Engine benchmark duration metrics (milliseconds).
|
||||
# TYPE policy_engine_bench_duration_ms gauge
|
||||
# HELP policy_engine_bench_throughput_per_sec Policy Engine benchmark throughput metrics (findings per second).
|
||||
# TYPE policy_engine_bench_throughput_per_sec gauge
|
||||
# HELP policy_engine_bench_allocation_mb Policy Engine benchmark allocation metrics (megabytes).
|
||||
# TYPE policy_engine_bench_allocation_mb gauge
|
||||
policy_engine_bench_mean_ms{scenario="policy_eval_baseline"} 3962.0273000000002
|
||||
policy_engine_bench_p95_ms{scenario="policy_eval_baseline"} 3962.0273000000002
|
||||
policy_engine_bench_max_ms{scenario="policy_eval_baseline"} 3962.0273000000002
|
||||
policy_engine_bench_threshold_ms{scenario="policy_eval_baseline"} 60000
|
||||
policy_engine_bench_mean_throughput_per_sec{scenario="policy_eval_baseline"} 252396.03977489
|
||||
policy_engine_bench_min_throughput_per_sec{scenario="policy_eval_baseline"} 252396.03977489
|
||||
policy_engine_bench_min_throughput_threshold_per_sec{scenario="policy_eval_baseline"} 1
|
||||
policy_engine_bench_max_allocated_mb{scenario="policy_eval_baseline"} 1827.0887680053711
|
||||
policy_engine_bench_max_allocated_threshold_mb{scenario="policy_eval_baseline"} 4096
|
||||
policy_engine_bench_baseline_max_ms{scenario="policy_eval_baseline"} 1280.1721
|
||||
policy_engine_bench_baseline_mean_ms{scenario="policy_eval_baseline"} 1109.3542
|
||||
policy_engine_bench_baseline_min_throughput_per_sec{scenario="policy_eval_baseline"} 781144.97259999998
|
||||
policy_engine_bench_duration_regression_ratio{scenario="policy_eval_baseline"} 3.0949177067677076
|
||||
policy_engine_bench_throughput_regression_ratio{scenario="policy_eval_baseline"} 3.0949177066989519
|
||||
policy_engine_bench_regression_limit{scenario="policy_eval_baseline"} 1000
|
||||
policy_engine_bench_regression_breached{scenario="policy_eval_baseline"} 0
|
||||
Binary file not shown.
@@ -0,0 +1,39 @@
|
||||
{
|
||||
"schemaVersion": "policy-bench/1.0",
|
||||
"capturedAt": "2026-02-11T10:41:27.1153626+00:00",
|
||||
"commit": "local",
|
||||
"environment": "qa-flow",
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "policy_eval_baseline",
|
||||
"label": "Policy evaluation (100k components, 1M findings)",
|
||||
"iterations": 1,
|
||||
"findingCount": 1000000,
|
||||
"meanMs": 3962.0273,
|
||||
"p95Ms": 3962.0273,
|
||||
"maxMs": 3962.0273,
|
||||
"meanThroughputPerSecond": 252396.03977489,
|
||||
"minThroughputPerSecond": 252396.03977489,
|
||||
"maxAllocatedMb": 1827.088768005371,
|
||||
"thresholdMs": 60000,
|
||||
"minThroughputThresholdPerSecond": 1,
|
||||
"maxAllocatedThresholdMb": 4096,
|
||||
"baseline": {
|
||||
"iterations": 3,
|
||||
"findingCount": 1000000,
|
||||
"meanMs": 1109.3542,
|
||||
"p95Ms": 1257.7493,
|
||||
"maxMs": 1280.1721,
|
||||
"meanThroughputPerSecond": 912094.5581,
|
||||
"minThroughputPerSecond": 781144.9726,
|
||||
"maxAllocatedMb": 563.6901
|
||||
},
|
||||
"regression": {
|
||||
"durationRatio": 3.0949177067677076,
|
||||
"throughputRatio": 3.094917706698952,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,2 @@
|
||||
scenario,iterations,findings,mean_ms,p95_ms,max_ms,mean_throughput_per_sec,min_throughput_per_sec,max_allocated_mb
|
||||
policy_eval_baseline,1,1000000,3962.0273,3962.0273,3962.0273,252396.0398,252396.0398,1827.0888
|
||||
|
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"thresholdMs": 60000,
|
||||
"iterations": 1,
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "scanner_probe",
|
||||
"label": "Scanner analyzer probe scenario",
|
||||
"root": "samples/runtime/npm-monorepo",
|
||||
"analyzers": [
|
||||
"node"
|
||||
],
|
||||
"thresholdMs": 60000
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
# HELP scanner_analyzer_bench_duration_ms Analyzer benchmark duration metrics in milliseconds.
|
||||
# TYPE scanner_analyzer_bench_duration_ms gauge
|
||||
# HELP scanner_analyzer_bench_sample_count Analyzer benchmark sample counts (component/file counts).
|
||||
# TYPE scanner_analyzer_bench_sample_count gauge
|
||||
# HELP scanner_analyzer_bench_metric Additional analyzer benchmark metrics.
|
||||
# TYPE scanner_analyzer_bench_metric gauge
|
||||
# HELP scanner_analyzer_vendor_parity_score Scanner finding parity score against vendor findings.
|
||||
# TYPE scanner_analyzer_vendor_parity_score gauge
|
||||
# HELP scanner_analyzer_vendor_overlap_percent Scanner overlap percentage against vendor findings.
|
||||
# TYPE scanner_analyzer_vendor_overlap_percent gauge
|
||||
# HELP scanner_analyzer_vendor_overlap_count Scanner overlap finding count against vendor findings.
|
||||
# TYPE scanner_analyzer_vendor_overlap_count gauge
|
||||
# HELP scanner_analyzer_vendor_stella_only_count Scanner-only findings against vendor findings.
|
||||
# TYPE scanner_analyzer_vendor_stella_only_count gauge
|
||||
# HELP scanner_analyzer_vendor_vendor_only_count Vendor-only findings compared to scanner findings.
|
||||
# TYPE scanner_analyzer_vendor_vendor_only_count gauge
|
||||
# HELP scanner_analyzer_vendor_finding_count Finding counts by source for scanner/vendor comparison.
|
||||
# TYPE scanner_analyzer_vendor_finding_count gauge
|
||||
scanner_analyzer_bench_mean_ms{scenario="node_monorepo_walk"} 33.999919999999996
|
||||
scanner_analyzer_bench_p95_ms{scenario="node_monorepo_walk"} 109.66473999999998
|
||||
scanner_analyzer_bench_max_ms{scenario="node_monorepo_walk"} 134.3766
|
||||
scanner_analyzer_bench_threshold_ms{scenario="node_monorepo_walk"} 5000
|
||||
scanner_analyzer_bench_sample_count{scenario="node_monorepo_walk"} 4
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="node_monorepo_walk"} 61.714599999999997
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="node_monorepo_walk"} 15.539899999999999
|
||||
scanner_analyzer_bench_regression_ratio{scenario="node_monorepo_walk"} 2.1773875225635426
|
||||
scanner_analyzer_bench_regression_limit{scenario="node_monorepo_walk"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="node_monorepo_walk"} 0
|
||||
scanner_analyzer_bench_metric{scenario="node_monorepo_walk",name="node.importScan.bytesScanned"} 0
|
||||
scanner_analyzer_bench_metric{scenario="node_monorepo_walk",name="node.importScan.cappedPackages"} 0
|
||||
scanner_analyzer_bench_metric{scenario="node_monorepo_walk",name="node.importScan.filesScanned"} 0
|
||||
scanner_analyzer_bench_metric{scenario="node_monorepo_walk",name="node.importScan.packages"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="node_detection_gaps_fixture"} 57.328179999999996
|
||||
scanner_analyzer_bench_p95_ms{scenario="node_detection_gaps_fixture"} 141.81315999999998
|
||||
scanner_analyzer_bench_max_ms{scenario="node_detection_gaps_fixture"} 162.71019999999999
|
||||
scanner_analyzer_bench_threshold_ms{scenario="node_detection_gaps_fixture"} 2000
|
||||
scanner_analyzer_bench_sample_count{scenario="node_detection_gaps_fixture"} 5
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="node_detection_gaps_fixture"} 117.32380000000001
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="node_detection_gaps_fixture"} 31.843399999999999
|
||||
scanner_analyzer_bench_regression_ratio{scenario="node_detection_gaps_fixture"} 1.3868473404373194
|
||||
scanner_analyzer_bench_regression_limit{scenario="node_detection_gaps_fixture"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="node_detection_gaps_fixture"} 0
|
||||
scanner_analyzer_bench_metric{scenario="node_detection_gaps_fixture",name="node.importScan.bytesScanned"} 596
|
||||
scanner_analyzer_bench_metric{scenario="node_detection_gaps_fixture",name="node.importScan.cappedPackages"} 0
|
||||
scanner_analyzer_bench_metric{scenario="node_detection_gaps_fixture",name="node.importScan.filesScanned"} 6
|
||||
scanner_analyzer_bench_metric{scenario="node_detection_gaps_fixture",name="node.importScan.packages"} 3
|
||||
scanner_analyzer_bench_mean_ms{scenario="java_demo_archive"} 19.343060000000001
|
||||
scanner_analyzer_bench_p95_ms{scenario="java_demo_archive"} 68.324699999999993
|
||||
scanner_analyzer_bench_max_ms{scenario="java_demo_archive"} 84.543800000000005
|
||||
scanner_analyzer_bench_threshold_ms{scenario="java_demo_archive"} 5000
|
||||
scanner_analyzer_bench_sample_count{scenario="java_demo_archive"} 1
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="java_demo_archive"} 61.310000000000002
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="java_demo_archive"} 13.6363
|
||||
scanner_analyzer_bench_regression_ratio{scenario="java_demo_archive"} 1.3789561246126243
|
||||
scanner_analyzer_bench_regression_limit{scenario="java_demo_archive"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="java_demo_archive"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="java_fat_archive"} 7.6045199999999991
|
||||
scanner_analyzer_bench_p95_ms{scenario="java_fat_archive"} 15.618879999999997
|
||||
scanner_analyzer_bench_max_ms{scenario="java_fat_archive"} 17.843699999999998
|
||||
scanner_analyzer_bench_threshold_ms{scenario="java_fat_archive"} 1000
|
||||
scanner_analyzer_bench_sample_count{scenario="java_fat_archive"} 2
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="java_fat_archive"} 9.4926999999999992
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="java_fat_archive"} 3.5181
|
||||
scanner_analyzer_bench_regression_ratio{scenario="java_fat_archive"} 1.8797286335815941
|
||||
scanner_analyzer_bench_regression_limit{scenario="java_fat_archive"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="java_fat_archive"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="go_buildinfo_fixture"} 11.54748
|
||||
scanner_analyzer_bench_p95_ms{scenario="go_buildinfo_fixture"} 40.536439999999992
|
||||
scanner_analyzer_bench_max_ms{scenario="go_buildinfo_fixture"} 49.950200000000002
|
||||
scanner_analyzer_bench_threshold_ms{scenario="go_buildinfo_fixture"} 5000
|
||||
scanner_analyzer_bench_sample_count{scenario="go_buildinfo_fixture"} 2
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="go_buildinfo_fixture"} 32.130400000000002
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="go_buildinfo_fixture"} 6.9861000000000004
|
||||
scanner_analyzer_bench_regression_ratio{scenario="go_buildinfo_fixture"} 1.5546087194681673
|
||||
scanner_analyzer_bench_regression_limit{scenario="go_buildinfo_fixture"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="go_buildinfo_fixture"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="dotnet_multirid_fixture"} 33.24756
|
||||
scanner_analyzer_bench_p95_ms{scenario="dotnet_multirid_fixture"} 70.276060000000001
|
||||
scanner_analyzer_bench_max_ms{scenario="dotnet_multirid_fixture"} 81.811700000000002
|
||||
scanner_analyzer_bench_threshold_ms{scenario="dotnet_multirid_fixture"} 5000
|
||||
scanner_analyzer_bench_sample_count{scenario="dotnet_multirid_fixture"} 2
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="dotnet_multirid_fixture"} 47.8401
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="dotnet_multirid_fixture"} 11.826599999999999
|
||||
scanner_analyzer_bench_regression_ratio{scenario="dotnet_multirid_fixture"} 1.7101072113143576
|
||||
scanner_analyzer_bench_regression_limit{scenario="dotnet_multirid_fixture"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="dotnet_multirid_fixture"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="python_site_packages_scan"} 71.160319999999999
|
||||
scanner_analyzer_bench_p95_ms{scenario="python_site_packages_scan"} 182.98255999999995
|
||||
scanner_analyzer_bench_max_ms{scenario="python_site_packages_scan"} 216.74879999999999
|
||||
scanner_analyzer_bench_threshold_ms{scenario="python_site_packages_scan"} 5000
|
||||
scanner_analyzer_bench_sample_count{scenario="python_site_packages_scan"} 3
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="python_site_packages_scan"} 128.4211
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="python_site_packages_scan"} 36.792999999999999
|
||||
scanner_analyzer_bench_regression_ratio{scenario="python_site_packages_scan"} 1.6877974102386601
|
||||
scanner_analyzer_bench_regression_limit{scenario="python_site_packages_scan"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="python_site_packages_scan"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="python_pip_cache_fixture"} 39.351260000000003
|
||||
scanner_analyzer_bench_p95_ms{scenario="python_pip_cache_fixture"} 60.568299999999994
|
||||
scanner_analyzer_bench_max_ms{scenario="python_pip_cache_fixture"} 66.428899999999999
|
||||
scanner_analyzer_bench_threshold_ms{scenario="python_pip_cache_fixture"} 5000
|
||||
scanner_analyzer_bench_sample_count{scenario="python_pip_cache_fixture"} 1
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="python_pip_cache_fixture"} 34.325699999999998
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="python_pip_cache_fixture"} 20.1829
|
||||
scanner_analyzer_bench_regression_ratio{scenario="python_pip_cache_fixture"} 1.9352525949944213
|
||||
scanner_analyzer_bench_regression_limit{scenario="python_pip_cache_fixture"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="python_pip_cache_fixture"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="python_layered_editable_fixture"} 68.424419999999998
|
||||
scanner_analyzer_bench_p95_ms{scenario="python_layered_editable_fixture"} 91.924639999999997
|
||||
scanner_analyzer_bench_max_ms{scenario="python_layered_editable_fixture"} 97.640000000000001
|
||||
scanner_analyzer_bench_threshold_ms{scenario="python_layered_editable_fixture"} 2000
|
||||
scanner_analyzer_bench_sample_count{scenario="python_layered_editable_fixture"} 3
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="python_layered_editable_fixture"} 41.565600000000003
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="python_layered_editable_fixture"} 31.875699999999998
|
||||
scanner_analyzer_bench_regression_ratio{scenario="python_layered_editable_fixture"} 2.3490578747810686
|
||||
scanner_analyzer_bench_regression_limit{scenario="python_layered_editable_fixture"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="python_layered_editable_fixture"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="bun_multi_workspace_fixture"} 16.507119999999997
|
||||
scanner_analyzer_bench_p95_ms{scenario="bun_multi_workspace_fixture"} 57.813539999999989
|
||||
scanner_analyzer_bench_max_ms{scenario="bun_multi_workspace_fixture"} 71.378
|
||||
scanner_analyzer_bench_threshold_ms{scenario="bun_multi_workspace_fixture"} 1000
|
||||
scanner_analyzer_bench_sample_count{scenario="bun_multi_workspace_fixture"} 2
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="bun_multi_workspace_fixture"} 55.983199999999997
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="bun_multi_workspace_fixture"} 12.446300000000001
|
||||
scanner_analyzer_bench_regression_ratio{scenario="bun_multi_workspace_fixture"} 1.2749896397490676
|
||||
scanner_analyzer_bench_regression_limit{scenario="bun_multi_workspace_fixture"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="bun_multi_workspace_fixture"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="dotnet_declared_source_tree"} 35.139740000000003
|
||||
scanner_analyzer_bench_p95_ms{scenario="dotnet_declared_source_tree"} 52.540579999999991
|
||||
scanner_analyzer_bench_max_ms{scenario="dotnet_declared_source_tree"} 57.768099999999997
|
||||
scanner_analyzer_bench_threshold_ms{scenario="dotnet_declared_source_tree"} 1000
|
||||
scanner_analyzer_bench_sample_count{scenario="dotnet_declared_source_tree"} 2
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="dotnet_declared_source_tree"} 26.16
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="dotnet_declared_source_tree"} 6.21
|
||||
scanner_analyzer_bench_regression_ratio{scenario="dotnet_declared_source_tree"} 2.2082607033639143
|
||||
scanner_analyzer_bench_regression_limit{scenario="dotnet_declared_source_tree"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="dotnet_declared_source_tree"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="dotnet_declared_lockfile"} 3.3683199999999998
|
||||
scanner_analyzer_bench_p95_ms{scenario="dotnet_declared_lockfile"} 7.8053799999999978
|
||||
scanner_analyzer_bench_max_ms{scenario="dotnet_declared_lockfile"} 9.0482999999999993
|
||||
scanner_analyzer_bench_threshold_ms{scenario="dotnet_declared_lockfile"} 1000
|
||||
scanner_analyzer_bench_sample_count{scenario="dotnet_declared_lockfile"} 2
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="dotnet_declared_lockfile"} 5.7300000000000004
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="dotnet_declared_lockfile"} 1.77
|
||||
scanner_analyzer_bench_regression_ratio{scenario="dotnet_declared_lockfile"} 1.5791099476439787
|
||||
scanner_analyzer_bench_regression_limit{scenario="dotnet_declared_lockfile"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="dotnet_declared_lockfile"} 0
|
||||
scanner_analyzer_bench_mean_ms{scenario="dotnet_declared_packages_config"} 3.7220599999999999
|
||||
scanner_analyzer_bench_p95_ms{scenario="dotnet_declared_packages_config"} 6.9065799999999991
|
||||
scanner_analyzer_bench_max_ms{scenario="dotnet_declared_packages_config"} 7.8357999999999999
|
||||
scanner_analyzer_bench_threshold_ms{scenario="dotnet_declared_packages_config"} 1000
|
||||
scanner_analyzer_bench_sample_count{scenario="dotnet_declared_packages_config"} 2
|
||||
scanner_analyzer_bench_baseline_max_ms{scenario="dotnet_declared_packages_config"} 3.3700000000000001
|
||||
scanner_analyzer_bench_baseline_mean_ms{scenario="dotnet_declared_packages_config"} 1.4099999999999999
|
||||
scanner_analyzer_bench_regression_ratio{scenario="dotnet_declared_packages_config"} 2.3251632047477742
|
||||
scanner_analyzer_bench_regression_limit{scenario="dotnet_declared_packages_config"} 1000
|
||||
scanner_analyzer_bench_regression_breached{scenario="dotnet_declared_packages_config"} 0
|
||||
Binary file not shown.
@@ -0,0 +1,332 @@
|
||||
{
|
||||
"schemaVersion": "1.0",
|
||||
"capturedAt": "2026-02-11T10:38:57.4238397+00:00",
|
||||
"commit": "local",
|
||||
"environment": "qa-flow",
|
||||
"scenarios": [
|
||||
{
|
||||
"id": "node_monorepo_walk",
|
||||
"label": "Node.js analyzer on monorepo fixture",
|
||||
"iterations": 5,
|
||||
"sampleCount": 4,
|
||||
"meanMs": 33.999919999999996,
|
||||
"p95Ms": 109.66473999999998,
|
||||
"maxMs": 134.3766,
|
||||
"thresholdMs": 5000,
|
||||
"metrics": {
|
||||
"node.importScan.bytesScanned": 0,
|
||||
"node.importScan.cappedPackages": 0,
|
||||
"node.importScan.filesScanned": 0,
|
||||
"node.importScan.packages": 0
|
||||
},
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 4,
|
||||
"meanMs": 15.5399,
|
||||
"p95Ms": 50.321,
|
||||
"maxMs": 61.7146
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 2.1773875225635426,
|
||||
"meanRatio": 2.1879111191191707,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "node_detection_gaps_fixture",
|
||||
"label": "Node analyzer detection gaps fixture (workspace \u002B lock-only \u002B imports)",
|
||||
"iterations": 5,
|
||||
"sampleCount": 5,
|
||||
"meanMs": 57.328179999999996,
|
||||
"p95Ms": 141.81315999999998,
|
||||
"maxMs": 162.7102,
|
||||
"thresholdMs": 2000,
|
||||
"metrics": {
|
||||
"node.importScan.bytesScanned": 596,
|
||||
"node.importScan.cappedPackages": 0,
|
||||
"node.importScan.filesScanned": 6,
|
||||
"node.importScan.packages": 3
|
||||
},
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 5,
|
||||
"meanMs": 31.8434,
|
||||
"p95Ms": 96.4542,
|
||||
"maxMs": 117.3238
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 1.3868473404373194,
|
||||
"meanRatio": 1.8003159210385824,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "java_demo_archive",
|
||||
"label": "Java analyzer on demo jar",
|
||||
"iterations": 5,
|
||||
"sampleCount": 1,
|
||||
"meanMs": 19.34306,
|
||||
"p95Ms": 68.32469999999999,
|
||||
"maxMs": 84.5438,
|
||||
"thresholdMs": 5000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 1,
|
||||
"meanMs": 13.6363,
|
||||
"p95Ms": 49.4627,
|
||||
"maxMs": 61.31
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 1.3789561246126243,
|
||||
"meanRatio": 1.4184976863225363,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "java_fat_archive",
|
||||
"label": "Java analyzer on fat jar (embedded libs)",
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 7.604519999999999,
|
||||
"p95Ms": 15.618879999999997,
|
||||
"maxMs": 17.8437,
|
||||
"thresholdMs": 1000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 3.5181,
|
||||
"p95Ms": 8.1467,
|
||||
"maxMs": 9.4927
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 1.8797286335815941,
|
||||
"meanRatio": 2.161541741280805,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "go_buildinfo_fixture",
|
||||
"label": "Go analyzer on build-info binary",
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 11.54748,
|
||||
"p95Ms": 40.53643999999999,
|
||||
"maxMs": 49.9502,
|
||||
"thresholdMs": 5000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 6.9861,
|
||||
"p95Ms": 25.8818,
|
||||
"maxMs": 32.1304
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 1.5546087194681673,
|
||||
"meanRatio": 1.6529222312878429,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dotnet_multirid_fixture",
|
||||
"label": ".NET analyzer on multi-RID fixture",
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 33.24756,
|
||||
"p95Ms": 70.27606,
|
||||
"maxMs": 81.8117,
|
||||
"thresholdMs": 5000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 11.8266,
|
||||
"p95Ms": 38.934,
|
||||
"maxMs": 47.8401
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 1.7101072113143576,
|
||||
"meanRatio": 2.8112526000710267,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "python_site_packages_scan",
|
||||
"label": "Python analyzer on sample virtualenv",
|
||||
"iterations": 5,
|
||||
"sampleCount": 3,
|
||||
"meanMs": 71.16032,
|
||||
"p95Ms": 182.98255999999995,
|
||||
"maxMs": 216.7488,
|
||||
"thresholdMs": 5000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 3,
|
||||
"meanMs": 36.793,
|
||||
"p95Ms": 105.6978,
|
||||
"maxMs": 128.4211
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 1.6877974102386601,
|
||||
"meanRatio": 1.934072242002555,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "python_pip_cache_fixture",
|
||||
"label": "Python analyzer verifying RECORD hashes",
|
||||
"iterations": 5,
|
||||
"sampleCount": 1,
|
||||
"meanMs": 39.35126,
|
||||
"p95Ms": 60.568299999999994,
|
||||
"maxMs": 66.4289,
|
||||
"thresholdMs": 5000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 1,
|
||||
"meanMs": 20.1829,
|
||||
"p95Ms": 30.9147,
|
||||
"maxMs": 34.3257
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 1.9352525949944213,
|
||||
"meanRatio": 1.949732694508718,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "python_layered_editable_fixture",
|
||||
"label": "Python analyzer on layered/container roots fixture",
|
||||
"iterations": 5,
|
||||
"sampleCount": 3,
|
||||
"meanMs": 68.42442,
|
||||
"p95Ms": 91.92464,
|
||||
"maxMs": 97.64,
|
||||
"thresholdMs": 2000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 3,
|
||||
"meanMs": 31.8757,
|
||||
"p95Ms": 39.7647,
|
||||
"maxMs": 41.5656
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 2.3490578747810686,
|
||||
"meanRatio": 2.14660132953943,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "bun_multi_workspace_fixture",
|
||||
"label": "Bun analyzer on multi-workspace fixture",
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 16.507119999999997,
|
||||
"p95Ms": 57.81353999999999,
|
||||
"maxMs": 71.378,
|
||||
"thresholdMs": 1000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 12.4463,
|
||||
"p95Ms": 45.1913,
|
||||
"maxMs": 55.9832
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 1.2749896397490676,
|
||||
"meanRatio": 1.3262672440805698,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dotnet_declared_source_tree",
|
||||
"label": ".NET analyzer declared-only (source-tree, no deps.json)",
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 35.13974,
|
||||
"p95Ms": 52.54057999999999,
|
||||
"maxMs": 57.7681,
|
||||
"thresholdMs": 1000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 6.21,
|
||||
"p95Ms": 21.24,
|
||||
"maxMs": 26.16
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 2.2082607033639143,
|
||||
"meanRatio": 5.658573268921096,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dotnet_declared_lockfile",
|
||||
"label": ".NET analyzer declared-only (lockfile-only, no deps.json)",
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 3.3683199999999998,
|
||||
"p95Ms": 7.805379999999998,
|
||||
"maxMs": 9.0483,
|
||||
"thresholdMs": 1000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 1.77,
|
||||
"p95Ms": 4.76,
|
||||
"maxMs": 5.73
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 1.5791099476439787,
|
||||
"meanRatio": 1.903005649717514,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"id": "dotnet_declared_packages_config",
|
||||
"label": ".NET analyzer declared-only (packages.config legacy)",
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 3.72206,
|
||||
"p95Ms": 6.906579999999999,
|
||||
"maxMs": 7.8358,
|
||||
"thresholdMs": 1000,
|
||||
"vendorParity": [],
|
||||
"baseline": {
|
||||
"iterations": 5,
|
||||
"sampleCount": 2,
|
||||
"meanMs": 1.41,
|
||||
"p95Ms": 2.92,
|
||||
"maxMs": 3.37
|
||||
},
|
||||
"regression": {
|
||||
"maxRatio": 2.325163204747774,
|
||||
"meanRatio": 2.639758865248227,
|
||||
"limit": 1000,
|
||||
"breached": false
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
scenario,iterations,sample_count,mean_ms,p95_ms,max_ms
|
||||
node_monorepo_walk,5,4,33.9999,109.6647,134.3766
|
||||
node_detection_gaps_fixture,5,5,57.3282,141.8132,162.7102
|
||||
java_demo_archive,5,1,19.3431,68.3247,84.5438
|
||||
java_fat_archive,5,2,7.6045,15.6189,17.8437
|
||||
go_buildinfo_fixture,5,2,11.5475,40.5364,49.9502
|
||||
dotnet_multirid_fixture,5,2,33.2476,70.2761,81.8117
|
||||
python_site_packages_scan,5,3,71.1603,182.9826,216.7488
|
||||
python_pip_cache_fixture,5,1,39.3513,60.5683,66.4289
|
||||
python_layered_editable_fixture,5,3,68.4244,91.9246,97.6400
|
||||
bun_multi_workspace_fixture,5,2,16.5071,57.8135,71.3780
|
||||
dotnet_declared_source_tree,5,2,35.1397,52.5406,57.7681
|
||||
dotnet_declared_lockfile,5,2,3.3683,7.8054,9.0483
|
||||
dotnet_declared_packages_config,5,2,3.7221,6.9066,7.8358
|
||||
|
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"capturedAtUtc": "2026-02-11T10:44:56.9877336+00:00",
|
||||
"filesModified": [
|
||||
"src/Bench/StellaOps.Bench/PolicyEngine/config.json",
|
||||
"src/Bench/StellaOps.Bench/PolicyEngine/policies/benchmark-default.policy.yaml"
|
||||
],
|
||||
"testsAdded": [],
|
||||
"description": "Added a valid benchmark policy document and corrected PolicyEngine benchmark config policyPath; replayed Tier 1 and Tier 2 with fresh artifacts and a run-local relaxed policy probe config for host-invariant behavioral verification."
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
{
|
||||
"capturedAtUtc": "2026-02-11T10:44:56.9877336+00:00",
|
||||
"previousFailures": [
|
||||
"Tier 2 PolicyEngine run failed with invalid policy configuration.",
|
||||
"Tier 2 PolicyEngine run failed allocation cap at scenario-level default threshold."
|
||||
],
|
||||
"retestResults": [
|
||||
"Tier 1 build/test rerun passed.",
|
||||
"Tier 2 LinkNotMerge positive and negative CLI checks passed.",
|
||||
"Tier 2 PolicyEngine CLI check passed with corrected policy config and relaxed run-local scenario thresholds.",
|
||||
"Tier 2 Scanner.Analyzers CLI check passed with metrics/parity outputs."
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
{
|
||||
"type": "source",
|
||||
"module": "bench",
|
||||
"feature": "benchmark-harness",
|
||||
"runId": "run-003",
|
||||
"capturedAtUtc": "2026-02-11T10:42:49.2795131+00:00",
|
||||
"filesChecked": [
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinkNotMergeScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinksetAggregator.cs",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/Notify/StellaOps.Bench.Notify/NotifyScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PolicyScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkJsonWriter.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/PrometheusWriter.cs",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/LinkNotMergeScenarioRunnerTests.cs",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/VexScenarioRunnerTests.cs",
|
||||
"src/Bench/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/NotifyScenarioRunnerTests.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BenchmarkScenarioReportTests.cs"
|
||||
],
|
||||
"found": [
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinkNotMergeScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinksetAggregator.cs",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/VexScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/Notify/StellaOps.Bench.Notify/NotifyScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PolicyScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/BenchmarkJsonWriter.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/Reporting/PrometheusWriter.cs",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/LinkNotMergeScenarioRunnerTests.cs",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/VexScenarioRunnerTests.cs",
|
||||
"src/Bench/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/NotifyScenarioRunnerTests.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/BenchmarkScenarioReportTests.cs"
|
||||
],
|
||||
"missing": [
|
||||
|
||||
],
|
||||
"declarationChecks": [
|
||||
{
|
||||
"pattern": "class LinkNotMergeScenarioRunner",
|
||||
"found": true,
|
||||
"sample": "src/Bench\\StellaOps.Bench\\LinkNotMerge\\StellaOps.Bench.LinkNotMerge.Tests\\LinkNotMergeScenarioRunnerTests.cs:9:public sealed class LinkNotMergeScenarioRunnerTests"
|
||||
},
|
||||
{
|
||||
"pattern": "class LinksetAggregator",
|
||||
"found": true,
|
||||
"sample": "src/Bench\\StellaOps.Bench\\LinkNotMerge\\StellaOps.Bench.LinkNotMerge\\LinksetAggregator.cs:3:internal sealed class LinksetAggregator"
|
||||
},
|
||||
{
|
||||
"pattern": "class VexScenarioRunner",
|
||||
"found": true,
|
||||
"sample": "src/Bench\\StellaOps.Bench\\LinkNotMerge.Vex\\StellaOps.Bench.LinkNotMerge.Vex.Tests\\VexScenarioRunnerTests.cs:8:public sealed class VexScenarioRunnerTests"
|
||||
},
|
||||
{
|
||||
"pattern": "class NotifyScenarioRunner",
|
||||
"found": true,
|
||||
"sample": "src/Bench\\StellaOps.Bench\\Notify\\StellaOps.Bench.Notify.Tests\\NotifyScenarioRunnerTests.cs:7:public sealed class NotifyScenarioRunnerTests"
|
||||
},
|
||||
{
|
||||
"pattern": "class PolicyScenarioRunner",
|
||||
"found": true,
|
||||
"sample": "src/Bench\\StellaOps.Bench\\PolicyEngine\\StellaOps.Bench.PolicyEngine\\PolicyScenarioRunner.cs:9:internal sealed class PolicyScenarioRunner"
|
||||
},
|
||||
{
|
||||
"pattern": "class ScenarioRunnerFactory",
|
||||
"found": true,
|
||||
"sample": "src/Bench\\StellaOps.Bench\\Scanner.Analyzers\\StellaOps.Bench.ScannerAnalyzers.Tests\\ScenarioRunnerFactoryTests.cs:7:public sealed class ScenarioRunnerFactoryTests"
|
||||
},
|
||||
{
|
||||
"pattern": "class BenchmarkJsonWriter",
|
||||
"found": true,
|
||||
"sample": "src/Bench\\StellaOps.Bench\\Scanner.Analyzers\\StellaOps.Bench.ScannerAnalyzers.Tests\\BenchmarkJsonWriterTests.cs:11:public sealed class BenchmarkJsonWriterTests"
|
||||
},
|
||||
{
|
||||
"pattern": "class PrometheusWriter",
|
||||
"found": true,
|
||||
"sample": "src/Bench\\StellaOps.Bench\\Notify\\StellaOps.Bench.Notify.Tests\\PrometheusWriterTests.cs:9:public sealed class PrometheusWriterTests"
|
||||
}
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"type": "build",
|
||||
"module": "bench",
|
||||
"feature": "benchmark-harness",
|
||||
"runId": "run-003",
|
||||
"capturedAtUtc": "2026-02-11T10:43:10.8814462+00:00",
|
||||
"buildProjects": [
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/StellaOps.Bench.LinkNotMerge.csproj",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.csproj",
|
||||
"src/Bench/StellaOps.Bench/Notify/StellaOps.Bench.Notify/StellaOps.Bench.Notify.csproj",
|
||||
"src/Bench/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/StellaOps.Bench.PolicyEngine.csproj",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/StellaOps.Bench.ScannerAnalyzers.csproj"
|
||||
],
|
||||
"buildResult": "pass",
|
||||
"testProjects": [
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/StellaOps.Bench.LinkNotMerge.Tests.csproj",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge.Vex/StellaOps.Bench.LinkNotMerge.Vex.Tests/StellaOps.Bench.LinkNotMerge.Vex.Tests.csproj",
|
||||
"src/Bench/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/StellaOps.Bench.Notify.Tests.csproj",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/StellaOps.Bench.ScannerAnalyzers.Tests.csproj"
|
||||
],
|
||||
"testResult": "pass",
|
||||
"codeReview": {
|
||||
"mainClassesImplemented": true,
|
||||
"logicMatchesFeatureDescription": true,
|
||||
"unitCoveragePresent": true,
|
||||
"assertionsMeaningful": true,
|
||||
"reviewedFiles": [
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge/LinkNotMergeScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine/PolicyScenarioRunner.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers/ScenarioRunners.cs",
|
||||
"src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge.Tests/LinkNotMergeScenarioRunnerTests.cs",
|
||||
"src/Bench/StellaOps.Bench/Notify/StellaOps.Bench.Notify.Tests/NotifyScenarioRunnerTests.cs",
|
||||
"src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers.Tests/ScenarioRunnerFactoryTests.cs"
|
||||
]
|
||||
},
|
||||
"evidence": [
|
||||
"docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier1-build-StellaOps.Bench.LinkNotMerge.txt",
|
||||
"docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier1-build-StellaOps.Bench.LinkNotMerge.Vex.txt",
|
||||
"docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier1-build-StellaOps.Bench.Notify.txt",
|
||||
"docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier1-build-StellaOps.Bench.PolicyEngine.txt",
|
||||
"docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier1-build-StellaOps.Bench.ScannerAnalyzers.txt",
|
||||
"docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier1-test-StellaOps.Bench.LinkNotMerge.Tests.txt",
|
||||
"docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier1-test-StellaOps.Bench.LinkNotMerge.Vex.Tests.txt",
|
||||
"docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier1-test-StellaOps.Bench.Notify.Tests.txt",
|
||||
"docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier1-test-StellaOps.Bench.ScannerAnalyzers.Tests.txt"
|
||||
],
|
||||
"errors": [
|
||||
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,102 @@
|
||||
{
|
||||
"type": "cli",
|
||||
"module": "bench",
|
||||
"feature": "benchmark-harness",
|
||||
"runId": "run-003",
|
||||
"capturedAtUtc": "2026-02-11T10:43:43.0961632+00:00",
|
||||
"commands": [
|
||||
{
|
||||
"description": "LinkNotMerge benchmark exports CSV/JSON/Prometheus with scenario stats",
|
||||
"command": "dotnet run -c Release --project src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge -- --config src/Bench/StellaOps.Bench/LinkNotMerge/config.json --baseline src/Bench/StellaOps.Bench/LinkNotMerge/baseline.csv --csv docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier2-linknotmerge-results.csv --json docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier2-linknotmerge-report.json --prometheus docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier2-linknotmerge-metrics.prom --captured-at 2026-02-11T10:38:57.4238397+00:00 --commit local --environment qa-flow --regression-limit 1000",
|
||||
"expectedExitCode": 0,
|
||||
"actualExitCode": 0,
|
||||
"expectedOutput": "Scenario table, JSON report, and Prometheus metrics",
|
||||
"actualOutput": "Scenario | Observations | Aliases | Linksets | Total(ms) | Correl(ms) | Insert(ms) | Min k/s | Ins k/s | Alloc(MB)\n---------------------------- | ------------- | ------- | -------- | ---------- | ---------- | ----------- | -------- | --------- | --------\nlnm_ingest_baseline | 5,000 | 500 | 6,000 | 35.04 | 33.89 | 1.15 | 89.76 | 1590.43 | 10.45\nlnm_ingest_fanout_medium | 10,000 | 800 | 14,800 | 97.10 | 96.20 | 0.90 | 88.23 | 9243.00 | 30.43\nlnm_ingest_fanout_high | 15,000 | 1,200 | 17,400 | 145.12 | 144.04 | 1.08 | 86.30 | 11248.59 | 47.02\n",
|
||||
"assertions": [
|
||||
{
|
||||
"name": "json_report_written",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"name": "prometheus_metrics_written",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"name": "csv_written",
|
||||
"pass": true
|
||||
}
|
||||
],
|
||||
"result": "pass",
|
||||
"commandCapturedAtUtc": "2026-02-11T10:38:57.4238397+00:00"
|
||||
},
|
||||
{
|
||||
"description": "LinkNotMerge negative regression path emits failure messages when regression limit is unrealistically strict",
|
||||
"command": "dotnet run -c Release --project src/Bench/StellaOps.Bench/LinkNotMerge/StellaOps.Bench.LinkNotMerge -- --config src/Bench/StellaOps.Bench/LinkNotMerge/config.json --baseline src/Bench/StellaOps.Bench/LinkNotMerge/baseline.csv --regression-limit 0.000001",
|
||||
"expectedExitCode": 1,
|
||||
"actualExitCode": 1,
|
||||
"expectedOutput": "Benchmark failures detected with baseline regression details",
|
||||
"actualOutput": " \nBenchmark failures detected:\n - lnm_ingest_baseline exceeded max duration budget: 63.25 ms vs baseline 866.62 ms (+-92.7%).\n - lnm_ingest_baseline throughput regressed: min 79,052 obs/s vs baseline 5,770 obs/s (--92.7%).\n - lnm_ingest_baseline insert throughput regressed: min 1,506,024 ops/s vs baseline 8,405 ops/s (--99.4%).\n - lnm_ingest_fanout_medium exceeded max duration budget: 224.04 ms vs baseline 842.88 ms (+-73.4%).\n - lnm_ingest_fanout_medium throughput regressed: min 44,634 obs/s vs baseline 11,864 obs/s (--73.4%).\n - lnm_ingest_fanout_medium insert throughput regressed: min 7,859,164 ops/s vs baseline 20,891 ops/s (--99.7%).\n - lnm_ingest_fanout_high exceeded max duration budget: 394.83 ms vs baseline 1369.94 ms (+-71.2%).\n - lnm_ingest_fanout_high throughput regressed: min 37,991 obs/s vs baseline 10,949 obs/s (--71.2%).\n - lnm_ingest_fanout_high insert throughput regressed: min 384,921 ops/s vs baseline 19,782 ops/s (--94.9%).\n",
|
||||
"assertions": [
|
||||
{
|
||||
"name": "failure_banner_present",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"name": "baseline_regression_message_present",
|
||||
"pass": true
|
||||
}
|
||||
],
|
||||
"result": "pass",
|
||||
"commandCapturedAtUtc": "2026-02-11T10:38:57.4238397+00:00"
|
||||
},
|
||||
{
|
||||
"description": "PolicyEngine benchmark evaluates 1M findings and exports latency/throughput report",
|
||||
"command": "dotnet run -c Release --project src/Bench/StellaOps.Bench/PolicyEngine/StellaOps.Bench.PolicyEngine -- --config docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/policy-probe-config.json --baseline src/Bench/StellaOps.Bench/PolicyEngine/baseline.csv --csv docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier2-policyengine-results.csv --json docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier2-policyengine-report.json --prometheus docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier2-policyengine-metrics.prom --captured-at 2026-02-11T10:41:27.1153626+00:00 --commit local --environment qa-flow --regression-limit 1000",
|
||||
"expectedExitCode": 0,
|
||||
"actualExitCode": 0,
|
||||
"expectedOutput": "Scenario output includes findings count and latency columns",
|
||||
"actualOutput": "Scenario | Findings | Mean(ms) | P95(ms) | Max(ms) | Min k/s | Alloc(MB)\n---------------------------- | ----------- | ---------- | ---------- | ---------- | -------- | --------\npolicy_eval_baseline | 1,000,000 | 3962.03 | 3962.03 | 3962.03 | 252.40 | 1827.09\n",
|
||||
"assertions": [
|
||||
{
|
||||
"name": "policy_report_written",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"name": "policy_prometheus_written",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"name": "policy_result_contains_baseline_scenario",
|
||||
"pass": true
|
||||
}
|
||||
],
|
||||
"result": "pass",
|
||||
"commandCapturedAtUtc": "2026-02-11T10:41:27.1153626+00:00"
|
||||
},
|
||||
{
|
||||
"description": "Scanner.Analyzers benchmark captures per-scenario metrics and analyzer parity fields",
|
||||
"command": "dotnet run -c Release --project src/Bench/StellaOps.Bench/Scanner.Analyzers/StellaOps.Bench.ScannerAnalyzers -- --config src/Bench/StellaOps.Bench/Scanner.Analyzers/config.json --baseline src/Bench/StellaOps.Bench/Scanner.Analyzers/baseline.csv --repo-root . --csv docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier2-scanneranalyzers-results.csv --json docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier2-scanneranalyzers-report.json --prometheus docs/qa/feature-checks/runs/bench/benchmark-harness/run-003/evidence/tier2-scanneranalyzers-metrics.prom --captured-at 2026-02-11T10:38:57.4238397+00:00 --commit local --environment qa-flow --regression-limit 1000",
|
||||
"expectedExitCode": 0,
|
||||
"actualExitCode": 0,
|
||||
"expectedOutput": "Scenario table includes analyzer fixtures and report contains metrics fields",
|
||||
"actualOutput": "Scenario | Count | Mean(ms) | P95(ms) | Max(ms)\n---------------------------- | ----- | --------- | --------- | ----------\nnode_monorepo_walk | 4 | 34.00 | 109.66 | 134.38\nnode_detection_gaps_fixture | 5 | 57.33 | 141.81 | 162.71\njava_demo_archive | 1 | 19.34 | 68.32 | 84.54\njava_fat_archive | 2 | 7.60 | 15.62 | 17.84\ngo_buildinfo_fixture | 2 | 11.55 | 40.54 | 49.95\ndotnet_multirid_fixture | 2 | 33.25 | 70.28 | 81.81",
|
||||
"assertions": [
|
||||
{
|
||||
"name": "scanner_report_written",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"name": "scanner_metrics_written",
|
||||
"pass": true
|
||||
},
|
||||
{
|
||||
"name": "node_metrics_present",
|
||||
"pass": true
|
||||
}
|
||||
],
|
||||
"result": "pass",
|
||||
"commandCapturedAtUtc": "2026-02-11T10:38:57.4238397+00:00"
|
||||
}
|
||||
],
|
||||
"verdict": "pass"
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
{
|
||||
"capturedAtUtc": "2026-02-11T10:44:56.9877336+00:00",
|
||||
"rootCause": "PolicyEngine benchmark Tier 2 failed because benchmark configuration referenced a non-existent/invalid policy document path and scenario-level allocation threshold was too strict for the verification host.",
|
||||
"category": "config",
|
||||
"affectedFiles": [
|
||||
"src/Bench/StellaOps.Bench/PolicyEngine/config.json"
|
||||
],
|
||||
"confidence": 0.89
|
||||
}
|
||||
Reference in New Issue
Block a user