sprints enhancements

This commit is contained in:
StellaOps Bot
2025-12-25 19:52:30 +02:00
parent ef6ac36323
commit b8b2d83f4a
138 changed files with 25133 additions and 594 deletions

View File

@@ -13,6 +13,7 @@ using StellaOps.Policy.Engine.ExceptionCache;
using StellaOps.Policy.Engine.Gates;
using StellaOps.Policy.Engine.Options;
using StellaOps.Policy.Engine.ReachabilityFacts;
using StellaOps.Policy.Engine.Scoring.EvidenceWeightedScore;
using StellaOps.Policy.Engine.Services;
using StellaOps.Policy.Engine.Vex;
using StellaOps.Policy.Engine.WhatIfSimulation;
@@ -292,6 +293,10 @@ public static class PolicyEngineServiceCollectionExtensions
/// <summary>
/// Adds all Policy Engine services with default configuration.
/// </summary>
/// <remarks>
/// Includes core services, event pipeline, worker, explainer, and Evidence-Weighted Score services.
/// EWS services are registered but only activate when <see cref="PolicyEvidenceWeightedScoreOptions.Enabled"/> is true.
/// </remarks>
public static IServiceCollection AddPolicyEngine(this IServiceCollection services)
{
services.AddPolicyEngineCore();
@@ -299,6 +304,10 @@ public static class PolicyEngineServiceCollectionExtensions
services.AddPolicyEngineWorker();
services.AddPolicyEngineExplainer();
// Evidence-Weighted Score services (Sprint 8200.0012.0003)
// Always registered; activation controlled by PolicyEvidenceWeightedScoreOptions.Enabled
services.AddEvidenceWeightedScore();
return services;
}
@@ -313,6 +322,32 @@ public static class PolicyEngineServiceCollectionExtensions
return services.AddPolicyEngine();
}
/// <summary>
/// Adds all Policy Engine services with conditional EWS based on configuration.
/// </summary>
/// <remarks>
/// Unlike <see cref="AddPolicyEngine()"/>, this method reads configuration at registration
/// time and only registers EWS services if <see cref="PolicyEvidenceWeightedScoreOptions.Enabled"/>
/// is true. Use this for zero-overhead deployments where EWS is disabled.
/// </remarks>
/// <param name="services">Service collection.</param>
/// <param name="configuration">Configuration root for reading options.</param>
/// <returns>The service collection for chaining.</returns>
public static IServiceCollection AddPolicyEngine(
this IServiceCollection services,
Microsoft.Extensions.Configuration.IConfiguration configuration)
{
services.AddPolicyEngineCore();
services.AddPolicyEngineEventPipeline();
services.AddPolicyEngineWorker();
services.AddPolicyEngineExplainer();
// Conditional EWS registration based on configuration
services.AddEvidenceWeightedScoreIfEnabled(configuration);
return services;
}
/// <summary>
/// Adds exception integration services for automatic exception loading during policy evaluation.
/// Requires IExceptionRepository to be registered.

View File

@@ -43,6 +43,18 @@ internal sealed class PolicyEvaluator
}
public PolicyEvaluationResult Evaluate(PolicyEvaluationRequest request)
{
return Evaluate(request, injectedScore: null);
}
/// <summary>
/// Evaluate a policy with an optional pre-computed EWS score.
/// When injectedScore is provided, it will be used instead of computing EWS from context.
/// This is primarily for testing score-based policy rules.
/// </summary>
public PolicyEvaluationResult Evaluate(
PolicyEvaluationRequest request,
global::StellaOps.Signals.EvidenceWeightedScore.EvidenceWeightedScoreResult? injectedScore)
{
if (request is null)
{
@@ -54,8 +66,8 @@ internal sealed class PolicyEvaluator
throw new ArgumentNullException(nameof(request.Document));
}
// Pre-compute EWS so it's available during rule evaluation for score-based rules
var precomputedScore = PrecomputeEvidenceWeightedScore(request.Context);
// Use injected score if provided, otherwise compute from context
var precomputedScore = injectedScore ?? PrecomputeEvidenceWeightedScore(request.Context);
var evaluator = new PolicyExpressionEvaluator(request.Context, precomputedScore);
var orderedRules = request.Document.Rules

View File

@@ -282,9 +282,34 @@ internal sealed class PolicyExpressionEvaluator
{
var leftValue = Evaluate(left, scope).Raw;
var rightValue = Evaluate(right, scope).Raw;
// For ScoreScope, use the numeric value for comparison
if (leftValue is ScoreScope leftScope)
{
leftValue = leftScope.ScoreValue;
}
if (rightValue is ScoreScope rightScope)
{
rightValue = rightScope.ScoreValue;
}
// Normalize numeric types for comparison (decimal vs int, etc.)
if (IsNumeric(leftValue) && IsNumeric(rightValue))
{
var leftDecimal = Convert.ToDecimal(leftValue, CultureInfo.InvariantCulture);
var rightDecimal = Convert.ToDecimal(rightValue, CultureInfo.InvariantCulture);
return new EvaluationValue(comparer(leftDecimal, rightDecimal));
}
return new EvaluationValue(comparer(leftValue, rightValue));
}
private static bool IsNumeric(object? value)
{
return value is decimal or double or float or int or long or short or byte;
}
private EvaluationValue CompareNumeric(PolicyExpression left, PolicyExpression right, EvaluationScope scope, Func<decimal, decimal, bool> comparer)
{
var leftValue = Evaluate(left, scope);
@@ -314,6 +339,13 @@ internal sealed class PolicyExpressionEvaluator
return true;
}
// Support direct score comparisons (score >= 70)
if (value.Raw is ScoreScope scoreScope)
{
number = scoreScope.ScoreValue;
return true;
}
number = 0m;
return false;
}
@@ -384,6 +416,7 @@ internal sealed class PolicyExpressionEvaluator
int i => i,
long l => l,
string s when decimal.TryParse(s, NumberStyles.Any, CultureInfo.InvariantCulture, out var value) => value,
ScoreScope scoreScope => scoreScope.ScoreValue,
_ => null,
};
}
@@ -968,6 +1001,11 @@ internal sealed class PolicyExpressionEvaluator
this.score = score;
}
/// <summary>
/// Gets the numeric score value for direct comparison (e.g., score >= 80).
/// </summary>
public decimal ScoreValue => score.Score;
public EvaluationValue Get(string member) => member.ToLowerInvariant() switch
{
// Core score value (allows direct comparison: score >= 80)

View File

@@ -25,6 +25,7 @@ public static class EvidenceWeightedScoreServiceCollectionExtensions
/// - <see cref="IScoreEnrichmentCache"/> for caching (when enabled)
/// - <see cref="IDualEmitVerdictEnricher"/> for dual-emit mode
/// - <see cref="IMigrationTelemetryService"/> for migration metrics
/// - <see cref="IEwsTelemetryService"/> for calculation/cache telemetry
/// - <see cref="ConfidenceToEwsAdapter"/> for legacy score translation
/// </remarks>
/// <param name="services">Service collection.</param>
@@ -50,6 +51,9 @@ public static class EvidenceWeightedScoreServiceCollectionExtensions
// Migration telemetry
services.TryAddSingleton<IMigrationTelemetryService, MigrationTelemetryService>();
// EWS telemetry (calculation duration, cache stats)
services.TryAddSingleton<IEwsTelemetryService, EwsTelemetryService>();
// Confidence adapter for legacy comparison
services.TryAddSingleton<ConfidenceToEwsAdapter>();

View File

@@ -0,0 +1,375 @@
// SPDX-License-Identifier: AGPL-3.0-or-later
// Copyright © 2025 StellaOps
// Sprint: SPRINT_8200_0012_0003_policy_engine_integration
// Task: PINT-8200-039 - Add telemetry: score calculation duration, cache hit rate
using System.Diagnostics;
using System.Diagnostics.Metrics;
using Microsoft.Extensions.Options;
namespace StellaOps.Policy.Engine.Scoring.EvidenceWeightedScore;
/// <summary>
/// Telemetry service for Evidence-Weighted Score metrics.
/// </summary>
/// <remarks>
/// Exposes the following metrics:
/// - stellaops.policy.ews.calculations_total: Total calculations performed
/// - stellaops.policy.ews.calculation_duration_ms: Calculation duration histogram
/// - stellaops.policy.ews.cache_hits_total: Cache hits
/// - stellaops.policy.ews.cache_misses_total: Cache misses
/// - stellaops.policy.ews.cache_hit_rate: Current cache hit rate (gauge)
/// - stellaops.policy.ews.scores_by_bucket: Score distribution by bucket
/// - stellaops.policy.ews.enabled: Whether EWS is enabled (gauge)
/// </remarks>
public interface IEwsTelemetryService
{
/// <summary>
/// Records a successful score calculation.
/// </summary>
void RecordCalculation(string bucket, TimeSpan duration, bool fromCache);
/// <summary>
/// Records a failed calculation.
/// </summary>
void RecordFailure(string reason);
/// <summary>
/// Records a skipped calculation (feature disabled).
/// </summary>
void RecordSkipped();
/// <summary>
/// Updates cache statistics.
/// </summary>
void UpdateCacheStats(long hits, long misses, int count);
/// <summary>
/// Gets current telemetry snapshot.
/// </summary>
EwsTelemetrySnapshot GetSnapshot();
}
/// <summary>
/// Snapshot of current EWS telemetry state.
/// </summary>
public sealed record EwsTelemetrySnapshot
{
public required long TotalCalculations { get; init; }
public required long CacheHits { get; init; }
public required long CacheMisses { get; init; }
public required long Failures { get; init; }
public required long Skipped { get; init; }
public required double AverageCalculationDurationMs { get; init; }
public required double P95CalculationDurationMs { get; init; }
public required double CacheHitRate { get; init; }
public required int CurrentCacheSize { get; init; }
public required IReadOnlyDictionary<string, long> ScoresByBucket { get; init; }
public required bool IsEnabled { get; init; }
public required DateTimeOffset SnapshotTime { get; init; }
}
/// <summary>
/// Implementation of EWS telemetry using System.Diagnostics.Metrics.
/// </summary>
public sealed class EwsTelemetryService : IEwsTelemetryService
{
private static readonly Meter s_meter = new("StellaOps.Policy.EvidenceWeightedScore", "1.0.0");
// Counters
private readonly Counter<long> _calculationsTotal;
private readonly Counter<long> _cacheHitsTotal;
private readonly Counter<long> _cacheMissesTotal;
private readonly Counter<long> _failuresTotal;
private readonly Counter<long> _skippedTotal;
private readonly Counter<long> _scoresByBucket;
// Histograms
private readonly Histogram<double> _calculationDuration;
// Gauges (observable)
private readonly ObservableGauge<double> _cacheHitRate;
private readonly ObservableGauge<int> _cacheSize;
private readonly ObservableGauge<int> _enabledGauge;
// Internal state for observable gauges
private long _totalHits;
private long _totalMisses;
private int _cacheCount;
// For aggregated statistics
private readonly object _lock = new();
private long _totalCalculations;
private long _failures;
private long _skipped;
private readonly Dictionary<string, long> _bucketCounts = new(StringComparer.OrdinalIgnoreCase);
private readonly List<double> _recentDurations = new(1000);
private int _durationIndex;
private const int MaxRecentDurations = 1000;
private readonly IOptionsMonitor<PolicyEvidenceWeightedScoreOptions> _options;
public EwsTelemetryService(IOptionsMonitor<PolicyEvidenceWeightedScoreOptions> options)
{
_options = options ?? throw new ArgumentNullException(nameof(options));
// Initialize counters
_calculationsTotal = s_meter.CreateCounter<long>(
"stellaops.policy.ews.calculations_total",
unit: "{calculations}",
description: "Total number of EWS calculations performed");
_cacheHitsTotal = s_meter.CreateCounter<long>(
"stellaops.policy.ews.cache_hits_total",
unit: "{hits}",
description: "Total number of EWS cache hits");
_cacheMissesTotal = s_meter.CreateCounter<long>(
"stellaops.policy.ews.cache_misses_total",
unit: "{misses}",
description: "Total number of EWS cache misses");
_failuresTotal = s_meter.CreateCounter<long>(
"stellaops.policy.ews.failures_total",
unit: "{failures}",
description: "Total number of EWS calculation failures");
_skippedTotal = s_meter.CreateCounter<long>(
"stellaops.policy.ews.skipped_total",
unit: "{skipped}",
description: "Total number of skipped EWS calculations (feature disabled)");
_scoresByBucket = s_meter.CreateCounter<long>(
"stellaops.policy.ews.scores_by_bucket",
unit: "{scores}",
description: "Score distribution by bucket");
// Initialize histogram
_calculationDuration = s_meter.CreateHistogram<double>(
"stellaops.policy.ews.calculation_duration_ms",
unit: "ms",
description: "EWS calculation duration in milliseconds");
// Initialize observable gauges
_cacheHitRate = s_meter.CreateObservableGauge(
"stellaops.policy.ews.cache_hit_rate",
() => GetCacheHitRate(),
unit: "{ratio}",
description: "Current EWS cache hit rate (0-1)");
_cacheSize = s_meter.CreateObservableGauge(
"stellaops.policy.ews.cache_size",
() => _cacheCount,
unit: "{entries}",
description: "Current EWS cache size");
_enabledGauge = s_meter.CreateObservableGauge(
"stellaops.policy.ews.enabled",
() => _options.CurrentValue.Enabled ? 1 : 0,
unit: "{boolean}",
description: "Whether EWS is currently enabled (1=enabled, 0=disabled)");
}
/// <inheritdoc />
public void RecordCalculation(string bucket, TimeSpan duration, bool fromCache)
{
var durationMs = duration.TotalMilliseconds;
// Update counters
_calculationsTotal.Add(1);
_calculationDuration.Record(durationMs);
_scoresByBucket.Add(1, new KeyValuePair<string, object?>("bucket", bucket));
if (fromCache)
{
_cacheHitsTotal.Add(1);
Interlocked.Increment(ref _totalHits);
}
else
{
_cacheMissesTotal.Add(1);
Interlocked.Increment(ref _totalMisses);
}
// Update internal state for snapshots
lock (_lock)
{
_totalCalculations++;
if (!_bucketCounts.TryGetValue(bucket, out var count))
{
_bucketCounts[bucket] = 1;
}
else
{
_bucketCounts[bucket] = count + 1;
}
// Circular buffer for recent durations
if (_recentDurations.Count < MaxRecentDurations)
{
_recentDurations.Add(durationMs);
}
else
{
_recentDurations[_durationIndex] = durationMs;
_durationIndex = (_durationIndex + 1) % MaxRecentDurations;
}
}
}
/// <inheritdoc />
public void RecordFailure(string reason)
{
_failuresTotal.Add(1, new KeyValuePair<string, object?>("reason", reason));
lock (_lock)
{
_failures++;
}
}
/// <inheritdoc />
public void RecordSkipped()
{
_skippedTotal.Add(1);
lock (_lock)
{
_skipped++;
}
}
/// <inheritdoc />
public void UpdateCacheStats(long hits, long misses, int count)
{
Interlocked.Exchange(ref _totalHits, hits);
Interlocked.Exchange(ref _totalMisses, misses);
Interlocked.Exchange(ref _cacheCount, count);
}
/// <inheritdoc />
public EwsTelemetrySnapshot GetSnapshot()
{
lock (_lock)
{
var (avgDuration, p95Duration) = CalculateDurationStats();
return new EwsTelemetrySnapshot
{
TotalCalculations = _totalCalculations,
CacheHits = Interlocked.Read(ref _totalHits),
CacheMisses = Interlocked.Read(ref _totalMisses),
Failures = _failures,
Skipped = _skipped,
AverageCalculationDurationMs = avgDuration,
P95CalculationDurationMs = p95Duration,
CacheHitRate = GetCacheHitRate(),
CurrentCacheSize = _cacheCount,
ScoresByBucket = new Dictionary<string, long>(_bucketCounts),
IsEnabled = _options.CurrentValue.Enabled,
SnapshotTime = DateTimeOffset.UtcNow
};
}
}
private double GetCacheHitRate()
{
var hits = Interlocked.Read(ref _totalHits);
var misses = Interlocked.Read(ref _totalMisses);
var total = hits + misses;
return total == 0 ? 0.0 : (double)hits / total;
}
private (double Average, double P95) CalculateDurationStats()
{
if (_recentDurations.Count == 0)
{
return (0.0, 0.0);
}
var sorted = _recentDurations.ToArray();
Array.Sort(sorted);
var average = sorted.Average();
var p95Index = (int)(sorted.Length * 0.95);
var p95 = sorted[Math.Min(p95Index, sorted.Length - 1)];
return (average, p95);
}
}
/// <summary>
/// Extension methods for EWS telemetry reporting.
/// </summary>
public static class EwsTelemetryExtensions
{
/// <summary>
/// Formats the telemetry snapshot as a summary report.
/// </summary>
public static string ToReport(this EwsTelemetrySnapshot snapshot)
{
var bucketLines = snapshot.ScoresByBucket.Count > 0
? string.Join("\n", snapshot.ScoresByBucket.Select(kv => $" - {kv.Key}: {kv.Value}"))
: " (none)";
return $"""
EWS Telemetry Report
====================
Generated: {snapshot.SnapshotTime:O}
Enabled: {snapshot.IsEnabled}
Calculations:
Total: {snapshot.TotalCalculations}
Failures: {snapshot.Failures}
Skipped: {snapshot.Skipped}
Performance:
Avg Duration: {snapshot.AverageCalculationDurationMs:F2}ms
P95 Duration: {snapshot.P95CalculationDurationMs:F2}ms
Cache:
Size: {snapshot.CurrentCacheSize}
Hits: {snapshot.CacheHits}
Misses: {snapshot.CacheMisses}
Hit Rate: {snapshot.CacheHitRate:P1}
Scores by Bucket:
{bucketLines}
""";
}
/// <summary>
/// Formats the telemetry snapshot as a single-line summary.
/// </summary>
public static string ToSummaryLine(this EwsTelemetrySnapshot snapshot)
{
return $"EWS: {snapshot.TotalCalculations} calcs, " +
$"{snapshot.Failures} failures, " +
$"avg={snapshot.AverageCalculationDurationMs:F1}ms, " +
$"p95={snapshot.P95CalculationDurationMs:F1}ms, " +
$"cache={snapshot.CacheHitRate:P0} hit rate";
}
/// <summary>
/// Gets Prometheus-compatible metric lines.
/// </summary>
public static IEnumerable<string> ToPrometheusMetrics(this EwsTelemetrySnapshot snapshot)
{
yield return $"stellaops_policy_ews_enabled {(snapshot.IsEnabled ? 1 : 0)}";
yield return $"stellaops_policy_ews_calculations_total {snapshot.TotalCalculations}";
yield return $"stellaops_policy_ews_failures_total {snapshot.Failures}";
yield return $"stellaops_policy_ews_skipped_total {snapshot.Skipped}";
yield return $"stellaops_policy_ews_cache_hits_total {snapshot.CacheHits}";
yield return $"stellaops_policy_ews_cache_misses_total {snapshot.CacheMisses}";
yield return $"stellaops_policy_ews_cache_size {snapshot.CurrentCacheSize}";
yield return $"stellaops_policy_ews_cache_hit_rate {snapshot.CacheHitRate:F4}";
yield return $"stellaops_policy_ews_calculation_duration_avg_ms {snapshot.AverageCalculationDurationMs:F2}";
yield return $"stellaops_policy_ews_calculation_duration_p95_ms {snapshot.P95CalculationDurationMs:F2}";
foreach (var (bucket, count) in snapshot.ScoresByBucket)
{
yield return $"stellaops_policy_ews_scores_by_bucket{{bucket=\"{bucket}\"}} {count}";
}
}
}

View File

@@ -25,6 +25,18 @@ internal sealed partial class PolicyEvaluationService
}
internal Evaluation.PolicyEvaluationResult Evaluate(PolicyIrDocument document, Evaluation.PolicyEvaluationContext context)
{
return Evaluate(document, context, evidenceWeightedScore: null);
}
/// <summary>
/// Evaluate a policy with an optional pre-computed EWS score.
/// This overload is primarily for testing score-based policy rules.
/// </summary>
internal Evaluation.PolicyEvaluationResult Evaluate(
PolicyIrDocument document,
Evaluation.PolicyEvaluationContext context,
global::StellaOps.Signals.EvidenceWeightedScore.EvidenceWeightedScoreResult? evidenceWeightedScore)
{
if (document is null)
{
@@ -37,7 +49,7 @@ internal sealed partial class PolicyEvaluationService
}
var request = new Evaluation.PolicyEvaluationRequest(document, context);
return _evaluator.Evaluate(request);
return _evaluator.Evaluate(request, evidenceWeightedScore);
}
// PathScopeSimulationService partial class relies on _pathMetrics.