git.stella-ops.org/src/Concelier/StellaOps.Concelier.WebService/Services/AdvisoryAiTelemetry.cs

using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.Linq;
using Microsoft.Extensions.Logging;
using StellaOps.Concelier.WebService.Diagnostics;

namespace StellaOps.Concelier.WebService.Services;

internal interface IAdvisoryAiTelemetry
{
    void TrackChunkResult(AdvisoryAiChunkRequestTelemetry telemetry);

    void TrackChunkFailure(string? tenant, string advisoryKey, string failureReason, string result);
}

internal sealed class AdvisoryAiTelemetry : IAdvisoryAiTelemetry
{
    private readonly ILogger<AdvisoryAiTelemetry> _logger;

    public AdvisoryAiTelemetry(ILogger<AdvisoryAiTelemetry> logger)
    {
        _logger = logger ?? throw new ArgumentNullException(nameof(logger));
    }

    public void TrackChunkResult(AdvisoryAiChunkRequestTelemetry telemetry)
    {
        ArgumentNullException.ThrowIfNull(telemetry);

        var tenant = NormalizeTenant(telemetry.Tenant);
        var result = NormalizeResult(telemetry.Result);

        AdvisoryAiMetrics.ChunkRequestCounter.Add(1,
            AdvisoryAiMetrics.BuildChunkRequestTags(tenant, result, telemetry.Truncated, telemetry.CacheHit));

        AdvisoryAiMetrics.ChunkLatencyHistogram.Record(
            telemetry.Duration.TotalMilliseconds,
            AdvisoryAiMetrics.BuildLatencyTags(tenant, result, telemetry.Truncated, telemetry.CacheHit));

        AdvisoryAiMetrics.ChunkResultHistogram.Record(
            telemetry.ChunkCount,
            AdvisoryAiMetrics.BuildChunkResultTags(tenant, result, telemetry.Truncated));

        AdvisoryAiMetrics.ChunkSourceHistogram.Record(
            telemetry.SourceCount,
            AdvisoryAiMetrics.BuildSourceTags(tenant, result));

        if (telemetry.CacheHit)
        {
            AdvisoryAiMetrics.ChunkCacheHitCounter.Add(1,
                AdvisoryAiMetrics.BuildCacheTags(tenant, "hit"));
        }

        if (!telemetry.CacheHit && telemetry.GuardrailCounts.Count > 0)
        {
            foreach (var kvp in telemetry.GuardrailCounts)
            {
                AdvisoryAiMetrics.GuardrailBlockCounter.Add(kvp.Value,
                    AdvisoryAiMetrics.BuildGuardrailTags(tenant, GetReasonTag(kvp.Key), telemetry.CacheHit));
            }

            _logger.LogInformation(
                "Advisory chunk guardrails blocked {BlockCount} segments for tenant {Tenant} and key {Key}. Details: {Summary}",
                telemetry.TotalGuardrailBlocks,
                tenant,
                telemetry.AdvisoryKey,
                FormatGuardrailSummary(telemetry.GuardrailCounts));
        }

        _logger.LogInformation(
            "Advisory chunk request for tenant {Tenant} key {Key} returned {Chunks} chunks across {Sources} sources (observationsLoaded: {Observations}, truncated: {Truncated}, cacheHit: {CacheHit}, guardrailBlocks: {GuardrailBlocks}, durationMs: {Duration}).",
            tenant,
            telemetry.AdvisoryKey,
            telemetry.ChunkCount,
            telemetry.SourceCount,
            telemetry.ObservationCount,
            telemetry.Truncated,
            telemetry.CacheHit,
            telemetry.TotalGuardrailBlocks,
            telemetry.Duration.TotalMilliseconds.ToString("F2", CultureInfo.InvariantCulture));
    }

    public void TrackChunkFailure(string? tenant, string advisoryKey, string failureReason, string result)
    {
        var normalizedTenant = NormalizeTenant(tenant);
        var normalizedResult = NormalizeResult(result);

        AdvisoryAiMetrics.ChunkRequestCounter.Add(1,
            AdvisoryAiMetrics.BuildChunkRequestTags(normalizedTenant, normalizedResult, truncated: false, cacheHit: false));

        _logger.LogWarning(
            "Advisory chunk request for tenant {Tenant} key {Key} failed ({Result}): {Reason}",
            normalizedTenant,
            advisoryKey,
            normalizedResult,
            failureReason);
    }

    private static string NormalizeTenant(string? tenant)
        => string.IsNullOrWhiteSpace(tenant) ? "unknown" : tenant;

    private static string NormalizeResult(string? result)
        => string.IsNullOrWhiteSpace(result) ? "unknown" : result;

    private static string GetReasonTag(AdvisoryChunkGuardrailReason reason)
        => reason switch
        {
            AdvisoryChunkGuardrailReason.NormalizationFailed => "normalization_failed",
            AdvisoryChunkGuardrailReason.BelowMinimumLength => "below_minimum_length",
            AdvisoryChunkGuardrailReason.MissingAlphabeticCharacters => "missing_alpha_characters",
            _ => reason.ToString().ToLowerInvariant()
        };

    private static string FormatGuardrailSummary(IReadOnlyDictionary<AdvisoryChunkGuardrailReason, int> counts)
    {
        if (counts.Count == 0)
        {
            return "none";
        }

        var parts = counts
            .OrderBy(static kvp => kvp.Key)
            .Select(kvp => $"{GetReasonTag(kvp.Key)}={kvp.Value}");
        return string.Join(",", parts);
    }
}

internal sealed record AdvisoryAiChunkRequestTelemetry(
    string? Tenant,
    string AdvisoryKey,
    string Result,
    bool Truncated,
    bool CacheHit,
    int ObservationCount,
    int SourceCount,
    int ChunkCount,
    TimeSpan Duration,
    IReadOnlyDictionary<AdvisoryChunkGuardrailReason, int> GuardrailCounts)
{
    public int TotalGuardrailBlocks => GuardrailCounts.Count == 0
        ? 0
        : GuardrailCounts.Values.Sum();
}