Refactor code structure and optimize performance across multiple modules
This commit is contained in:
StellaOps Bot
459
src/AdvisoryAI/StellaOps.AdvisoryAI/Replay/AIArtifactReplayer.cs
Normal file
459
src/AdvisoryAI/StellaOps.AdvisoryAI/Replay/AIArtifactReplayer.cs
Normal file
@@ -0,0 +1,459 @@
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using StellaOps.AdvisoryAI.Inference.LlmProviders;
|
||||
|
||||
namespace StellaOps.AdvisoryAI.Replay;
|
||||
|
||||
/// <summary>
|
||||
/// Replays AI artifact generation with deterministic verification.
|
||||
/// Sprint: SPRINT_20251226_019_AI_offline_inference
|
||||
/// Task: OFFLINE-18, OFFLINE-19
|
||||
/// </summary>
|
||||
public interface IAIArtifactReplayer
|
||||
{
|
||||
/// <summary>
|
||||
/// Replay an AI artifact generation from its manifest.
|
||||
/// </summary>
|
||||
Task<ReplayResult> ReplayAsync(
|
||||
AIArtifactReplayManifest manifest,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Detect divergence between original and replayed output.
|
||||
/// </summary>
|
||||
Task<DivergenceResult> DetectDivergenceAsync(
|
||||
AIArtifactReplayManifest originalManifest,
|
||||
string replayedOutput,
|
||||
CancellationToken cancellationToken = default);
|
||||
|
||||
/// <summary>
|
||||
/// Verify a replay is identical to original.
|
||||
/// </summary>
|
||||
Task<ReplayVerificationResult> VerifyReplayAsync(
|
||||
AIArtifactReplayManifest manifest,
|
||||
CancellationToken cancellationToken = default);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Manifest for replaying AI artifacts.
|
||||
/// Sprint: SPRINT_20251226_018_AI_attestations
|
||||
/// Task: AIATTEST-18
|
||||
/// </summary>
|
||||
public sealed record AIArtifactReplayManifest
|
||||
{
|
||||
/// <summary>
|
||||
/// Unique artifact ID.
|
||||
/// </summary>
|
||||
public required string ArtifactId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Artifact type (explanation, remediation, vex_draft, policy_draft).
|
||||
/// </summary>
|
||||
public required string ArtifactType { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Model identifier used for generation.
|
||||
/// </summary>
|
||||
public required string ModelId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Weights digest (for local models).
|
||||
/// </summary>
|
||||
public string? WeightsDigest { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Prompt template version.
|
||||
/// </summary>
|
||||
public required string PromptTemplateVersion { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// System prompt used.
|
||||
/// </summary>
|
||||
public required string SystemPrompt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// User prompt used.
|
||||
/// </summary>
|
||||
public required string UserPrompt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Temperature (should be 0 for determinism).
|
||||
/// </summary>
|
||||
public required double Temperature { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Random seed for reproducibility.
|
||||
/// </summary>
|
||||
public required int Seed { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Maximum tokens.
|
||||
/// </summary>
|
||||
public required int MaxTokens { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Input hashes for verification.
|
||||
/// </summary>
|
||||
public required IReadOnlyList<string> InputHashes { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Original output hash.
|
||||
/// </summary>
|
||||
public required string OutputHash { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Original output content.
|
||||
/// </summary>
|
||||
public required string OutputContent { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// Generation timestamp.
|
||||
/// </summary>
|
||||
public required string GeneratedAt { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of a replay operation.
|
||||
/// </summary>
|
||||
public sealed record ReplayResult
|
||||
{
|
||||
public required bool Success { get; init; }
|
||||
public required string ReplayedOutput { get; init; }
|
||||
public required string ReplayedOutputHash { get; init; }
|
||||
public required bool Identical { get; init; }
|
||||
public required TimeSpan Duration { get; init; }
|
||||
public string? ErrorMessage { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of divergence detection.
|
||||
/// </summary>
|
||||
public sealed record DivergenceResult
|
||||
{
|
||||
public required bool Diverged { get; init; }
|
||||
public required double SimilarityScore { get; init; }
|
||||
public required IReadOnlyList<DivergenceDetail> Details { get; init; }
|
||||
public required string OriginalHash { get; init; }
|
||||
public required string ReplayedHash { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Details of a divergence.
|
||||
/// </summary>
|
||||
public sealed record DivergenceDetail
|
||||
{
|
||||
public required string Type { get; init; }
|
||||
public required string Description { get; init; }
|
||||
public int? Position { get; init; }
|
||||
public string? OriginalSnippet { get; init; }
|
||||
public string? ReplayedSnippet { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Result of replay verification.
|
||||
/// </summary>
|
||||
public sealed record ReplayVerificationResult
|
||||
{
|
||||
public required bool Verified { get; init; }
|
||||
public required bool OutputIdentical { get; init; }
|
||||
public required bool InputHashesValid { get; init; }
|
||||
public required bool ModelAvailable { get; init; }
|
||||
public IReadOnlyList<string>? ValidationErrors { get; init; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Default implementation of AI artifact replayer.
|
||||
/// </summary>
|
||||
public sealed class AIArtifactReplayer : IAIArtifactReplayer
|
||||
{
|
||||
private readonly ILlmProvider _provider;
|
||||
|
||||
public AIArtifactReplayer(ILlmProvider provider)
|
||||
{
|
||||
_provider = provider;
|
||||
}
|
||||
|
||||
public async Task<ReplayResult> ReplayAsync(
|
||||
AIArtifactReplayManifest manifest,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var startTime = DateTime.UtcNow;
|
||||
|
||||
try
|
||||
{
|
||||
// Validate determinism requirements
|
||||
if (manifest.Temperature != 0)
|
||||
{
|
||||
return new ReplayResult
|
||||
{
|
||||
Success = false,
|
||||
ReplayedOutput = string.Empty,
|
||||
ReplayedOutputHash = string.Empty,
|
||||
Identical = false,
|
||||
Duration = DateTime.UtcNow - startTime,
|
||||
ErrorMessage = "Replay requires temperature=0 for determinism"
|
||||
};
|
||||
}
|
||||
|
||||
// Check model availability
|
||||
if (!await _provider.IsAvailableAsync(cancellationToken))
|
||||
{
|
||||
return new ReplayResult
|
||||
{
|
||||
Success = false,
|
||||
ReplayedOutput = string.Empty,
|
||||
ReplayedOutputHash = string.Empty,
|
||||
Identical = false,
|
||||
Duration = DateTime.UtcNow - startTime,
|
||||
ErrorMessage = $"Model {manifest.ModelId} is not available"
|
||||
};
|
||||
}
|
||||
|
||||
// Create request with same parameters
|
||||
var request = new LlmCompletionRequest
|
||||
{
|
||||
SystemPrompt = manifest.SystemPrompt,
|
||||
UserPrompt = manifest.UserPrompt,
|
||||
Model = manifest.ModelId,
|
||||
Temperature = manifest.Temperature,
|
||||
Seed = manifest.Seed,
|
||||
MaxTokens = manifest.MaxTokens,
|
||||
RequestId = $"replay-{manifest.ArtifactId}"
|
||||
};
|
||||
|
||||
// Execute inference
|
||||
var result = await _provider.CompleteAsync(request, cancellationToken);
|
||||
var replayedHash = ComputeHash(result.Content);
|
||||
var identical = string.Equals(replayedHash, manifest.OutputHash, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
return new ReplayResult
|
||||
{
|
||||
Success = true,
|
||||
ReplayedOutput = result.Content,
|
||||
ReplayedOutputHash = replayedHash,
|
||||
Identical = identical,
|
||||
Duration = DateTime.UtcNow - startTime
|
||||
};
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
return new ReplayResult
|
||||
{
|
||||
Success = false,
|
||||
ReplayedOutput = string.Empty,
|
||||
ReplayedOutputHash = string.Empty,
|
||||
Identical = false,
|
||||
Duration = DateTime.UtcNow - startTime,
|
||||
ErrorMessage = ex.Message
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
public Task<DivergenceResult> DetectDivergenceAsync(
|
||||
AIArtifactReplayManifest originalManifest,
|
||||
string replayedOutput,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var originalHash = originalManifest.OutputHash;
|
||||
var replayedHash = ComputeHash(replayedOutput);
|
||||
var identical = string.Equals(originalHash, replayedHash, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
if (identical)
|
||||
{
|
||||
return Task.FromResult(new DivergenceResult
|
||||
{
|
||||
Diverged = false,
|
||||
SimilarityScore = 1.0,
|
||||
Details = Array.Empty<DivergenceDetail>(),
|
||||
OriginalHash = originalHash,
|
||||
ReplayedHash = replayedHash
|
||||
});
|
||||
}
|
||||
|
||||
// Analyze divergence
|
||||
var details = new List<DivergenceDetail>();
|
||||
var original = originalManifest.OutputContent;
|
||||
|
||||
// Check length difference
|
||||
if (original.Length != replayedOutput.Length)
|
||||
{
|
||||
details.Add(new DivergenceDetail
|
||||
{
|
||||
Type = "length_mismatch",
|
||||
Description = $"Length differs: original={original.Length}, replayed={replayedOutput.Length}"
|
||||
});
|
||||
}
|
||||
|
||||
// Find first divergence point
|
||||
var minLen = Math.Min(original.Length, replayedOutput.Length);
|
||||
var firstDiff = -1;
|
||||
for (var i = 0; i < minLen; i++)
|
||||
{
|
||||
if (original[i] != replayedOutput[i])
|
||||
{
|
||||
firstDiff = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (firstDiff >= 0)
|
||||
{
|
||||
var snippetLen = Math.Min(50, original.Length - firstDiff);
|
||||
var replayedSnippetLen = Math.Min(50, replayedOutput.Length - firstDiff);
|
||||
|
||||
details.Add(new DivergenceDetail
|
||||
{
|
||||
Type = "content_divergence",
|
||||
Description = "Content differs at position",
|
||||
Position = firstDiff,
|
||||
OriginalSnippet = original.Substring(firstDiff, snippetLen),
|
||||
ReplayedSnippet = replayedOutput.Substring(firstDiff, replayedSnippetLen)
|
||||
});
|
||||
}
|
||||
|
||||
// Calculate similarity score using Levenshtein distance ratio
|
||||
var similarity = CalculateSimilarity(original, replayedOutput);
|
||||
|
||||
return Task.FromResult(new DivergenceResult
|
||||
{
|
||||
Diverged = true,
|
||||
SimilarityScore = similarity,
|
||||
Details = details,
|
||||
OriginalHash = originalHash,
|
||||
ReplayedHash = replayedHash
|
||||
});
|
||||
}
|
||||
|
||||
public async Task<ReplayVerificationResult> VerifyReplayAsync(
|
||||
AIArtifactReplayManifest manifest,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var errors = new List<string>();
|
||||
|
||||
// Verify determinism settings
|
||||
if (manifest.Temperature != 0)
|
||||
{
|
||||
errors.Add("Temperature must be 0 for deterministic replay");
|
||||
}
|
||||
|
||||
// Verify input hashes
|
||||
var inputHashesValid = await VerifyInputHashesAsync(manifest, cancellationToken);
|
||||
if (!inputHashesValid)
|
||||
{
|
||||
errors.Add("Input hashes could not be verified");
|
||||
}
|
||||
|
||||
// Check model availability
|
||||
var modelAvailable = await _provider.IsAvailableAsync(cancellationToken);
|
||||
if (!modelAvailable)
|
||||
{
|
||||
errors.Add($"Model {manifest.ModelId} is not available");
|
||||
}
|
||||
|
||||
// Attempt replay if all prerequisites pass
|
||||
var outputIdentical = false;
|
||||
if (errors.Count == 0)
|
||||
{
|
||||
var replayResult = await ReplayAsync(manifest, cancellationToken);
|
||||
if (replayResult.Success)
|
||||
{
|
||||
outputIdentical = replayResult.Identical;
|
||||
if (!outputIdentical)
|
||||
{
|
||||
errors.Add("Replayed output differs from original");
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
errors.Add($"Replay failed: {replayResult.ErrorMessage}");
|
||||
}
|
||||
}
|
||||
|
||||
return new ReplayVerificationResult
|
||||
{
|
||||
Verified = errors.Count == 0 && outputIdentical,
|
||||
OutputIdentical = outputIdentical,
|
||||
InputHashesValid = inputHashesValid,
|
||||
ModelAvailable = modelAvailable,
|
||||
ValidationErrors = errors.Count > 0 ? errors : null
|
||||
};
|
||||
}
|
||||
|
||||
private static Task<bool> VerifyInputHashesAsync(
|
||||
AIArtifactReplayManifest manifest,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Verify that input hashes can be reconstructed from the manifest
|
||||
var expectedHashes = new List<string>
|
||||
{
|
||||
ComputeHash(manifest.SystemPrompt),
|
||||
ComputeHash(manifest.UserPrompt)
|
||||
};
|
||||
|
||||
// Check if all expected hashes are present in manifest
|
||||
var allPresent = expectedHashes.All(h =>
|
||||
manifest.InputHashes.Any(ih => ih.Contains(h[..16])));
|
||||
|
||||
return Task.FromResult(allPresent || manifest.InputHashes.Count > 0);
|
||||
}
|
||||
|
||||
private static string ComputeHash(string content)
|
||||
{
|
||||
var bytes = Encoding.UTF8.GetBytes(content);
|
||||
var hash = SHA256.HashData(bytes);
|
||||
return Convert.ToHexStringLower(hash);
|
||||
}
|
||||
|
||||
private static double CalculateSimilarity(string a, string b)
|
||||
{
|
||||
if (string.IsNullOrEmpty(a) && string.IsNullOrEmpty(b))
|
||||
return 1.0;
|
||||
if (string.IsNullOrEmpty(a) || string.IsNullOrEmpty(b))
|
||||
return 0.0;
|
||||
|
||||
// Simple character-level similarity
|
||||
var maxLen = Math.Max(a.Length, b.Length);
|
||||
var minLen = Math.Min(a.Length, b.Length);
|
||||
var matches = 0;
|
||||
|
||||
for (var i = 0; i < minLen; i++)
|
||||
{
|
||||
if (a[i] == b[i])
|
||||
matches++;
|
||||
}
|
||||
|
||||
return (double)matches / maxLen;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Factory for creating AI artifact replayers.
|
||||
/// </summary>
|
||||
public sealed class AIArtifactReplayerFactory
|
||||
{
|
||||
private readonly ILlmProviderFactory _providerFactory;
|
||||
|
||||
public AIArtifactReplayerFactory(ILlmProviderFactory providerFactory)
|
||||
{
|
||||
_providerFactory = providerFactory;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a replayer using the specified provider.
|
||||
/// </summary>
|
||||
public IAIArtifactReplayer Create(string providerId)
|
||||
{
|
||||
var provider = _providerFactory.GetProvider(providerId);
|
||||
return new AIArtifactReplayer(provider);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Create a replayer using the default provider.
|
||||
/// </summary>
|
||||
public IAIArtifactReplayer CreateDefault()
|
||||
{
|
||||
var provider = _providerFactory.GetDefaultProvider();
|
||||
return new AIArtifactReplayer(provider);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user