Add Ruby language analyzer and related functionality

- Introduced global usings for Ruby analyzer.
- Implemented RubyLockData, RubyLockEntry, and RubyLockParser for handling Gemfile.lock files.
- Created RubyPackage and RubyPackageCollector to manage Ruby packages and vendor cache.
- Developed RubyAnalyzerPlugin and RubyLanguageAnalyzer for analyzing Ruby projects.
- Added tests for Ruby language analyzer with sample Gemfile.lock and expected output.
- Included necessary project files and references for the Ruby analyzer.
- Added third-party licenses for tree-sitter dependencies.
This commit is contained in:
master
2025-11-03 01:15:43 +02:00
parent ff0eca3a51
commit bf2bf4b395
88 changed files with 6557 additions and 1568 deletions

View File

@@ -0,0 +1,27 @@
using System;
using System.Collections.Generic;
namespace StellaOps.AdvisoryAI.Orchestration;
/// <summary>
/// Queue payload sent to workers to execute a pipeline plan.
/// </summary>
public sealed class AdvisoryPipelineExecutionMessage
{
public AdvisoryPipelineExecutionMessage(
string planCacheKey,
AdvisoryTaskRequest request,
IReadOnlyDictionary<string, string> planMetadata)
{
ArgumentException.ThrowIfNullOrWhiteSpace(planCacheKey);
PlanCacheKey = planCacheKey;
Request = request ?? throw new ArgumentNullException(nameof(request));
PlanMetadata = planMetadata ?? throw new ArgumentNullException(nameof(planMetadata));
}
public string PlanCacheKey { get; }
public AdvisoryTaskRequest Request { get; }
public IReadOnlyDictionary<string, string> PlanMetadata { get; }
}

View File

@@ -0,0 +1,159 @@
using System.Collections.ObjectModel;
namespace StellaOps.AdvisoryAI.Orchestration;
public sealed class AdvisoryPipelineOptions
{
private static readonly AdvisoryTaskConfiguration SummaryDefaults = new()
{
PromptTemplate = "prompts/advisory/summary.liquid",
StructuredMaxChunks = 25,
VectorTopK = 5,
SbomMaxTimelineEntries = 10,
SbomMaxDependencyPaths = 20,
IncludeEnvironmentFlags = true,
IncludeBlastRadius = true,
Budget = new AdvisoryTaskBudget { PromptTokens = 2048, CompletionTokens = 512 },
VectorQueries = { "Summarize key facts", "What is impacted?" },
};
private static readonly AdvisoryTaskConfiguration ConflictDefaults = new()
{
PromptTemplate = "prompts/advisory/conflict.liquid",
StructuredMaxChunks = 30,
VectorTopK = 6,
SbomMaxTimelineEntries = 8,
SbomMaxDependencyPaths = 15,
IncludeEnvironmentFlags = true,
IncludeBlastRadius = false,
Budget = new AdvisoryTaskBudget { PromptTokens = 2048, CompletionTokens = 512 },
VectorQueries = { "Highlight conflicting statements", "Where do sources disagree?" },
};
private static readonly AdvisoryTaskConfiguration RemediationDefaults = new()
{
PromptTemplate = "prompts/advisory/remediation.liquid",
StructuredMaxChunks = 35,
VectorTopK = 6,
SbomMaxTimelineEntries = 12,
SbomMaxDependencyPaths = 25,
IncludeEnvironmentFlags = true,
IncludeBlastRadius = true,
Budget = new AdvisoryTaskBudget { PromptTokens = 2048, CompletionTokens = 640 },
VectorQueries = { "Provide remediation steps", "Outline mitigations and fixes" },
};
public IDictionary<AdvisoryTaskType, AdvisoryTaskConfiguration> Tasks { get; } =
new Dictionary<AdvisoryTaskType, AdvisoryTaskConfiguration>
{
[AdvisoryTaskType.Summary] = SummaryDefaults.Clone(),
[AdvisoryTaskType.Conflict] = ConflictDefaults.Clone(),
[AdvisoryTaskType.Remediation] = RemediationDefaults.Clone(),
};
public void ApplyDefaults()
{
if (!Tasks.ContainsKey(AdvisoryTaskType.Summary))
{
Tasks[AdvisoryTaskType.Summary] = SummaryDefaults.Clone();
}
if (!Tasks.ContainsKey(AdvisoryTaskType.Conflict))
{
Tasks[AdvisoryTaskType.Conflict] = ConflictDefaults.Clone();
}
if (!Tasks.ContainsKey(AdvisoryTaskType.Remediation))
{
Tasks[AdvisoryTaskType.Remediation] = RemediationDefaults.Clone();
}
foreach (var entry in Tasks)
{
entry.Value.ApplyDefaults();
}
}
public AdvisoryTaskConfiguration GetConfiguration(AdvisoryTaskType taskType)
{
ApplyDefaults();
if (!Tasks.TryGetValue(taskType, out var configuration))
{
throw new InvalidOperationException($"No configuration registered for task type '{taskType}'.");
}
configuration.ApplyDefaults();
return configuration;
}
}
public sealed class AdvisoryTaskConfiguration
{
public string PromptTemplate { get; set; } = string.Empty;
public List<string> VectorQueries { get; set; } = new();
public int VectorTopK { get; set; }
public int StructuredMaxChunks { get; set; }
public int SbomMaxTimelineEntries { get; set; }
public int SbomMaxDependencyPaths { get; set; }
public bool IncludeEnvironmentFlags { get; set; }
public bool IncludeBlastRadius { get; set; }
public AdvisoryTaskBudget Budget { get; set; } = new();
internal AdvisoryTaskConfiguration Clone()
{
var clone = (AdvisoryTaskConfiguration)MemberwiseClone();
clone.VectorQueries = new List<string>(VectorQueries);
clone.Budget = new AdvisoryTaskBudget
{
PromptTokens = Budget.PromptTokens,
CompletionTokens = Budget.CompletionTokens,
};
return clone;
}
internal void ApplyDefaults()
{
if (string.IsNullOrWhiteSpace(PromptTemplate))
{
PromptTemplate = "prompts/advisory/default.liquid";
}
if (VectorQueries.Count == 0)
{
VectorQueries.Add("Provide relevant advisory details");
}
if (VectorTopK <= 0)
{
VectorTopK = 5;
}
if (StructuredMaxChunks <= 0)
{
StructuredMaxChunks = 20;
}
if (SbomMaxTimelineEntries < 0)
{
SbomMaxTimelineEntries = 0;
}
if (SbomMaxDependencyPaths < 0)
{
SbomMaxDependencyPaths = 0;
}
Budget ??= new AdvisoryTaskBudget();
}
internal IReadOnlyList<string> GetVectorQueries()
=> new ReadOnlyCollection<string>(VectorQueries);
}

View File

@@ -0,0 +1,230 @@
using System.Collections.Immutable;
using System.Globalization;
using System.Security.Cryptography;
using System.Text;
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Options;
using StellaOps.AdvisoryAI.Abstractions;
using StellaOps.AdvisoryAI.Context;
using StellaOps.AdvisoryAI.Tools;
namespace StellaOps.AdvisoryAI.Orchestration;
internal sealed class AdvisoryPipelineOrchestrator : IAdvisoryPipelineOrchestrator
{
private readonly IAdvisoryStructuredRetriever _structuredRetriever;
private readonly IAdvisoryVectorRetriever _vectorRetriever;
private readonly ISbomContextRetriever _sbomContextRetriever;
private readonly IDeterministicToolset _toolset;
private readonly AdvisoryPipelineOptions _options;
private readonly ILogger<AdvisoryPipelineOrchestrator>? _logger;
public AdvisoryPipelineOrchestrator(
IAdvisoryStructuredRetriever structuredRetriever,
IAdvisoryVectorRetriever vectorRetriever,
ISbomContextRetriever sbomContextRetriever,
IDeterministicToolset toolset,
IOptions<AdvisoryPipelineOptions> options,
ILogger<AdvisoryPipelineOrchestrator>? logger = null)
{
_structuredRetriever = structuredRetriever ?? throw new ArgumentNullException(nameof(structuredRetriever));
_vectorRetriever = vectorRetriever ?? throw new ArgumentNullException(nameof(vectorRetriever));
_sbomContextRetriever = sbomContextRetriever ?? throw new ArgumentNullException(nameof(sbomContextRetriever));
_toolset = toolset ?? throw new ArgumentNullException(nameof(toolset));
_options = options?.Value ?? throw new ArgumentNullException(nameof(options));
_options.ApplyDefaults();
_logger = logger;
}
public async Task<AdvisoryTaskPlan> CreatePlanAsync(AdvisoryTaskRequest request, CancellationToken cancellationToken)
{
ArgumentNullException.ThrowIfNull(request);
var config = _options.GetConfiguration(request.TaskType);
var structuredRequest = new AdvisoryRetrievalRequest(
request.AdvisoryKey,
request.PreferredSections,
config.StructuredMaxChunks);
var structured = await _structuredRetriever
.RetrieveAsync(structuredRequest, cancellationToken)
.ConfigureAwait(false);
var vectorResults = await RetrieveVectorMatchesAsync(request, structuredRequest, config, cancellationToken).ConfigureAwait(false);
var (sbomContext, dependencyAnalysis) = await RetrieveSbomContextAsync(request, config, cancellationToken).ConfigureAwait(false);
var metadata = BuildMetadata(request, structured, vectorResults, sbomContext, dependencyAnalysis);
var cacheKey = ComputeCacheKey(request, structured, vectorResults, sbomContext, dependencyAnalysis);
var plan = new AdvisoryTaskPlan(
request,
cacheKey,
config.PromptTemplate,
structured.Chunks.ToImmutableArray(),
vectorResults,
sbomContext,
dependencyAnalysis,
config.Budget,
metadata);
return plan;
}
private async Task<ImmutableArray<AdvisoryVectorResult>> RetrieveVectorMatchesAsync(
AdvisoryTaskRequest request,
AdvisoryRetrievalRequest structuredRequest,
AdvisoryTaskConfiguration configuration,
CancellationToken cancellationToken)
{
if (configuration.VectorQueries.Count == 0)
{
return ImmutableArray<AdvisoryVectorResult>.Empty;
}
var builder = ImmutableArray.CreateBuilder<AdvisoryVectorResult>(configuration.VectorQueries.Count);
foreach (var query in configuration.GetVectorQueries())
{
var vectorRequest = new VectorRetrievalRequest(structuredRequest, query, configuration.VectorTopK);
var matches = await _vectorRetriever
.SearchAsync(vectorRequest, cancellationToken)
.ConfigureAwait(false);
builder.Add(new AdvisoryVectorResult(query, matches.ToImmutableArray()));
}
return builder.MoveToImmutable();
}
private async Task<(SbomContextResult? Context, DependencyAnalysisResult? Analysis)> RetrieveSbomContextAsync(
AdvisoryTaskRequest request,
AdvisoryTaskConfiguration configuration,
CancellationToken cancellationToken)
{
if (string.IsNullOrEmpty(request.ArtifactId))
{
return (null, null);
}
var sbomRequest = new SbomContextRequest(
artifactId: request.ArtifactId!,
purl: request.ArtifactPurl,
maxTimelineEntries: configuration.SbomMaxTimelineEntries,
maxDependencyPaths: configuration.SbomMaxDependencyPaths,
includeEnvironmentFlags: configuration.IncludeEnvironmentFlags,
includeBlastRadius: configuration.IncludeBlastRadius);
var context = await _sbomContextRetriever
.RetrieveAsync(sbomRequest, cancellationToken)
.ConfigureAwait(false);
var analysis = _toolset.AnalyzeDependencies(context);
return (context, analysis);
}
private static ImmutableDictionary<string, string> BuildMetadata(
AdvisoryTaskRequest request,
AdvisoryRetrievalResult structured,
ImmutableArray<AdvisoryVectorResult> vectors,
SbomContextResult? sbom,
DependencyAnalysisResult? dependency)
{
var builder = ImmutableDictionary.CreateBuilder<string, string>(StringComparer.Ordinal);
builder["task_type"] = request.TaskType.ToString();
builder["advisory_key"] = request.AdvisoryKey;
builder["profile"] = request.Profile;
builder["structured_chunk_count"] = structured.Chunks.Count.ToString(CultureInfo.InvariantCulture);
builder["vector_query_count"] = vectors.Length.ToString(CultureInfo.InvariantCulture);
builder["vector_match_count"] = vectors.Sum(result => result.Matches.Length).ToString(CultureInfo.InvariantCulture);
builder["includes_sbom"] = (sbom is not null).ToString();
builder["dependency_node_count"] = (dependency?.Nodes.Length ?? 0).ToString(CultureInfo.InvariantCulture);
builder["force_refresh"] = request.ForceRefresh.ToString();
if (!string.IsNullOrEmpty(request.PolicyVersion))
{
builder["policy_version"] = request.PolicyVersion!;
}
if (sbom is not null)
{
builder["sbom_version_count"] = sbom.VersionTimeline.Count.ToString(CultureInfo.InvariantCulture);
builder["sbom_dependency_path_count"] = sbom.DependencyPaths.Count.ToString(CultureInfo.InvariantCulture);
}
return builder.ToImmutable();
}
private static string ComputeCacheKey(
AdvisoryTaskRequest request,
AdvisoryRetrievalResult structured,
ImmutableArray<AdvisoryVectorResult> vectors,
SbomContextResult? sbom,
DependencyAnalysisResult? dependency)
{
var builder = new StringBuilder();
builder.Append(request.TaskType)
.Append('|').Append(request.AdvisoryKey)
.Append('|').Append(request.ArtifactId ?? string.Empty)
.Append('|').Append(request.PolicyVersion ?? string.Empty)
.Append('|').Append(request.Profile);
if (request.PreferredSections is not null)
{
foreach (var section in request.PreferredSections.OrderBy(s => s, StringComparer.OrdinalIgnoreCase))
{
builder.Append('|').Append(section);
}
}
foreach (var chunkId in structured.Chunks
.Select(chunk => chunk.ChunkId)
.OrderBy(id => id, StringComparer.Ordinal))
{
builder.Append("|chunk:").Append(chunkId);
}
foreach (var vector in vectors)
{
builder.Append("|query:").Append(vector.Query);
foreach (var match in vector.Matches
.OrderBy(m => m.ChunkId, StringComparer.Ordinal)
.ThenBy(m => m.Score))
{
builder.Append("|match:")
.Append(match.ChunkId)
.Append('@')
.Append(match.Score.ToString("G", CultureInfo.InvariantCulture));
}
}
if (sbom is not null)
{
builder.Append("|sbom:timeline=").Append(sbom.VersionTimeline.Count);
builder.Append("|sbom:paths=").Append(sbom.DependencyPaths.Count);
foreach (var kvp in sbom.Metadata.OrderBy(k => k.Key, StringComparer.Ordinal))
{
builder.Append("|sbommeta:")
.Append(kvp.Key)
.Append('=')
.Append(kvp.Value);
}
}
if (dependency is not null)
{
foreach (var node in dependency.Nodes
.OrderBy(n => n.Identifier, StringComparer.Ordinal))
{
builder.Append("|dep:")
.Append(node.Identifier)
.Append(':')
.Append(node.RuntimeOccurrences)
.Append(':')
.Append(node.DevelopmentOccurrences);
}
}
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(builder.ToString()));
return Convert.ToHexString(hash);
}
}

View File

@@ -0,0 +1,110 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using StellaOps.AdvisoryAI.Tools;
namespace StellaOps.AdvisoryAI.Orchestration;
/// <summary>
/// DTO exposed via service/CLI APIs describing the orchestration plan without leaking raw chunks.
/// </summary>
public sealed class AdvisoryPipelinePlanResponse
{
private AdvisoryPipelinePlanResponse(
string taskType,
string cacheKey,
string promptTemplate,
AdvisoryTaskBudget budget,
IReadOnlyList<PipelineChunkSummary> chunks,
IReadOnlyList<PipelineVectorSummary> vectors,
PipelineSbomSummary? sbom,
IReadOnlyDictionary<string, string> metadata)
{
TaskType = taskType;
CacheKey = cacheKey;
PromptTemplate = promptTemplate;
Budget = budget;
Chunks = chunks;
Vectors = vectors;
Sbom = sbom;
Metadata = metadata;
}
public string TaskType { get; }
public string CacheKey { get; }
public string PromptTemplate { get; }
public AdvisoryTaskBudget Budget { get; }
public IReadOnlyList<PipelineChunkSummary> Chunks { get; }
public IReadOnlyList<PipelineVectorSummary> Vectors { get; }
public PipelineSbomSummary? Sbom { get; }
public IReadOnlyDictionary<string, string> Metadata { get; }
public static AdvisoryPipelinePlanResponse FromPlan(AdvisoryTaskPlan plan)
{
ArgumentNullException.ThrowIfNull(plan);
var chunkSummaries = plan.StructuredChunks
.Select(chunk => new PipelineChunkSummary(
chunk.DocumentId,
chunk.ChunkId,
chunk.Section,
chunk.Metadata.ContainsKey("section") ? chunk.Metadata["section"] : chunk.Section))
.ToImmutableArray();
var vectorSummaries = plan.VectorResults
.Select(vector => new PipelineVectorSummary(
vector.Query,
vector.Matches
.Select(match => new PipelineVectorMatchSummary(match.ChunkId, match.Score))
.ToImmutableArray()))
.ToImmutableArray();
PipelineSbomSummary? sbomSummary = null;
if (plan.SbomContext is not null)
{
sbomSummary = new PipelineSbomSummary(
plan.SbomContext.ArtifactId,
plan.SbomContext.VersionTimeline.Count,
plan.SbomContext.DependencyPaths.Count,
plan.DependencyAnalysis?.Nodes.Length ?? 0);
}
return new AdvisoryPipelinePlanResponse(
plan.Request.TaskType.ToString(),
plan.CacheKey,
plan.PromptTemplate,
plan.Budget,
chunkSummaries,
vectorSummaries,
sbomSummary,
plan.Metadata);
}
}
public sealed record PipelineChunkSummary(
string DocumentId,
string ChunkId,
string Section,
string DisplaySection);
public sealed record PipelineVectorSummary(
string Query,
ImmutableArray<PipelineVectorMatchSummary> Matches);
public sealed record PipelineVectorMatchSummary(
string ChunkId,
double Score);
public sealed record PipelineSbomSummary(
string ArtifactId,
int VersionTimelineCount,
int DependencyPathCount,
int DependencyNodeCount);

View File

@@ -0,0 +1,69 @@
using System.Collections.Immutable;
using StellaOps.AdvisoryAI.Abstractions;
using StellaOps.AdvisoryAI.Context;
using StellaOps.AdvisoryAI.Tools;
namespace StellaOps.AdvisoryAI.Orchestration;
public sealed class AdvisoryTaskPlan
{
public AdvisoryTaskPlan(
AdvisoryTaskRequest request,
string cacheKey,
string promptTemplate,
ImmutableArray<AdvisoryChunk> structuredChunks,
ImmutableArray<AdvisoryVectorResult> vectorResults,
SbomContextResult? sbomContext,
DependencyAnalysisResult? dependencyAnalysis,
AdvisoryTaskBudget budget,
ImmutableDictionary<string, string> metadata)
{
Request = request ?? throw new ArgumentNullException(nameof(request));
CacheKey = cacheKey ?? throw new ArgumentNullException(nameof(cacheKey));
PromptTemplate = promptTemplate ?? throw new ArgumentNullException(nameof(promptTemplate));
StructuredChunks = structuredChunks;
VectorResults = vectorResults;
SbomContext = sbomContext;
DependencyAnalysis = dependencyAnalysis;
Budget = budget ?? throw new ArgumentNullException(nameof(budget));
Metadata = metadata ?? throw new ArgumentNullException(nameof(metadata));
}
public AdvisoryTaskRequest Request { get; }
public string CacheKey { get; }
public string PromptTemplate { get; }
public ImmutableArray<AdvisoryChunk> StructuredChunks { get; }
public ImmutableArray<AdvisoryVectorResult> VectorResults { get; }
public SbomContextResult? SbomContext { get; }
public DependencyAnalysisResult? DependencyAnalysis { get; }
public AdvisoryTaskBudget Budget { get; }
public ImmutableDictionary<string, string> Metadata { get; }
}
public sealed class AdvisoryVectorResult
{
public AdvisoryVectorResult(string query, ImmutableArray<VectorRetrievalMatch> matches)
{
Query = string.IsNullOrWhiteSpace(query) ? throw new ArgumentException(nameof(query)) : query;
Matches = matches;
}
public string Query { get; }
public ImmutableArray<VectorRetrievalMatch> Matches { get; }
}
public sealed class AdvisoryTaskBudget
{
public int PromptTokens { get; init; } = 2048;
public int CompletionTokens { get; init; } = 512;
}

View File

@@ -0,0 +1,49 @@
using System.Collections.Generic;
namespace StellaOps.AdvisoryAI.Orchestration;
public sealed class AdvisoryTaskRequest
{
public AdvisoryTaskRequest(
AdvisoryTaskType taskType,
string advisoryKey,
string? artifactId = null,
string? artifactPurl = null,
string? policyVersion = null,
string profile = "default",
IReadOnlyCollection<string>? preferredSections = null,
bool forceRefresh = false)
{
if (!Enum.IsDefined(typeof(AdvisoryTaskType), taskType))
{
throw new ArgumentOutOfRangeException(nameof(taskType));
}
ArgumentException.ThrowIfNullOrWhiteSpace(advisoryKey);
TaskType = taskType;
AdvisoryKey = advisoryKey.Trim();
ArtifactId = string.IsNullOrWhiteSpace(artifactId) ? null : artifactId.Trim();
ArtifactPurl = string.IsNullOrWhiteSpace(artifactPurl) ? null : artifactPurl.Trim();
PolicyVersion = string.IsNullOrWhiteSpace(policyVersion) ? null : policyVersion.Trim();
Profile = string.IsNullOrWhiteSpace(profile) ? "default" : profile.Trim();
PreferredSections = preferredSections;
ForceRefresh = forceRefresh;
}
public AdvisoryTaskType TaskType { get; }
public string AdvisoryKey { get; }
public string? ArtifactId { get; }
public string? ArtifactPurl { get; }
public string? PolicyVersion { get; }
public string Profile { get; }
public IReadOnlyCollection<string>? PreferredSections { get; }
public bool ForceRefresh { get; }
}

View File

@@ -0,0 +1,8 @@
namespace StellaOps.AdvisoryAI.Orchestration;
public enum AdvisoryTaskType
{
Summary,
Conflict,
Remediation,
}

View File

@@ -0,0 +1,9 @@
using System.Threading;
using System.Threading.Tasks;
namespace StellaOps.AdvisoryAI.Orchestration;
public interface IAdvisoryPipelineOrchestrator
{
Task<AdvisoryTaskPlan> CreatePlanAsync(AdvisoryTaskRequest request, CancellationToken cancellationToken);
}