feat: Implement Runtime Facts ingestion service and NDJSON reader
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Added RuntimeFactsNdjsonReader for reading NDJSON formatted runtime facts.
- Introduced IRuntimeFactsIngestionService interface and its implementation.
- Enhanced Program.cs to register new services and endpoints for runtime facts.
- Updated CallgraphIngestionService to include CAS URI in stored artifacts.
- Created RuntimeFactsValidationException for validation errors during ingestion.
- Added tests for RuntimeFactsIngestionService and RuntimeFactsNdjsonReader.
- Implemented SignalsSealedModeMonitor for compliance checks in sealed mode.
- Updated project dependencies for testing utilities.
This commit is contained in:
master
2025-11-10 07:56:15 +02:00
parent 9df52d84aa
commit 69c59defdc
132 changed files with 19718 additions and 9334 deletions

View File

@@ -28,14 +28,15 @@ internal static class CommandFactory
{
TreatUnmatchedTokensAsErrors = true
};
root.Add(verboseOption);
root.Add(BuildScannerCommand(services, verboseOption, cancellationToken));
root.Add(BuildScanCommand(services, options, verboseOption, cancellationToken));
root.Add(BuildDatabaseCommand(services, verboseOption, cancellationToken));
root.Add(BuildSourcesCommand(services, verboseOption, cancellationToken));
root.Add(BuildAocCommand(services, verboseOption, cancellationToken));
root.Add(BuildAuthCommand(services, options, verboseOption, cancellationToken));
root.Add(verboseOption);
root.Add(BuildScannerCommand(services, verboseOption, cancellationToken));
root.Add(BuildScanCommand(services, options, verboseOption, cancellationToken));
root.Add(BuildRubyCommand(services, verboseOption, cancellationToken));
root.Add(BuildDatabaseCommand(services, verboseOption, cancellationToken));
root.Add(BuildSourcesCommand(services, verboseOption, cancellationToken));
root.Add(BuildAocCommand(services, verboseOption, cancellationToken));
root.Add(BuildAuthCommand(services, options, verboseOption, cancellationToken));
root.Add(BuildPolicyCommand(services, options, verboseOption, cancellationToken));
root.Add(BuildTaskRunnerCommand(services, verboseOption, cancellationToken));
root.Add(BuildFindingsCommand(services, verboseOption, cancellationToken));
@@ -177,14 +178,82 @@ internal static class CommandFactory
scan.Add(entryTrace);
scan.Add(run);
scan.Add(upload);
return scan;
}
scan.Add(upload);
return scan;
}
private static Command BuildRubyCommand(IServiceProvider services, Option<bool> verboseOption, CancellationToken cancellationToken)
{
var ruby = new Command("ruby", "Work with Ruby analyzer outputs.");
var inspect = new Command("inspect", "Inspect a local Ruby workspace.");
var inspectRootOption = new Option<string?>("--root")
{
Description = "Path to the Ruby workspace (defaults to current directory)."
};
var inspectFormatOption = new Option<string?>("--format")
{
Description = "Output format (table or json)."
};
inspect.Add(inspectRootOption);
inspect.Add(inspectFormatOption);
inspect.SetAction((parseResult, _) =>
{
var root = parseResult.GetValue(inspectRootOption);
var format = parseResult.GetValue(inspectFormatOption) ?? "table";
var verbose = parseResult.GetValue(verboseOption);
return CommandHandlers.HandleRubyInspectAsync(
services,
root,
format,
verbose,
cancellationToken);
});
var resolve = new Command("resolve", "Fetch Ruby packages for a completed scan.");
var resolveImageOption = new Option<string?>("--image")
{
Description = "Image reference (digest or tag) used by the scan."
};
var resolveScanIdOption = new Option<string?>("--scan-id")
{
Description = "Explicit scan identifier."
};
var resolveFormatOption = new Option<string?>("--format")
{
Description = "Output format (table or json)."
};
resolve.Add(resolveImageOption);
resolve.Add(resolveScanIdOption);
resolve.Add(resolveFormatOption);
resolve.SetAction((parseResult, _) =>
{
var image = parseResult.GetValue(resolveImageOption);
var scanId = parseResult.GetValue(resolveScanIdOption);
var format = parseResult.GetValue(resolveFormatOption) ?? "table";
var verbose = parseResult.GetValue(verboseOption);
return CommandHandlers.HandleRubyResolveAsync(
services,
image,
scanId,
format,
verbose,
cancellationToken);
});
ruby.Add(inspect);
ruby.Add(resolve);
return ruby;
}
private static Command BuildKmsCommand(IServiceProvider services, Option<bool> verboseOption, CancellationToken cancellationToken)
{
var kms = new Command("kms", "Manage file-backed signing keys.");
var kms = new Command("kms", "Manage file-backed signing keys.");
var export = new Command("export", "Export key material to a portable bundle.");
var exportRootOption = new Option<string>("--root")
{

File diff suppressed because it is too large Load Diff

View File

@@ -20,7 +20,8 @@ using StellaOps.Auth.Client;
using StellaOps.Cli.Configuration;
using StellaOps.Cli.Services.Models;
using StellaOps.Cli.Services.Models.AdvisoryAi;
using StellaOps.Cli.Services.Models.Transport;
using StellaOps.Cli.Services.Models.Ruby;
using StellaOps.Cli.Services.Models.Transport;
namespace StellaOps.Cli.Services;
@@ -858,9 +859,9 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient
return MapPolicyFindingExplain(document);
}
public async Task<EntryTraceResponseModel?> GetEntryTraceAsync(string scanId, CancellationToken cancellationToken)
{
EnsureBackendConfigured();
public async Task<EntryTraceResponseModel?> GetEntryTraceAsync(string scanId, CancellationToken cancellationToken)
{
EnsureBackendConfigured();
if (string.IsNullOrWhiteSpace(scanId))
{
@@ -882,15 +883,46 @@ internal sealed class BackendOperationsClient : IBackendOperationsClient
throw new InvalidOperationException(failure);
}
var result = await response.Content.ReadFromJsonAsync<EntryTraceResponseModel>(SerializerOptions, cancellationToken).ConfigureAwait(false);
if (result is null)
{
throw new InvalidOperationException("EntryTrace response payload was empty.");
}
var result = await response.Content.ReadFromJsonAsync<EntryTraceResponseModel>(SerializerOptions, cancellationToken).ConfigureAwait(false);
if (result is null)
{
throw new InvalidOperationException("EntryTrace response payload was empty.");
}
return result;
}
public async Task<IReadOnlyList<RubyPackageArtifactModel>> GetRubyPackagesAsync(string scanId, CancellationToken cancellationToken)
{
EnsureBackendConfigured();
if (string.IsNullOrWhiteSpace(scanId))
{
throw new ArgumentException("Scan identifier is required.", nameof(scanId));
}
using var request = CreateRequest(HttpMethod.Get, $"api/scans/{scanId}/ruby-packages");
await AuthorizeRequestAsync(request, cancellationToken).ConfigureAwait(false);
using var response = await _httpClient.SendAsync(request, cancellationToken).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound)
{
return Array.Empty<RubyPackageArtifactModel>();
}
if (!response.IsSuccessStatusCode)
{
var failure = await CreateFailureMessageAsync(response, cancellationToken).ConfigureAwait(false);
throw new InvalidOperationException(failure);
}
var packages = await response.Content
.ReadFromJsonAsync<IReadOnlyList<RubyPackageArtifactModel>>(SerializerOptions, cancellationToken)
.ConfigureAwait(false);
return packages ?? Array.Empty<RubyPackageArtifactModel>();
}
public async Task<AdvisoryPipelinePlanResponseModel> CreateAdvisoryPipelinePlanAsync(
AdvisoryAiTaskType taskType,
AdvisoryPipelinePlanRequestModel request,

View File

@@ -5,6 +5,7 @@ using System.Threading.Tasks;
using StellaOps.Cli.Configuration;
using StellaOps.Cli.Services.Models;
using StellaOps.Cli.Services.Models.AdvisoryAi;
using StellaOps.Cli.Services.Models.Ruby;
namespace StellaOps.Cli.Services;
@@ -48,6 +49,8 @@ internal interface IBackendOperationsClient
Task<EntryTraceResponseModel?> GetEntryTraceAsync(string scanId, CancellationToken cancellationToken);
Task<IReadOnlyList<RubyPackageArtifactModel>> GetRubyPackagesAsync(string scanId, CancellationToken cancellationToken);
Task<AdvisoryPipelinePlanResponseModel> CreateAdvisoryPipelinePlanAsync(AdvisoryAiTaskType taskType, AdvisoryPipelinePlanRequestModel request, CancellationToken cancellationToken);
Task<AdvisoryPipelineOutputModel?> TryGetAdvisoryPipelineOutputAsync(string cacheKey, AdvisoryAiTaskType taskType, string profile, CancellationToken cancellationToken);

View File

@@ -0,0 +1,28 @@
using System.Collections.Generic;
using System.Text.Json.Serialization;
namespace StellaOps.Cli.Services.Models.Ruby;
internal sealed record RubyPackageArtifactModel(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("name")] string Name,
[property: JsonPropertyName("version")] string? Version,
[property: JsonPropertyName("source")] string? Source,
[property: JsonPropertyName("platform")] string? Platform,
[property: JsonPropertyName("groups")] IReadOnlyList<string>? Groups,
[property: JsonPropertyName("declaredOnly")] bool? DeclaredOnly,
[property: JsonPropertyName("runtimeUsed")] bool? RuntimeUsed,
[property: JsonPropertyName("provenance")] RubyPackageProvenance? Provenance,
[property: JsonPropertyName("runtime")] RubyPackageRuntime? Runtime,
[property: JsonPropertyName("metadata")] IDictionary<string, string?>? Metadata);
internal sealed record RubyPackageProvenance(
[property: JsonPropertyName("source")] string? Source,
[property: JsonPropertyName("lockfile")] string? Lockfile,
[property: JsonPropertyName("locator")] string? Locator);
internal sealed record RubyPackageRuntime(
[property: JsonPropertyName("entrypoints")] IReadOnlyList<string>? Entrypoints,
[property: JsonPropertyName("files")] IReadOnlyList<string>? Files,
[property: JsonPropertyName("reasons")] IReadOnlyList<string>? Reasons);

View File

@@ -47,6 +47,13 @@
<ProjectReference Include="../../Authority/StellaOps.Authority/StellaOps.Auth.Client/StellaOps.Auth.Client.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Plugin/StellaOps.Plugin.csproj" />
<ProjectReference Include="../../Scanner/__Libraries/StellaOps.Scanner.EntryTrace/StellaOps.Scanner.EntryTrace.csproj" />
<ProjectReference Include="../../Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang/StellaOps.Scanner.Analyzers.Lang.csproj" />
<ProjectReference Include="../../Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Node/StellaOps.Scanner.Analyzers.Lang.Node.csproj" />
<ProjectReference Include="../../Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/StellaOps.Scanner.Analyzers.Lang.Python.csproj" />
<ProjectReference Include="../../Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Ruby/StellaOps.Scanner.Analyzers.Lang.Ruby.csproj" />
<ProjectReference Include="../../Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Java/StellaOps.Scanner.Analyzers.Lang.Java.csproj" />
<ProjectReference Include="../../Scanner/__Libraries/StellaOps.Scanner.Surface.Env/StellaOps.Scanner.Surface.Env.csproj" />
<ProjectReference Include="../../Scanner/__Libraries/StellaOps.Scanner.Surface.Validation/StellaOps.Scanner.Surface.Validation.csproj" />
</ItemGroup>
<ItemGroup Condition="'$(StellaOpsEnableCryptoPro)' == 'true'">

View File

@@ -0,0 +1,6 @@
# CLI Guild — Active Tasks
| Task ID | State | Notes |
| --- | --- | --- |
| `SCANNER-CLI-0001` | DOING (2025-11-09) | Add Ruby-specific verbs/help, refresh docs & goldens per Sprint 138. |

View File

@@ -21,6 +21,11 @@ internal static class CliMetrics
private static readonly Counter<long> PolicyFindingsGetCounter = Meter.CreateCounter<long>("stellaops.cli.policy.findings.get.count");
private static readonly Counter<long> PolicyFindingsExplainCounter = Meter.CreateCounter<long>("stellaops.cli.policy.findings.explain.count");
private static readonly Counter<long> AdvisoryRunCounter = Meter.CreateCounter<long>("stellaops.cli.advisory.run.count");
private static readonly Counter<long> NodeLockValidateCounter = Meter.CreateCounter<long>("stellaops.cli.node.lock_validate.count");
private static readonly Counter<long> PythonLockValidateCounter = Meter.CreateCounter<long>("stellaops.cli.python.lock_validate.count");
private static readonly Counter<long> JavaLockValidateCounter = Meter.CreateCounter<long>("stellaops.cli.java.lock_validate.count");
private static readonly Counter<long> RubyInspectCounter = Meter.CreateCounter<long>("stellaops.cli.ruby.inspect.count");
private static readonly Counter<long> RubyResolveCounter = Meter.CreateCounter<long>("stellaops.cli.ruby.resolve.count");
private static readonly Histogram<double> CommandDurationHistogram = Meter.CreateHistogram<double>("stellaops.cli.command.duration.ms");
public static void RecordScannerDownload(string channel, bool fromCache)
@@ -108,6 +113,36 @@ internal static class CliMetrics
new("outcome", string.IsNullOrWhiteSpace(outcome) ? "unknown" : outcome)
});
public static void RecordNodeLockValidate(string outcome)
=> NodeLockValidateCounter.Add(1, new KeyValuePair<string, object?>[]
{
new("outcome", string.IsNullOrWhiteSpace(outcome) ? "unknown" : outcome)
});
public static void RecordPythonLockValidate(string outcome)
=> PythonLockValidateCounter.Add(1, new KeyValuePair<string, object?>[]
{
new("outcome", string.IsNullOrWhiteSpace(outcome) ? "unknown" : outcome)
});
public static void RecordJavaLockValidate(string outcome)
=> JavaLockValidateCounter.Add(1, new KeyValuePair<string, object?>[]
{
new("outcome", string.IsNullOrWhiteSpace(outcome) ? "unknown" : outcome)
});
public static void RecordRubyInspect(string outcome)
=> RubyInspectCounter.Add(1, new KeyValuePair<string, object?>[]
{
new("outcome", string.IsNullOrWhiteSpace(outcome) ? "unknown" : outcome)
});
public static void RecordRubyResolve(string outcome)
=> RubyResolveCounter.Add(1, new KeyValuePair<string, object?>[]
{
new("outcome", string.IsNullOrWhiteSpace(outcome) ? "unknown" : outcome)
});
public static IDisposable MeasureCommandDuration(string command)
{
var start = DateTime.UtcNow;

View File

@@ -0,0 +1,36 @@
using System;
using System.CommandLine;
using System.Linq;
using System.Threading;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using StellaOps.Cli.Commands;
using StellaOps.Cli.Configuration;
namespace StellaOps.Cli.Tests.Commands;
public sealed class CommandFactoryTests
{
[Fact]
public void Create_RegistersRubyInspectAndResolveCommands()
{
using var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Critical));
var services = new ServiceCollection().BuildServiceProvider();
var root = CommandFactory.Create(services, new StellaOpsCliOptions(), CancellationToken.None, loggerFactory);
var ruby = Assert.Single(root.Subcommands, command => string.Equals(command.Name, "ruby", StringComparison.Ordinal));
var inspect = Assert.Single(ruby.Subcommands, command => string.Equals(command.Name, "inspect", StringComparison.Ordinal));
var inspectOptions = inspect.Children.OfType<Option>().ToArray();
var inspectAliases = inspectOptions.SelectMany(option => option.Aliases).ToArray();
Assert.Contains("--root", inspectAliases, StringComparer.Ordinal);
Assert.Contains("--format", inspectAliases, StringComparer.Ordinal);
var resolve = Assert.Single(ruby.Subcommands, command => string.Equals(command.Name, "resolve", StringComparison.Ordinal));
var resolveOptions = resolve.Children.OfType<Option>().ToArray();
var resolveAliases = resolveOptions.SelectMany(option => option.Aliases).ToArray();
Assert.Contains("--image", resolveAliases, StringComparer.Ordinal);
Assert.Contains("--scan-id", resolveAliases, StringComparer.Ordinal);
Assert.Contains("--format", resolveAliases, StringComparer.Ordinal);
}
}