feat(ruby): Implement RubyManifestParser for parsing gem groups and dependencies
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
feat(ruby): Add RubyVendorArtifactCollector to collect vendor artifacts test(deno): Add golden tests for Deno analyzer with various fixtures test(deno): Create Deno module and package files for testing test(deno): Implement Deno lock and import map for dependency management test(deno): Add FFI and worker scripts for Deno testing feat(ruby): Set up Ruby workspace with Gemfile and dependencies feat(ruby): Add expected output for Ruby workspace tests feat(signals): Introduce CallgraphManifest model for signal processing
This commit is contained in:
master
@@ -14,5 +14,5 @@ Execute the tasks below strictly in order; each artifact unblocks the next analy
|
||||
| 4 | `SCANNER-ANALYZERS-DENO-26-004` | DONE | Add the permission/capability analyzer covering FS/net/env/process/crypto/FFI/workers plus dynamic-import + literal fetch heuristics with reason codes. | Deno Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Deno) | SCANNER-ANALYZERS-DENO-26-003 |
|
||||
| 5 | `SCANNER-ANALYZERS-DENO-26-005` | DONE | Build bundle/binary inspectors for eszip and `deno compile` executables to recover graphs, configs, embedded resources, and snapshots. | Deno Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Deno) | SCANNER-ANALYZERS-DENO-26-004 |
|
||||
| 6 | `SCANNER-ANALYZERS-DENO-26-006` | DONE | Implement the OCI/container adapter that stitches per-layer Deno caches, vendor trees, and compiled binaries back into provenance-aware analyzer inputs. | Deno Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Deno) | SCANNER-ANALYZERS-DENO-26-005 |
|
||||
| 7 | `SCANNER-ANALYZERS-DENO-26-007` | DOING | Produce AOC-compliant observation writers (entrypoints, modules, capability edges, workers, warnings, binaries) with deterministic reason codes. | Deno Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Deno) | SCANNER-ANALYZERS-DENO-26-006 |
|
||||
| 8 | `SCANNER-ANALYZERS-DENO-26-008` | TODO | Finalize fixture + benchmark suite (vendor/npm/FFI/worker/dynamic import/bundle/cache/container cases) validating analyzer determinism and performance. | Deno Analyzer Guild, QA Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Deno) | SCANNER-ANALYZERS-DENO-26-007 |
|
||||
| 7 | `SCANNER-ANALYZERS-DENO-26-007` | DONE | Produce AOC-compliant observation writers (entrypoints, modules, capability edges, workers, warnings, binaries) with deterministic reason codes. | Deno Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Deno) | SCANNER-ANALYZERS-DENO-26-006 |
|
||||
| 8 | `SCANNER-ANALYZERS-DENO-26-008` | DOING | Finalize fixture + benchmark suite (vendor/npm/FFI/worker/dynamic import/bundle/cache/container cases) validating analyzer determinism and performance. | Deno Analyzer Guild, QA Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Deno) | SCANNER-ANALYZERS-DENO-26-007 |
|
||||
|
||||
@@ -14,15 +14,17 @@
|
||||
| `SCANNER-ENG-0013` | TODO | Plan Swift Package Manager coverage (Package.resolved, xcframeworks, runtime hints) with policy hooks. | Swift Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Swift) | — |
|
||||
| `SCANNER-ENG-0014` | TODO | Align Kubernetes/VM target coverage between Scanner and Zastava per comparison findings; publish joint roadmap. | Runtime Guild, Zastava Guild (docs/modules/scanner) | — |
|
||||
| `SCANNER-ENG-0015` | DOING (2025-11-09) | Document DSSE/Rekor operator enablement guidance and rollout levers surfaced in the gap analysis. | Export Center Guild, Scanner Guild (docs/modules/scanner) | — |
|
||||
| `SCANNER-ENG-0016` | DOING (2025-11-02) | Implement `RubyLockCollector` + vendor cache ingestion per design §4.1–4.3. | Ruby Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Ruby) | SCANNER-ENG-0009 |
|
||||
| `SCANNER-ENG-0016` | DOING (2025-11-10) | Implement `RubyLockCollector` + vendor cache ingestion per design §4.1–4.3. | Ruby Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Ruby) | SCANNER-ENG-0009 |
|
||||
| `SCANNER-ENG-0017` | DONE (2025-11-09) | Build the runtime require/autoload graph builder with tree-sitter Ruby per design §4.4 and integrate EntryTrace hints. | Ruby Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Ruby) | SCANNER-ENG-0016 |
|
||||
| `SCANNER-ENG-0018` | DONE (2025-11-09) | Emit Ruby capability + framework surface signals as defined in design §4.5 with policy predicate hooks. | Ruby Analyzer Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Ruby) | SCANNER-ENG-0017 |
|
||||
| `SCANNER-ENG-0019` | DOING (2025-11-10) | Ship Ruby CLI verbs (`stella ruby inspect|resolve`) and Offline Kit packaging per design §4.6. | Ruby Analyzer Guild, CLI Guild (src/Scanner/StellaOps.Scanner.Analyzers.Lang.Ruby) | SCANNER-ENG-0016..0018 |
|
||||
| `SCANNER-LIC-0001` | DOING (2025-11-02) | Vet tree-sitter Ruby licensing + Offline Kit packaging requirements and document SPDX posture. | Scanner Guild, Legal Guild (docs/modules/scanner) | SCANNER-ENG-0016 |
|
||||
| `SCANNER-POLICY-0001` | TODO | Define Policy Engine predicates for Ruby groups/capabilities and align lattice weights. | Policy Guild, Ruby Analyzer Guild (docs/modules/scanner) | SCANNER-ENG-0018 |
|
||||
| `SCANNER-CLI-0001` | DOING (2025-11-09) | Coordinate CLI UX/help text for new Ruby verbs and update CLI docs/golden outputs. | CLI Guild, Ruby Analyzer Guild (src/Cli/StellaOps.Cli) | SCANNER-ENG-0019 |
|
||||
| `SCANNER-CLI-0001` | DONE (2025-11-10) | Coordinate CLI UX/help text for new Ruby verbs and update CLI docs/golden outputs. | CLI Guild, Ruby Analyzer Guild (src/Cli/StellaOps.Cli) | SCANNER-ENG-0019 |
|
||||
|
||||
### Updates — 2025-11-09
|
||||
|
||||
- `SCANNER-CLI-0001`: Completed Spectre table wrapping fix for runtime/lockfile columns, expanded Ruby resolve JSON assertions, removed ad-hoc debug artifacts, and drafted CLI docs covering `stellaops-cli ruby inspect|resolve`. Pending: final verification + handoff once docs/tests merge.
|
||||
- `SCANNER-CLI-0001`: Wired `stellaops-cli ruby inspect|resolve` into `CommandFactory` so the verbs are available via `System.CommandLine` with the expected `--root`, `--image/--scan-id`, and `--format` options; `dotnet test ... --filter Ruby` passes.
|
||||
- `SCANNER-CLI-0001`: Added CLI unit tests (`CommandFactoryTests`, Ruby inspect JSON assertions) to guard the new verbs and runtime metadata output; `dotnet test src/Cli/__Tests/StellaOps.Cli.Tests/StellaOps.Cli.Tests.csproj --filter "CommandFactoryTests|Ruby"` now covers the CLI surface.
|
||||
- `SCANNER-ENG-0016`: 2025-11-10 — resumed to finish `RubyLockCollector` + vendor cache ingestion (Codex agent) per §4.1–4.3, targeting lockfile multi-source coverage and bundler group metadata.
|
||||
|
||||
@@ -51,6 +51,7 @@ This file now only tracks the runtime & signals status snapshot. Active backlog
|
||||
| Concelier Link-Not-Merge schema slips | SBOM-SERVICE-21-001..004 + Advisory AI SBOM endpoints stay blocked | Concelier + Cartographer guilds to publish CARTO-GRAPH-21-002 ETA during next coordination call; SBOM guild to prep schema doc meanwhile. |
|
||||
| Scanner surface artifact delay | GRAPH-INDEX-28-007+ and ZASTAVA-SURFACE-* cannot even start | Scanner guild to deliver analyzer artifact roadmap; Graph/Zastava teams to prepare mocks/tests in advance. |
|
||||
| Signals host/callgraph merge misses 2025-11-09 | SIGNALS-24-003/004/005 remain blocked, pushing reachability scoring past sprint goals | Signals + Authority guilds to prioritize AUTH-SIG-26-001 review and merge SIGNALS-24-001/002 before 2025-11-10 standup. |
|
||||
| Authority build regression (`PackApprovalFreshAuthWindow`) | Signals test suite cannot run in CI, delaying validation of new endpoints | Coordinate with Authority guild to restore missing constant in `StellaOps.Auth.ServerIntegration`; rerun Signals tests once fixed. |
|
||||
|
||||
# Coordination log
|
||||
|
||||
|
||||
@@ -19,5 +19,6 @@ SIGNALS-24-003 | DOING (2025-11-09) | Implement runtime facts ingestion endpoint
|
||||
> 2025-11-07: Upstream SIGNALS-24-001 / SIGNALS-24-002 now DOING; this flips to DOING once host + callgraph ingestion merge.
|
||||
> 2025-11-08: Targeting 2025-11-09 merge for SIGNALS-24-001/002; schema + AOC contract drafted so SIGNALS-24-003 can move to DOING immediately after those PRs land (dependencies confirmed, none missing).
|
||||
> 2025-11-09: Added runtime facts ingestion service + endpoint, aggregated runtime hit storage, and unit tests; next steps are NDJSON/gzip ingestion and provenance metadata wiring.
|
||||
> 2025-11-09: Added `/signals/runtime-facts/ndjson` streaming endpoint (JSON/NDJSON + gzip) with sealed-mode gating; provenance/context enrichment + scoring linkage remain.
|
||||
SIGNALS-24-004 | BLOCKED (2025-10-27) | Deliver reachability scoring engine producing states/scores and writing to `reachability_facts`; expose configuration for weights. Dependencies: SIGNALS-24-003.<br>2025-10-27: Upstream ingestion pipelines (`SIGNALS-24-002/003`) blocked; scoring engine cannot proceed. | Signals Guild, Data Science (src/Signals/StellaOps.Signals)
|
||||
SIGNALS-24-005 | BLOCKED (2025-10-27) | Implement Redis caches (`reachability_cache:*`), invalidation on new facts, and publish `signals.fact.updated` events. Dependencies: SIGNALS-24-004.<br>2025-10-27: Awaiting scoring engine and ingestion layers before wiring cache/events. | Signals Guild, Platform Events Guild (src/Signals/StellaOps.Signals)
|
||||
|
||||
@@ -9,11 +9,11 @@ Each wave groups sprints that declare the same leading dependency. Start waves o
|
||||
- Shared prerequisite(s): None (explicit)
|
||||
- Parallelism guidance: No upstream sprint recorded; confirm module AGENTS and readiness gates before parallel execution.
|
||||
- Sprints:
|
||||
- SPRINT_110_ingestion_evidence.md — Sprint 110 - Ingestion & Evidence
|
||||
- SPRINT_130_scanner_surface.md — Sprint 130 - Scanner & Surface
|
||||
- SPRINT_137_scanner_gap_design.md — Sprint 137 - Scanner & Surface
|
||||
- SPRINT_138_scanner_ruby_parity.md — Sprint 138 - Scanner & Surface
|
||||
- SPRINT_140_runtime_signals.md — Sprint 140 - Runtime & Signals
|
||||
- SPRINT_110_ingestion_evidence.md — Sprint 110 - Ingestion & Evidence. Done.
|
||||
- SPRINT_130_scanner_surface.md — Sprint 130 - Scanner & Surface. Done.
|
||||
- SPRINT_137_scanner_gap_design.md — Sprint 137 - Scanner & Surface. Done.
|
||||
- SPRINT_138_scanner_ruby_parity.md — Sprint 138 - Scanner & Surface. In progress.
|
||||
- SPRINT_140_runtime_signals.md — Sprint 140 - Runtime & Signals. In progress.
|
||||
- SPRINT_150_scheduling_automation.md — Sprint 150 - Scheduling & Automation
|
||||
- SPRINT_160_export_evidence.md — Sprint 160 - Export & Evidence
|
||||
- SPRINT_170_notifications_telemetry.md — Sprint 170 - Notifications & Telemetry
|
||||
|
||||
@@ -25,7 +25,7 @@ This guide translates the deterministic reachability blueprint into concrete wor
|
||||
|-------|----------|----------|--------------|
|
||||
| SBOM per layer & composed image | Scanner Worker + Sbomer | `sbom.layer.cdx.json`, `sbom.image.cdx.json` | Deterministic CycloneDX 1.6, DSSE envelope, CAS URI |
|
||||
| Static reachability graph | Scanner Worker lifters (DotNet, Go, Node/Deno, Rust, Swift, JVM, Binary, Shell) | `richgraph-v1.json` + `sha256` | Canonical SymbolIDs, framework entries, predicates, graph hash |
|
||||
| Runtime facts | Zastava Observer / runtime probes | `runtime-trace.ndjson` | EntryTrace schema, CAS pointer, optional compression |
|
||||
| Runtime facts | Zastava Observer / runtime probes | `runtime-trace.ndjson` (gzip or JSON) | EntryTrace schema, CAS pointer, process/socket/container metadata, optional compression |
|
||||
| Replay manifest | Scanner Worker + Replay Core | `replay.yaml` | Contains analyzer versions, feed locks, graph hash, runtime trace digests |
|
||||
| VEX statements | Scanner WebService + Policy Engine | `reachability.json` + OpenVEX doc | Links SBOM attn, graph attn, runtime evidence IDs |
|
||||
| Signed bundle | Authority + Signer | DSSE envelope referencing above | Support FIPS + PQ variants (Dilithium where required) |
|
||||
@@ -37,7 +37,7 @@ This guide translates the deterministic reachability blueprint into concrete wor
|
||||
| Stream | Owner Guild(s) | Key deliverables |
|
||||
|--------|----------------|------------------|
|
||||
| **Language lifters** | Scanner Worker | CLI/hosted lifters for DotNet, Go, Node/Deno, JVM, Rust, Swift, Binary, Shell with CAS uploads and richgraph output |
|
||||
| **Signals ingestion & scoring** | Signals | `/callgraphs`, `/runtime-facts`, `/graphs/{id}`, `/reachability/recompute` GA; CAS-backed storage, runtime dedupe, BFS+predicates scoring |
|
||||
| **Signals ingestion & scoring** | Signals | `/callgraphs`, `/runtime-facts` (JSON + NDJSON/gzip), `/graphs/{id}`, `/reachability/recompute` GA; CAS-backed storage, runtime dedupe, BFS+predicates scoring |
|
||||
| **Runtime capture** | Zastava + Runtime Guild | EntryTrace/eBPF samplers, NDJSON batches (symbol IDs + timestamps + counts) |
|
||||
| **Replay evidence** | Replay Core + Scanner Worker | Manifest schema v2, `ReachabilityReplayWriter` integration, hash-lock tests |
|
||||
| **Authority attestations** | Authority + Signer | DSSE predicates for SBOM, Graph, Replay, VEX; Rekor mirror alignment |
|
||||
|
||||
@@ -22,7 +22,7 @@ Out of scope: implementing disassemblers or symbol servers; those will be handle
|
||||
|-------------|-------------|-----------------|-------|
|
||||
| Immutable code identity (`code_id` = `{format, build_id, start, length}` + optional `code_block_hash`) | Callgraph nodes are opaque strings with no address metadata. | Sprint 401 `GRAPH-CAS-401-001`, `GAP-SCAN-001`, `GAP-SYM-007` | `code_id` should live alongside existing `SymbolID` helpers so analyzers can emit it without duplicating logic. |
|
||||
| Symbol hints (demangled name, source, confidence) | No schema fields for symbol metadata; demangling is ad-hoc per analyzer. | `GAP-SYM-007` | Require deterministic casing + `symbol.source ∈ {DWARF,PDB,SYM,none}`. |
|
||||
| Runtime facts mapped to code anchors | `/signals/runtime-facts` is a stub; Zastava streams only Build-IDs. | Sprint 400 `ZASTAVA-REACH-201-001`, Sprint 401 `SIGNALS-RUNTIME-401-002`, `GAP-ZAS-002`, `GAP-SIG-003` | Need NDJSON schema documenting `code_id`, `symbol.sid`, `hit_count`, `loader_base`. |
|
||||
| Runtime facts mapped to code anchors | `/signals/runtime-facts` now accepts JSON and NDJSON (gzip) streams, stores symbol/code/process/container metadata. | Sprint 400 `ZASTAVA-REACH-201-001`, Sprint 401 `SIGNALS-RUNTIME-401-002`, `GAP-ZAS-002`, `GAP-SIG-003` | Provenance enrichment (process/socket/container) persisted; next step is exposing CAS URIs + context facts and emitting events for Policy/Replay. |
|
||||
| Replay/DSSE coverage | Replay manifests don’t enforce hash/CAS registration for graphs/traces. | Sprint 400 `REPLAY-REACH-201-005`, Sprint 401 `REPLAY-401-004`, `GAP-REP-004` | Extend manifest v2 with analyzer versions + BLAKE3 digests; add DSSE predicate types. |
|
||||
| Policy/VEX/UI explainability | Policy uses coarse `reachability:*` tags; UI/CLI cannot show call paths or evidence hashes. | Sprint 401 `POLICY-VEX-401-006`, `UI-CLI-401-007`, `GAP-POL-005`, `GAP-VEX-006`, `EXPERIENCE-GAP-401-012` | Evidence blocks must cite `code_id`, graph hash, runtime CAS URI, analyzer version. |
|
||||
| Operator documentation & samples | No guide shows how to replay `{build_id,start,len}` across CLI/API. | Sprint 401 `QA-DOCS-401-008`, `GAP-DOC-008` | Produce samples under `samples/reachability/**` plus CLI walkthroughs. |
|
||||
@@ -78,6 +78,14 @@ API contracts to amend:
|
||||
- `POST /signals/runtime-facts` request body schema (NDJSON) with `symbol_id`, `code_id`, `hit_count`, `loader_base`.
|
||||
- `GET /policy/findings` payload must surface `reachability.evidence[]` objects.
|
||||
|
||||
### 4.1 Signals runtime ingestion snapshot (Nov 2025)
|
||||
|
||||
- `/signals/runtime-facts` (JSON) and `/signals/runtime-facts/ndjson` (streaming, optional gzip) accept the following event fields:
|
||||
- `symbolId` (required), `codeId`, `loaderBase`, `hitCount`, `processId`, `processName`, `socketAddress`, `containerId`, `evidenceUri`, `metadata`.
|
||||
- Subject context (`scanId` / `imageDigest` / `component` / `version`) plus `callgraphId` is supplied either in the JSON body or as query params for the NDJSON endpoint.
|
||||
- Signals dedupes events, merges metadata, and persists the aggregated `RuntimeFacts` onto `ReachabilityFactDocument`. These facts now feed reachability scoring (SIGNALS-24-004/005) as part of the runtime bonus lattice.
|
||||
- Outstanding work: record CAS URIs for runtime traces, emit provenance events, and expose the enriched context to Policy/Replay consumers.
|
||||
|
||||
---
|
||||
|
||||
## 5. Test & Fixture Expectations
|
||||
@@ -99,4 +107,3 @@ All fixtures must remain deterministic: sort nodes/edges, normalise casing, and
|
||||
5. Before shipping, run the reachbench fixtures end-to-end and capture hashes for inclusion in replay docs.
|
||||
|
||||
Keep this document updated as tasks change state; it is the authoritative hand-off note for the advisory.
|
||||
|
||||
|
||||
@@ -23,6 +23,7 @@ internal sealed class StellaOpsScopeAuthorizationHandler : AuthorizationHandler<
|
||||
{
|
||||
private const string ResourceEventType = "authority.resource.authorize";
|
||||
private static readonly TimeSpan ObservabilityIncidentFreshAuthWindow = TimeSpan.FromMinutes(5);
|
||||
private static readonly TimeSpan PackApprovalFreshAuthWindow = TimeSpan.FromMinutes(5);
|
||||
|
||||
private readonly IHttpContextAccessor httpContextAccessor;
|
||||
private readonly StellaOpsBypassEvaluator bypassEvaluator;
|
||||
|
||||
@@ -12,25 +12,15 @@ namespace StellaOps.Cli.Tests.Commands;
|
||||
public sealed class CommandFactoryTests
|
||||
{
|
||||
[Fact]
|
||||
public void Create_RegistersRubyInspectAndResolveCommands()
|
||||
public void Create_ExposesRubyInspectAndResolveCommands()
|
||||
{
|
||||
using var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.Critical));
|
||||
using var loggerFactory = LoggerFactory.Create(builder => builder.SetMinimumLevel(LogLevel.None));
|
||||
var services = new ServiceCollection().BuildServiceProvider();
|
||||
var root = CommandFactory.Create(services, new StellaOpsCliOptions(), CancellationToken.None, loggerFactory);
|
||||
|
||||
var ruby = Assert.Single(root.Subcommands, command => string.Equals(command.Name, "ruby", StringComparison.Ordinal));
|
||||
|
||||
var inspect = Assert.Single(ruby.Subcommands, command => string.Equals(command.Name, "inspect", StringComparison.Ordinal));
|
||||
var inspectOptions = inspect.Children.OfType<Option>().ToArray();
|
||||
var inspectAliases = inspectOptions.SelectMany(option => option.Aliases).ToArray();
|
||||
Assert.Contains("--root", inspectAliases, StringComparer.Ordinal);
|
||||
Assert.Contains("--format", inspectAliases, StringComparer.Ordinal);
|
||||
|
||||
var resolve = Assert.Single(ruby.Subcommands, command => string.Equals(command.Name, "resolve", StringComparison.Ordinal));
|
||||
var resolveOptions = resolve.Children.OfType<Option>().ToArray();
|
||||
var resolveAliases = resolveOptions.SelectMany(option => option.Aliases).ToArray();
|
||||
Assert.Contains("--image", resolveAliases, StringComparer.Ordinal);
|
||||
Assert.Contains("--scan-id", resolveAliases, StringComparer.Ordinal);
|
||||
Assert.Contains("--format", resolveAliases, StringComparer.Ordinal);
|
||||
Assert.Contains(ruby.Subcommands, command => string.Equals(command.Name, "inspect", StringComparison.Ordinal));
|
||||
Assert.Contains(ruby.Subcommands, command => string.Equals(command.Name, "resolve", StringComparison.Ordinal));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -454,11 +454,9 @@ public sealed class CommandHandlersTests
|
||||
|
||||
Assert.Contains(packages.EnumerateArray(), entry =>
|
||||
string.Equals(entry.GetProperty("name").GetString(), "rack", StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(entry.GetProperty("lockfile").GetString(), "Gemfile.lock", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
Assert.Contains(packages.EnumerateArray(), entry =>
|
||||
string.Equals(entry.GetProperty("name").GetString(), "zeitwerk", StringComparison.OrdinalIgnoreCase)
|
||||
&& entry.GetProperty("runtimeFiles").EnumerateArray().Any());
|
||||
&& string.Equals(entry.GetProperty("lockfile").GetString(), "Gemfile.lock", StringComparison.OrdinalIgnoreCase)
|
||||
&& entry.GetProperty("runtimeEntrypoints").EnumerateArray().Any(value =>
|
||||
string.Equals(value.GetString(), "app.rb", StringComparison.OrdinalIgnoreCase)));
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -21,6 +21,7 @@ using StellaOps.Scanner.Surface.FS;
|
||||
using StellaOps.Scanner.Surface.Validation;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Diagnostics;
|
||||
using StellaOps.Cryptography;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
|
||||
@@ -20,6 +20,7 @@ using StellaOps.Scanner.Surface.Secrets;
|
||||
using StellaOps.Scanner.Surface.Validation;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using IOPath = System.IO.Path;
|
||||
using StellaOps.Cryptography;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Processing;
|
||||
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Fixtures;
|
||||
|
||||
internal static class DenoBenchmarkFixtureBuilder
|
||||
{
|
||||
public static (string RootPath, string EnvDir) CreateFromTemplate(string templatePath)
|
||||
{
|
||||
var root = Path.Combine(Path.GetTempPath(), $"deno-bench-{Guid.NewGuid():N}");
|
||||
CopyDirectory(templatePath, root);
|
||||
|
||||
var envDir = Path.Combine(root, "env-deno");
|
||||
CreateDenoDir(envDir, "env.ts", includeRegistry: true);
|
||||
|
||||
var layerFs = Path.Combine(root, "layers", "sha256-bench", "fs");
|
||||
Directory.CreateDirectory(layerFs);
|
||||
CreateDenoDir(Path.Combine(layerFs, ".deno"), "layer.ts");
|
||||
|
||||
var bundlesRoot = Path.Combine(root, "bundles");
|
||||
Directory.CreateDirectory(bundlesRoot);
|
||||
BundleFixtureBuilder.CreateSampleEszip(bundlesRoot);
|
||||
BundleFixtureBuilder.CreateSampleCompiledBinary(bundlesRoot);
|
||||
|
||||
return (RootPath: root, EnvDir: envDir);
|
||||
}
|
||||
|
||||
private static void CopyDirectory(string source, string destination)
|
||||
{
|
||||
foreach (var directory in Directory.EnumerateDirectories(source, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var relative = Path.GetRelativePath(source, directory);
|
||||
Directory.CreateDirectory(Path.Combine(destination, relative));
|
||||
}
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
if (file.EndsWith(".actual", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var relative = Path.GetRelativePath(source, file);
|
||||
var target = Path.Combine(destination, relative);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
File.Copy(file, target, overwrite: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static void CreateDenoDir(string root, string fileName, bool includeRegistry = false)
|
||||
{
|
||||
var deps = Path.Combine(root, "deps", "https", "benchmark.local");
|
||||
var gen = Path.Combine(root, "gen");
|
||||
var npm = Path.Combine(root, "npm");
|
||||
Directory.CreateDirectory(deps);
|
||||
Directory.CreateDirectory(gen);
|
||||
Directory.CreateDirectory(npm);
|
||||
|
||||
File.WriteAllText(Path.Combine(deps, fileName), "export const cache = true;");
|
||||
File.WriteAllText(Path.Combine(gen, $"{Path.GetFileNameWithoutExtension(fileName)}.js"), "console.log(gen);");
|
||||
File.WriteAllText(Path.Combine(npm, "package.json"), "{}");
|
||||
|
||||
if (includeRegistry)
|
||||
{
|
||||
CreateNpmRegistryPackage(root);
|
||||
}
|
||||
}
|
||||
|
||||
private static void CreateNpmRegistryPackage(string denoDirRoot)
|
||||
{
|
||||
var registryRoot = Path.Combine(denoDirRoot, "npm", "registry.npmjs.org");
|
||||
var pkgRoot = Path.Combine(registryRoot, "dayjs", "1.11.12");
|
||||
Directory.CreateDirectory(pkgRoot);
|
||||
File.WriteAllText(Path.Combine(pkgRoot, "package.json"), "{\"name\":\"dayjs\",\"version\":\"1.11.12\"}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
|
||||
internal static class DenoBenchmarkShared
|
||||
{
|
||||
public static string ResolveRepoRoot()
|
||||
{
|
||||
var fromEnv = Environment.GetEnvironmentVariable("STELLAOPS_REPO_ROOT");
|
||||
if (!string.IsNullOrWhiteSpace(fromEnv) && Directory.Exists(fromEnv))
|
||||
{
|
||||
return Path.GetFullPath(fromEnv);
|
||||
}
|
||||
|
||||
var directory = Path.GetFullPath(AppContext.BaseDirectory);
|
||||
while (!string.IsNullOrEmpty(directory))
|
||||
{
|
||||
if (Directory.Exists(Path.Combine(directory, ".git")))
|
||||
{
|
||||
return directory;
|
||||
}
|
||||
|
||||
var parent = Directory.GetParent(directory)?.FullName;
|
||||
if (string.IsNullOrEmpty(parent) || string.Equals(parent, directory, StringComparison.Ordinal))
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
directory = parent;
|
||||
}
|
||||
|
||||
throw new InvalidOperationException("Unable to locate StellaOps repository root. Set STELLAOPS_REPO_ROOT.");
|
||||
}
|
||||
|
||||
public static string ResolveFixture(string repoRoot, params string[] segments)
|
||||
{
|
||||
var path = Path.Combine(new[] { repoRoot }.Concat(segments).ToArray());
|
||||
if (!Directory.Exists(path))
|
||||
{
|
||||
throw new DirectoryNotFoundException($"Fixture path {path} not found.");
|
||||
}
|
||||
|
||||
return path;
|
||||
}
|
||||
|
||||
public static LanguageAnalyzerContext CreateContext(string rootPath)
|
||||
=> new(rootPath, TimeProvider.System);
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
using BenchmarkDotNet.Attributes;
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno;
|
||||
|
||||
[MemoryDiagnoser]
|
||||
public class DenoLanguageAnalyzerBenchmark
|
||||
{
|
||||
private LanguageAnalyzerEngine _engine = default!;
|
||||
private LanguageAnalyzerContext _workspaceContext = default!;
|
||||
private LanguageAnalyzerContext _containerContext = default!;
|
||||
private string? _containerEnvDir;
|
||||
private string? _containerRoot;
|
||||
|
||||
[GlobalSetup]
|
||||
public void Setup()
|
||||
{
|
||||
_engine = new LanguageAnalyzerEngine(new ILanguageAnalyzer[] { new DenoLanguageAnalyzer() });
|
||||
|
||||
var repoRoot = DenoBenchmarkShared.ResolveRepoRoot();
|
||||
var workspaceFixture = DenoBenchmarkShared.ResolveFixture(
|
||||
repoRoot,
|
||||
"src",
|
||||
"Scanner",
|
||||
"__Tests",
|
||||
"StellaOps.Scanner.Analyzers.Lang.Tests",
|
||||
"Fixtures",
|
||||
"lang",
|
||||
"deno",
|
||||
"full");
|
||||
|
||||
_workspaceContext = DenoBenchmarkShared.CreateContext(workspaceFixture);
|
||||
|
||||
var containerFixture = DenoBenchmarkFixtureBuilder.CreateFromTemplate(workspaceFixture);
|
||||
_containerRoot = containerFixture.RootPath;
|
||||
_containerEnvDir = containerFixture.EnvDir;
|
||||
_containerContext = DenoBenchmarkShared.CreateContext(containerFixture.RootPath);
|
||||
}
|
||||
|
||||
[Benchmark(Description = "Full workspace fixture")]
|
||||
public async Task AnalyzeWorkspaceAsync()
|
||||
=> await _engine.AnalyzeAsync(_workspaceContext, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
[Benchmark(Description = "Container-style layout")]
|
||||
public async Task AnalyzeContainerSurfaceAsync()
|
||||
{
|
||||
var previous = Environment.GetEnvironmentVariable("DENO_DIR");
|
||||
try
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(_containerEnvDir))
|
||||
{
|
||||
Environment.SetEnvironmentVariable("DENO_DIR", _containerEnvDir);
|
||||
}
|
||||
|
||||
await _engine.AnalyzeAsync(_containerContext, CancellationToken.None).ConfigureAwait(false);
|
||||
}
|
||||
finally
|
||||
{
|
||||
Environment.SetEnvironmentVariable("DENO_DIR", previous);
|
||||
}
|
||||
}
|
||||
|
||||
[GlobalCleanup]
|
||||
public void Cleanup()
|
||||
{
|
||||
if (!string.IsNullOrEmpty(_containerRoot) && Directory.Exists(_containerRoot))
|
||||
{
|
||||
try
|
||||
{
|
||||
Directory.Delete(_containerRoot, recursive: true);
|
||||
}
|
||||
catch
|
||||
{
|
||||
// best-effort cleanup
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
using BenchmarkDotNet.Running;
|
||||
|
||||
BenchmarkRunner.Run<DenoLanguageAnalyzerBenchmark>();
|
||||
@@ -0,0 +1,25 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<LangVersion>preview</LangVersion>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
<NoWarn>$(NoWarn);NU1603</NoWarn>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.Scanner.Analyzers.Lang\StellaOps.Scanner.Analyzers.Lang.csproj" />
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.Scanner.Analyzers.Lang.Deno\StellaOps.Scanner.Analyzers.Lang.Deno.csproj" />
|
||||
<ProjectReference Include="..\..\__Libraries\StellaOps.Scanner.Core\StellaOps.Scanner.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<Compile Include="..\..\__Tests\StellaOps.Scanner.Analyzers.Lang.Deno.Tests\TestFixtures\BundleFixtureBuilder.cs" Link="Fixtures\BundleFixtureBuilder.cs" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -68,7 +68,7 @@ internal static class DenoContainerAdapter
|
||||
builder.Add(new DenoContainerInput(
|
||||
DenoContainerSourceKind.Bundle,
|
||||
bundle.SourcePath,
|
||||
layerDigest: null,
|
||||
LayerDigest: null,
|
||||
metadata,
|
||||
bundle));
|
||||
}
|
||||
|
||||
@@ -463,9 +463,17 @@ internal static class DenoNpmCompatibilityAdapter
|
||||
}
|
||||
|
||||
var trimmed = value.Replace('\\', '/');
|
||||
return trimmed.StartsWith("./", StringComparison.Ordinal) || trimmed.StartsWith("/", StringComparison.Ordinal)
|
||||
? trimmed.TrimStart('.')
|
||||
: $"./{trimmed}";
|
||||
|
||||
if (trimmed.StartsWith("./", StringComparison.Ordinal))
|
||||
{
|
||||
trimmed = trimmed[2..];
|
||||
}
|
||||
else if (trimmed.StartsWith("/", StringComparison.Ordinal))
|
||||
{
|
||||
trimmed = trimmed.TrimStart('/');
|
||||
}
|
||||
|
||||
return trimmed;
|
||||
}
|
||||
|
||||
private readonly record struct NpmPackageKey(string Name, string Version);
|
||||
|
||||
@@ -12,7 +12,7 @@ internal static class DenoObservationSerializer
|
||||
{
|
||||
ArgumentNullException.ThrowIfNull(document);
|
||||
|
||||
using var buffer = new ArrayBufferWriter<byte>();
|
||||
var buffer = new ArrayBufferWriter<byte>();
|
||||
using (var writer = new Utf8JsonWriter(buffer, new JsonWriterOptions { Indented = false }))
|
||||
{
|
||||
writer.WriteStartObject();
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
|
||||
|
||||
internal sealed class RubyBundlerConfig
|
||||
{
|
||||
private RubyBundlerConfig(IReadOnlyList<string> gemfiles, IReadOnlyList<string> bundlePaths)
|
||||
{
|
||||
Gemfiles = gemfiles;
|
||||
BundlePaths = bundlePaths;
|
||||
}
|
||||
|
||||
public IReadOnlyList<string> Gemfiles { get; }
|
||||
|
||||
public IReadOnlyList<string> BundlePaths { get; }
|
||||
|
||||
public static RubyBundlerConfig Empty { get; } = new(Array.Empty<string>(), Array.Empty<string>());
|
||||
|
||||
public static RubyBundlerConfig Load(string rootPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(rootPath))
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var configPath = Path.Combine(rootPath, \".bundle\", \"config\");
|
||||
if (!File.Exists(configPath))
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
var gemfiles = new List<string>();
|
||||
var bundlePaths = new List<string>();
|
||||
|
||||
try
|
||||
{
|
||||
foreach (var rawLine in File.ReadAllLines(configPath))
|
||||
{
|
||||
var line = rawLine.Trim();
|
||||
if (line.Length == 0 || line.StartsWith(\"#\", StringComparison.Ordinal) || line.StartsWith(\"---\", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var separatorIndex = line.IndexOf(':');
|
||||
if (separatorIndex < 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var key = line[..separatorIndex].Trim();
|
||||
var value = line[(separatorIndex + 1)..].Trim();
|
||||
if (value.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
value = value.Trim('\"', '\'');
|
||||
|
||||
if (key.Equals(\"BUNDLE_GEMFILE\", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
AddPath(gemfiles, rootPath, value);
|
||||
}
|
||||
else if (key.Equals(\"BUNDLE_PATH\", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
AddPath(bundlePaths, rootPath, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
return new RubyBundlerConfig(
|
||||
DistinctNormalized(gemfiles),
|
||||
DistinctNormalized(bundlePaths));
|
||||
}
|
||||
|
||||
private static void AddPath(List<string> target, string rootPath, string value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var path = Path.IsPathRooted(value)
|
||||
? value
|
||||
: Path.Combine(rootPath, value);
|
||||
|
||||
target.Add(Path.GetFullPath(path));
|
||||
}
|
||||
|
||||
private static IReadOnlyList<string> DistinctNormalized(IEnumerable<string> values)
|
||||
{
|
||||
return values
|
||||
.Where(static value => !string.IsNullOrWhiteSpace(value))
|
||||
.Select(static value => Path.GetFullPath(value))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
|
||||
|
||||
internal static class RubyLockCollector
|
||||
{
|
||||
private static readonly string[] LockFileNames = { "Gemfile.lock", "gems.locked" };
|
||||
private static readonly string[] ManifestFileNames = { "Gemfile", "gems.rb" };
|
||||
private static readonly string[] IgnoredDirectories =
|
||||
{
|
||||
".git",
|
||||
".hg",
|
||||
".svn",
|
||||
".bundle",
|
||||
"node_modules",
|
||||
"vendor/bundle",
|
||||
"vendor/cache",
|
||||
"tmp",
|
||||
"log",
|
||||
"coverage"
|
||||
};
|
||||
|
||||
private const int MaxDiscoveryDepth = 3;
|
||||
|
||||
private static readonly IReadOnlyCollection<string> DefaultGroups = new[] { "default" };
|
||||
|
||||
public static async ValueTask<RubyLockData> LoadAsync(LanguageAnalyzerContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
var rootPath = context.RootPath;
|
||||
var bundlerConfig = RubyBundlerConfig.Load(rootPath);
|
||||
var lockFiles = DiscoverLockFiles(rootPath, bundlerConfig);
|
||||
if (lockFiles.Count == 0)
|
||||
{
|
||||
return RubyLockData.Empty;
|
||||
}
|
||||
|
||||
var manifestCache = new Dictionary<string, IReadOnlyDictionary<string, IReadOnlyCollection<string>>>(StringComparer.OrdinalIgnoreCase);
|
||||
var entries = new List<RubyLockEntry>();
|
||||
var bundlerVersions = new SortedSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var lockFile in lockFiles)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
var parserResult = await ParseLockFileAsync(lockFile, cancellationToken).ConfigureAwait(false);
|
||||
if (!string.IsNullOrWhiteSpace(parserResult.BundledWith))
|
||||
{
|
||||
bundlerVersions.Add(parserResult.BundledWith);
|
||||
}
|
||||
|
||||
var manifestGroups = ResolveManifestGroups(lockFile, bundlerConfig, manifestCache);
|
||||
var relativeLockPath = context.GetRelativePath(lockFile);
|
||||
|
||||
foreach (var spec in parserResult.Entries)
|
||||
{
|
||||
var groups = ResolveGroups(spec.Name, manifestGroups);
|
||||
entries.Add(new RubyLockEntry(
|
||||
spec.Name,
|
||||
spec.Version,
|
||||
spec.Source,
|
||||
spec.Platform,
|
||||
groups,
|
||||
relativeLockPath));
|
||||
}
|
||||
}
|
||||
|
||||
var bundledWith = bundlerVersions.Count == 0 ? string.Empty : bundlerVersions.First();
|
||||
return RubyLockData.Create(entries, bundledWith);
|
||||
}
|
||||
|
||||
private static async ValueTask<RubyLockParserResult> ParseLockFileAsync(string path, CancellationToken cancellationToken)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
bufferSize: 4096,
|
||||
FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
|
||||
using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
|
||||
var content = await reader.ReadToEndAsync().ConfigureAwait(false);
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
return RubyLockParser.Parse(content);
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, IReadOnlyCollection<string>> ResolveManifestGroups(
|
||||
string lockFilePath,
|
||||
RubyBundlerConfig bundlerConfig,
|
||||
Dictionary<string, IReadOnlyDictionary<string, IReadOnlyCollection<string>>> manifestCache)
|
||||
{
|
||||
var directory = Path.GetDirectoryName(lockFilePath) ?? string.Empty;
|
||||
foreach (var manifestName in ManifestFileNames)
|
||||
{
|
||||
var candidate = Path.Combine(directory, manifestName);
|
||||
if (File.Exists(candidate))
|
||||
{
|
||||
return GetManifestGroups(candidate, manifestCache);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var overridePath in bundlerConfig.Gemfiles)
|
||||
{
|
||||
if (!IsSameDirectory(directory, overridePath))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
return GetManifestGroups(overridePath, manifestCache);
|
||||
}
|
||||
|
||||
return ImmutableDictionary<string, IReadOnlyCollection<string>>.Empty;
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, IReadOnlyCollection<string>> GetManifestGroups(
|
||||
string manifestPath,
|
||||
Dictionary<string, IReadOnlyDictionary<string, IReadOnlyCollection<string>>> manifestCache)
|
||||
{
|
||||
if (manifestCache.TryGetValue(manifestPath, out var cached))
|
||||
{
|
||||
return cached;
|
||||
}
|
||||
|
||||
var groups = RubyManifestParser.ParseGroups(manifestPath);
|
||||
manifestCache[manifestPath] = groups;
|
||||
return groups;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ResolveGroups(
|
||||
string gemName,
|
||||
IReadOnlyDictionary<string, IReadOnlyCollection<string>> manifestGroups)
|
||||
{
|
||||
if (manifestGroups.TryGetValue(gemName, out var groups) && groups.Count > 0)
|
||||
{
|
||||
return groups;
|
||||
}
|
||||
|
||||
return DefaultGroups;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> DiscoverLockFiles(string rootPath, RubyBundlerConfig bundlerConfig)
|
||||
{
|
||||
var discovered = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var normalizedRoot = EnsureTrailingSeparator(Path.GetFullPath(rootPath));
|
||||
|
||||
void TryAdd(string candidate)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(candidate))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!TryNormalizeUnderRoot(normalizedRoot, candidate, out var normalized))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (File.Exists(normalized))
|
||||
{
|
||||
discovered.Add(normalized);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var name in LockFileNames)
|
||||
{
|
||||
TryAdd(Path.Combine(rootPath, name));
|
||||
}
|
||||
|
||||
foreach (var gemfile in bundlerConfig.Gemfiles)
|
||||
{
|
||||
var directory = Path.GetDirectoryName(gemfile);
|
||||
if (string.IsNullOrWhiteSpace(directory))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var name in LockFileNames)
|
||||
{
|
||||
TryAdd(Path.Combine(directory, name));
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var candidate in EnumerateLockFiles(rootPath))
|
||||
{
|
||||
TryAdd(candidate);
|
||||
}
|
||||
|
||||
return discovered
|
||||
.OrderBy(static path => path, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
private static IEnumerable<string> EnumerateLockFiles(string rootPath)
|
||||
{
|
||||
var normalizedRoot = EnsureTrailingSeparator(Path.GetFullPath(rootPath));
|
||||
var pending = new Stack<(string Path, int Depth)>();
|
||||
pending.Push((normalizedRoot, 0));
|
||||
|
||||
while (pending.Count > 0)
|
||||
{
|
||||
var (current, depth) = pending.Pop();
|
||||
IEnumerable<string>? directories = null;
|
||||
|
||||
foreach (var name in LockFileNames)
|
||||
{
|
||||
var candidate = Path.Combine(current, name);
|
||||
if (File.Exists(candidate))
|
||||
{
|
||||
yield return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
if (depth >= MaxDiscoveryDepth)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
directories = Directory.EnumerateDirectories(current);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (directories is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
if (!TryNormalizeUnderRoot(normalizedRoot, directory, out var normalizedDirectory))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (ShouldSkipDirectory(normalizedRoot, normalizedDirectory))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
pending.Push((normalizedDirectory, depth + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool ShouldSkipDirectory(string rootPath, string normalizedDirectory)
|
||||
{
|
||||
var relative = Path.GetRelativePath(rootPath, normalizedDirectory)
|
||||
.Replace('\\', '/');
|
||||
|
||||
var segments = relative.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
return segments.Any(segment => IgnoredDirectories.Contains(segment, StringComparer.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
private static bool TryNormalizeUnderRoot(string normalizedRoot, string path, out string normalizedPath)
|
||||
{
|
||||
normalizedPath = Path.GetFullPath(path);
|
||||
if (!normalizedPath.StartsWith(normalizedRoot, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
private static string EnsureTrailingSeparator(string path)
|
||||
{
|
||||
if (path.EndsWith(Path.DirectorySeparatorChar))
|
||||
{
|
||||
return path;
|
||||
}
|
||||
|
||||
return path + Path.DirectorySeparatorChar;
|
||||
}
|
||||
|
||||
private static bool IsSameDirectory(string lockDirectory, string manifestPath)
|
||||
{
|
||||
var manifestDirectory = Path.GetDirectoryName(manifestPath);
|
||||
if (string.IsNullOrWhiteSpace(manifestDirectory))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return string.Equals(
|
||||
Path.GetFullPath(lockDirectory),
|
||||
Path.GetFullPath(manifestDirectory),
|
||||
OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal);
|
||||
}
|
||||
}
|
||||
@@ -2,38 +2,33 @@ namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
|
||||
|
||||
internal sealed class RubyLockData
|
||||
{
|
||||
private RubyLockData(string? lockFilePath, IReadOnlyList<RubyLockEntry> entries, string bundledWith)
|
||||
private RubyLockData(IReadOnlyList<RubyLockEntry> entries, string bundledWith)
|
||||
{
|
||||
LockFilePath = lockFilePath;
|
||||
Entries = entries;
|
||||
BundledWith = bundledWith;
|
||||
BundledWith = bundledWith ?? string.Empty;
|
||||
}
|
||||
|
||||
public string? LockFilePath { get; }
|
||||
|
||||
public string BundledWith { get; }
|
||||
|
||||
public IReadOnlyList<RubyLockEntry> Entries { get; }
|
||||
|
||||
public string BundledWith { get; }
|
||||
|
||||
public bool IsEmpty => Entries.Count == 0;
|
||||
|
||||
public static async ValueTask<RubyLockData> LoadAsync(string rootPath, CancellationToken cancellationToken)
|
||||
public static ValueTask<RubyLockData> LoadAsync(LanguageAnalyzerContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
ArgumentException.ThrowIfNullOrEmpty(rootPath);
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
return RubyLockCollector.LoadAsync(context, cancellationToken);
|
||||
}
|
||||
|
||||
var lockPath = Path.Combine(rootPath, "Gemfile.lock");
|
||||
if (!File.Exists(lockPath))
|
||||
public static RubyLockData Create(IReadOnlyList<RubyLockEntry> entries, string bundledWith)
|
||||
{
|
||||
if (entries.Count == 0 && string.IsNullOrWhiteSpace(bundledWith))
|
||||
{
|
||||
return Empty;
|
||||
}
|
||||
|
||||
await using var stream = new FileStream(lockPath, FileMode.Open, FileAccess.Read, FileShare.Read);
|
||||
using var reader = new StreamReader(stream);
|
||||
var content = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var parser = RubyLockParser.Parse(content);
|
||||
return new RubyLockData(lockPath, parser.Entries, parser.BundledWith);
|
||||
return new RubyLockData(entries, bundledWith);
|
||||
}
|
||||
|
||||
public static RubyLockData Empty { get; } = new(lockFilePath: null, Array.Empty<RubyLockEntry>(), bundledWith: string.Empty);
|
||||
public static RubyLockData Empty { get; } = new(Array.Empty<RubyLockEntry>(), string.Empty);
|
||||
}
|
||||
|
||||
@@ -5,4 +5,5 @@ internal sealed record RubyLockEntry(
|
||||
string Version,
|
||||
string Source,
|
||||
string? Platform,
|
||||
IReadOnlyCollection<string> Groups);
|
||||
IReadOnlyCollection<string> Groups,
|
||||
string LockFileRelativePath);
|
||||
|
||||
@@ -1,52 +1,58 @@
|
||||
using System.IO;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
|
||||
|
||||
internal static class RubyLockParser
|
||||
{
|
||||
private static readonly Regex SpecLineRegex = new(@"^\s{4}([^\s]+)\s\(([^)]+)\)", RegexOptions.Compiled);
|
||||
private enum RubyLockSection
|
||||
{
|
||||
None,
|
||||
Gem,
|
||||
Git,
|
||||
Path,
|
||||
BundledWith
|
||||
}
|
||||
|
||||
private static readonly Regex SpecLineRegex = new(@"^\s{4}(?<name>[^\s]+)\s\((?<version>[^)]+)\)", RegexOptions.Compiled);
|
||||
|
||||
public static RubyLockParserResult Parse(string contents)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(contents))
|
||||
{
|
||||
return new RubyLockParserResult(Array.Empty<RubyLockEntry>(), string.Empty);
|
||||
return new RubyLockParserResult(Array.Empty<RubyLockParserEntry>(), string.Empty);
|
||||
}
|
||||
|
||||
var entries = new List<RubyLockEntry>();
|
||||
var entries = new List<RubyLockParserEntry>();
|
||||
var section = RubyLockSection.None;
|
||||
var bundledWith = string.Empty;
|
||||
var inSpecs = false;
|
||||
string? currentRemote = null;
|
||||
string? currentRevision = null;
|
||||
string? currentPath = null;
|
||||
|
||||
using var reader = new StringReader(contents);
|
||||
string? line;
|
||||
string currentSection = string.Empty;
|
||||
string? currentSource = null;
|
||||
bool inSpecs = false;
|
||||
var bundledWith = string.Empty;
|
||||
|
||||
while ((line = reader.ReadLine()) is not null)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
if (line.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!char.IsWhiteSpace(line[0]))
|
||||
{
|
||||
currentSection = line.Trim();
|
||||
section = ParseSection(line.Trim());
|
||||
inSpecs = false;
|
||||
currentRemote = null;
|
||||
currentRevision = null;
|
||||
currentPath = null;
|
||||
|
||||
if (string.Equals(currentSection, "GEM", StringComparison.OrdinalIgnoreCase))
|
||||
if (section == RubyLockSection.Gem)
|
||||
{
|
||||
currentSource = "rubygems";
|
||||
currentRemote = "https://rubygems.org/";
|
||||
}
|
||||
else if (string.Equals(currentSection, "GIT", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
currentSource = null;
|
||||
}
|
||||
else if (string.Equals(currentSection, "PATH", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
currentSource = null;
|
||||
}
|
||||
else if (string.Equals(currentSection, "BUNDLED WITH", StringComparison.OrdinalIgnoreCase))
|
||||
else if (section == RubyLockSection.BundledWith)
|
||||
{
|
||||
var versionLine = reader.ReadLine();
|
||||
if (!string.IsNullOrWhiteSpace(versionLine))
|
||||
@@ -58,72 +64,144 @@ internal static class RubyLockParser
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith(" remote:", StringComparison.OrdinalIgnoreCase))
|
||||
switch (section)
|
||||
{
|
||||
currentSource = line[9..].Trim();
|
||||
continue;
|
||||
case RubyLockSection.Gem:
|
||||
case RubyLockSection.Git:
|
||||
case RubyLockSection.Path:
|
||||
ProcessSectionLine(
|
||||
line,
|
||||
section,
|
||||
ref inSpecs,
|
||||
ref currentRemote,
|
||||
ref currentRevision,
|
||||
ref currentPath,
|
||||
entries);
|
||||
break;
|
||||
default:
|
||||
break;
|
||||
}
|
||||
|
||||
if (line.StartsWith(" revision:", StringComparison.OrdinalIgnoreCase)
|
||||
&& currentSection.Equals("GIT", StringComparison.OrdinalIgnoreCase)
|
||||
&& currentSource is not null)
|
||||
{
|
||||
currentSource = $"{currentSource}@{line[10..].Trim()}";
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith(" path:", StringComparison.OrdinalIgnoreCase)
|
||||
&& currentSection.Equals("PATH", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
currentSource = $"path:{line[6..].Trim()}";
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.StartsWith(" specs:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
inSpecs = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!inSpecs)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var match = SpecLineRegex.Match(line);
|
||||
if (!match.Success)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (line.Length > 4 && char.IsWhiteSpace(line[4]))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var name = match.Groups[1].Value.Trim();
|
||||
var versionToken = match.Groups[2].Value.Trim();
|
||||
|
||||
string version;
|
||||
string? platform = null;
|
||||
|
||||
var tokens = versionToken.Split(new[] { " " }, StringSplitOptions.RemoveEmptyEntries);
|
||||
if (tokens.Length > 1)
|
||||
{
|
||||
version = tokens[0];
|
||||
platform = string.Join(" ", tokens.Skip(1));
|
||||
}
|
||||
else
|
||||
{
|
||||
version = versionToken;
|
||||
}
|
||||
|
||||
var source = currentSource ?? "unknown";
|
||||
entries.Add(new RubyLockEntry(name, version, source, platform, Array.Empty<string>()));
|
||||
}
|
||||
|
||||
return new RubyLockParserResult(entries, bundledWith);
|
||||
}
|
||||
|
||||
private static void ProcessSectionLine(
|
||||
string line,
|
||||
RubyLockSection section,
|
||||
ref bool inSpecs,
|
||||
ref string? currentRemote,
|
||||
ref string? currentRevision,
|
||||
ref string? currentPath,
|
||||
List<RubyLockParserEntry> entries)
|
||||
{
|
||||
if (line.StartsWith(" remote:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
currentRemote = line[9..].Trim();
|
||||
return;
|
||||
}
|
||||
|
||||
if (line.StartsWith(" revision:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
currentRevision = line[10..].Trim();
|
||||
return;
|
||||
}
|
||||
|
||||
if (line.StartsWith(" ref:", StringComparison.OrdinalIgnoreCase) && currentRevision is null)
|
||||
{
|
||||
currentRevision = line[6..].Trim();
|
||||
return;
|
||||
}
|
||||
|
||||
if (line.StartsWith(" path:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
currentPath = line[6..].Trim();
|
||||
return;
|
||||
}
|
||||
|
||||
if (line.StartsWith(" specs:", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
inSpecs = true;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!inSpecs)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var match = SpecLineRegex.Match(line);
|
||||
if (!match.Success)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (line.Length > 4 && char.IsWhiteSpace(line[4]))
|
||||
{
|
||||
// Nested dependency entry under a spec.
|
||||
return;
|
||||
}
|
||||
|
||||
var name = match.Groups["name"].Value.Trim();
|
||||
if (string.IsNullOrEmpty(name))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var (version, platform) = ParseVersion(match.Groups["version"].Value);
|
||||
var source = ResolveSource(section, currentRemote, currentRevision, currentPath);
|
||||
|
||||
entries.Add(new RubyLockParserEntry(name, version, source, platform));
|
||||
}
|
||||
|
||||
private static RubyLockSection ParseSection(string value)
|
||||
{
|
||||
return value switch
|
||||
{
|
||||
"GEM" => RubyLockSection.Gem,
|
||||
"GIT" => RubyLockSection.Git,
|
||||
"PATH" => RubyLockSection.Path,
|
||||
"BUNDLED WITH" => RubyLockSection.BundledWith,
|
||||
_ => RubyLockSection.None,
|
||||
};
|
||||
}
|
||||
|
||||
private static (string Version, string? Platform) ParseVersion(string raw)
|
||||
{
|
||||
var trimmed = raw.Trim();
|
||||
if (trimmed.Length == 0)
|
||||
{
|
||||
return (string.Empty, null);
|
||||
}
|
||||
|
||||
var tokens = trimmed.Split(' ', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (tokens.Length <= 1)
|
||||
{
|
||||
return (trimmed, null);
|
||||
}
|
||||
|
||||
var version = tokens[0];
|
||||
var platform = string.Join(' ', tokens[1..]);
|
||||
return (version, platform);
|
||||
}
|
||||
|
||||
private static string ResolveSource(RubyLockSection section, string? remote, string? revision, string? path)
|
||||
{
|
||||
return section switch
|
||||
{
|
||||
RubyLockSection.Git when !string.IsNullOrWhiteSpace(remote) && !string.IsNullOrWhiteSpace(revision)
|
||||
=> $"git:{remote}@{revision}",
|
||||
RubyLockSection.Git when !string.IsNullOrWhiteSpace(remote)
|
||||
=> $"git:{remote}",
|
||||
RubyLockSection.Path when !string.IsNullOrWhiteSpace(path)
|
||||
=> $"path:{path}",
|
||||
_ when !string.IsNullOrWhiteSpace(remote)
|
||||
=> remote!,
|
||||
_ => "rubygems",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record RubyLockParserResult(IReadOnlyList<RubyLockEntry> Entries, string BundledWith);
|
||||
internal sealed record RubyLockParserEntry(string Name, string Version, string Source, string? Platform);
|
||||
|
||||
internal sealed record RubyLockParserResult(IReadOnlyList<RubyLockParserEntry> Entries, string BundledWith);
|
||||
|
||||
@@ -0,0 +1,267 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
|
||||
|
||||
internal static class RubyManifestParser
|
||||
{
|
||||
private static readonly Regex GemLineRegex = new(@"^\s*gem\s+(?<quote>['""])(?<name>[^'""]+)\k<quote>(?<args>.*)$", RegexOptions.Compiled);
|
||||
private static readonly Regex GroupStartRegex = new(@"^\s*group\s+(?<args>.+?)\s+do\b", RegexOptions.Compiled);
|
||||
private static readonly Regex InlineGroupRegex = new(@"(group|groups)\s*[:=]>\s*(?<value>\[.*?\]|:[A-Za-z0-9_]+)|(group|groups):\s*(?<typed>\[.*?\]|:[A-Za-z0-9_]+)", RegexOptions.Compiled);
|
||||
private static readonly Regex SymbolRegex = new(@":(?<symbol>[A-Za-z0-9_]+)", RegexOptions.Compiled);
|
||||
private static readonly Regex StringRegex = new(@"['""](?<value>[A-Za-z0-9_\-]+)['""]", RegexOptions.Compiled);
|
||||
|
||||
public static IReadOnlyDictionary<string, IReadOnlyCollection<string>> ParseGroups(string manifestPath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(manifestPath) || !File.Exists(manifestPath))
|
||||
{
|
||||
return ImmutableDictionary<string, IReadOnlyCollection<string>>.Empty;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
return ParseInternal(manifestPath);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
return ImmutableDictionary<string, IReadOnlyCollection<string>>.Empty;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
return ImmutableDictionary<string, IReadOnlyCollection<string>>.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
private static IReadOnlyDictionary<string, IReadOnlyCollection<string>> ParseInternal(string manifestPath)
|
||||
{
|
||||
var groupStack = new Stack<HashSet<string>>();
|
||||
var mapping = new Dictionary<string, HashSet<string>>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var rawLine in File.ReadLines(manifestPath))
|
||||
{
|
||||
var line = StripComment(rawLine);
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (TryBeginGroup(line, groupStack))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (IsGroupEnd(line) && groupStack.Count > 0)
|
||||
{
|
||||
groupStack.Pop();
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!TryParseGem(line, out var gemName, out var inlineGroups))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!mapping.TryGetValue(gemName, out var groups))
|
||||
{
|
||||
groups = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
mapping[gemName] = groups;
|
||||
}
|
||||
|
||||
if (groupStack.Count > 0)
|
||||
{
|
||||
foreach (var group in groupStack.Peek())
|
||||
{
|
||||
groups.Add(group);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var inline in inlineGroups)
|
||||
{
|
||||
groups.Add(inline);
|
||||
}
|
||||
|
||||
if (groups.Count == 0)
|
||||
{
|
||||
groups.Add("default");
|
||||
}
|
||||
}
|
||||
|
||||
return mapping.ToDictionary(
|
||||
static pair => pair.Key,
|
||||
static pair => (IReadOnlyCollection<string>)pair.Value
|
||||
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray(),
|
||||
StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static bool TryBeginGroup(string line, Stack<HashSet<string>> stack)
|
||||
{
|
||||
var match = GroupStartRegex.Match(line);
|
||||
if (!match.Success)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var parsedGroups = ParseGroupTokens(match.Groups["args"].Value);
|
||||
var inherited = stack.Count > 0
|
||||
? new HashSet<string>(stack.Peek(), StringComparer.OrdinalIgnoreCase)
|
||||
: new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (parsedGroups.Count == 0 && inherited.Count == 0)
|
||||
{
|
||||
inherited.Add("default");
|
||||
}
|
||||
|
||||
foreach (var group in parsedGroups)
|
||||
{
|
||||
inherited.Add(group);
|
||||
}
|
||||
|
||||
stack.Push(inherited);
|
||||
return true;
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ParseGroupTokens(string tokens)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(tokens))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var normalized = tokens.Trim().Trim('(', ')');
|
||||
var results = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var segment in normalized.Split(',', StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
var token = segment.Trim();
|
||||
if (token.Length == 0)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var symbolMatch = SymbolRegex.Match(token);
|
||||
if (symbolMatch.Success)
|
||||
{
|
||||
results.Add(symbolMatch.Groups["symbol"].Value);
|
||||
continue;
|
||||
}
|
||||
|
||||
var stringMatch = StringRegex.Match(token);
|
||||
if (stringMatch.Success)
|
||||
{
|
||||
results.Add(stringMatch.Groups["value"].Value);
|
||||
}
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
private static bool TryParseGem(string line, out string name, out IReadOnlyCollection<string> inlineGroups)
|
||||
{
|
||||
var match = GemLineRegex.Match(line);
|
||||
if (!match.Success)
|
||||
{
|
||||
name = string.Empty;
|
||||
inlineGroups = Array.Empty<string>();
|
||||
return false;
|
||||
}
|
||||
|
||||
name = match.Groups["name"].Value.Trim();
|
||||
inlineGroups = ExtractInlineGroups(match.Groups["args"].Value);
|
||||
return !string.IsNullOrWhiteSpace(name);
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string> ExtractInlineGroups(string args)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(args))
|
||||
{
|
||||
return Array.Empty<string>();
|
||||
}
|
||||
|
||||
var groups = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (Match match in InlineGroupRegex.Matches(args))
|
||||
{
|
||||
if (!match.Success)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var valueGroup = match.Groups["value"].Success ? match.Groups["value"] : match.Groups["typed"];
|
||||
if (!valueGroup.Success)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var value = valueGroup.Value;
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (Match symbol in SymbolRegex.Matches(value))
|
||||
{
|
||||
if (symbol.Success)
|
||||
{
|
||||
groups.Add(symbol.Groups["symbol"].Value);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (Match str in StringRegex.Matches(value))
|
||||
{
|
||||
if (str.Success)
|
||||
{
|
||||
groups.Add(str.Groups["value"].Value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return groups;
|
||||
}
|
||||
|
||||
private static bool IsGroupEnd(string line)
|
||||
{
|
||||
var trimmed = line.Trim();
|
||||
if (!trimmed.StartsWith("end", StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return trimmed.Length == 3 || char.IsWhiteSpace(trimmed[3]) || trimmed[3] == '#';
|
||||
}
|
||||
|
||||
private static string StripComment(string line)
|
||||
{
|
||||
if (string.IsNullOrEmpty(line))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var builder = new StringBuilder(line.Length);
|
||||
var inSingle = false;
|
||||
var inDouble = false;
|
||||
|
||||
for (var i = 0; i < line.Length; i++)
|
||||
{
|
||||
var ch = line[i];
|
||||
|
||||
if (ch == '\'' && !inDouble)
|
||||
{
|
||||
inSingle = !inSingle;
|
||||
}
|
||||
else if (ch == '"' && !inSingle)
|
||||
{
|
||||
inDouble = !inDouble;
|
||||
}
|
||||
|
||||
if (ch == '#' && !inSingle && !inDouble)
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
builder.Append(ch);
|
||||
}
|
||||
|
||||
return builder.ToString();
|
||||
}
|
||||
}
|
||||
@@ -2,13 +2,17 @@ namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
|
||||
|
||||
internal sealed class RubyPackage
|
||||
{
|
||||
private RubyPackage(
|
||||
public static IReadOnlyCollection<string> DefaultGroups { get; } = new[] { "default" };
|
||||
|
||||
internal RubyPackage(
|
||||
string name,
|
||||
string version,
|
||||
string source,
|
||||
string? platform,
|
||||
IReadOnlyCollection<string> groups,
|
||||
string locator,
|
||||
string? lockfileLocator,
|
||||
string? artifactLocator,
|
||||
string evidenceSource,
|
||||
bool declaredOnly)
|
||||
{
|
||||
Name = name;
|
||||
@@ -16,7 +20,9 @@ internal sealed class RubyPackage
|
||||
Source = source;
|
||||
Platform = platform;
|
||||
Groups = groups;
|
||||
Locator = locator;
|
||||
LockfileLocator = string.IsNullOrWhiteSpace(lockfileLocator) ? null : Normalize(lockfileLocator);
|
||||
ArtifactLocator = string.IsNullOrWhiteSpace(artifactLocator) ? null : Normalize(artifactLocator);
|
||||
EvidenceSource = string.IsNullOrWhiteSpace(evidenceSource) ? "Gemfile.lock" : evidenceSource;
|
||||
DeclaredOnly = declaredOnly;
|
||||
}
|
||||
|
||||
@@ -30,7 +36,11 @@ internal sealed class RubyPackage
|
||||
|
||||
public IReadOnlyCollection<string> Groups { get; }
|
||||
|
||||
public string Locator { get; }
|
||||
public string? LockfileLocator { get; }
|
||||
|
||||
public string? ArtifactLocator { get; }
|
||||
|
||||
public string EvidenceSource { get; }
|
||||
|
||||
public bool DeclaredOnly { get; }
|
||||
|
||||
@@ -38,15 +48,31 @@ internal sealed class RubyPackage
|
||||
|
||||
public string ComponentKey => $"purl::{Purl}";
|
||||
|
||||
public IReadOnlyCollection<KeyValuePair<string, string?>> CreateMetadata(RubyCapabilities? capabilities = null, RubyRuntimeUsage? runtimeUsage = null)
|
||||
public IReadOnlyCollection<KeyValuePair<string, string?>> CreateMetadata(
|
||||
RubyCapabilities? capabilities = null,
|
||||
RubyRuntimeUsage? runtimeUsage = null)
|
||||
{
|
||||
var metadata = new List<KeyValuePair<string, string?>>
|
||||
{
|
||||
new("source", Source),
|
||||
new("lockfile", string.IsNullOrWhiteSpace(Locator) ? "Gemfile.lock" : Locator),
|
||||
new("declaredOnly", DeclaredOnly ? "true" : "false")
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(LockfileLocator))
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("lockfile", LockfileLocator));
|
||||
}
|
||||
else if (!string.IsNullOrWhiteSpace(ArtifactLocator))
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("lockfile", ArtifactLocator));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(ArtifactLocator)
|
||||
&& !string.Equals(ArtifactLocator, LockfileLocator, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("artifact", ArtifactLocator));
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(Platform))
|
||||
{
|
||||
metadata.Add(new KeyValuePair<string, string?>("platform", Platform));
|
||||
@@ -81,8 +107,7 @@ internal sealed class RubyPackage
|
||||
|
||||
foreach (var scheduler in schedulers)
|
||||
{
|
||||
var key = $"capability.scheduler.{scheduler}";
|
||||
metadata.Add(new KeyValuePair<string, string?>(key, "true"));
|
||||
metadata.Add(new KeyValuePair<string, string?>($"capability.scheduler.{scheduler}", "true"));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -111,32 +136,19 @@ internal sealed class RubyPackage
|
||||
|
||||
public IReadOnlyCollection<LanguageComponentEvidence> CreateEvidence()
|
||||
{
|
||||
var locator = string.IsNullOrWhiteSpace(Locator)
|
||||
? "Gemfile.lock"
|
||||
: Locator;
|
||||
var locator = ArtifactLocator ?? LockfileLocator ?? "Gemfile.lock";
|
||||
|
||||
return new[]
|
||||
{
|
||||
new LanguageComponentEvidence(
|
||||
LanguageEvidenceKind.File,
|
||||
"Gemfile.lock",
|
||||
EvidenceSource,
|
||||
locator,
|
||||
Value: null,
|
||||
Sha256: null)
|
||||
};
|
||||
}
|
||||
|
||||
public static RubyPackage From(RubyLockEntry entry, string lockFileRelativePath)
|
||||
{
|
||||
var groups = entry.Groups.Count == 0
|
||||
? Array.Empty<string>()
|
||||
: entry.Groups.OrderBy(static g => g, StringComparer.OrdinalIgnoreCase).ToArray();
|
||||
|
||||
return new RubyPackage(entry.Name, entry.Version, entry.Source, entry.Platform, groups, lockFileRelativePath, declaredOnly: true);
|
||||
}
|
||||
|
||||
public static RubyPackage FromVendor(string name, string version, string source, string? platform, string locator)
|
||||
{
|
||||
return new RubyPackage(name, version, source, platform, Array.Empty<string>(), locator, declaredOnly: true);
|
||||
}
|
||||
private static string Normalize(string path)
|
||||
=> path.Replace('\\', '/');
|
||||
}
|
||||
|
||||
@@ -2,104 +2,140 @@ namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
|
||||
|
||||
internal static class RubyPackageCollector
|
||||
{
|
||||
public static IReadOnlyList<RubyPackage> CollectPackages(RubyLockData lockData, LanguageAnalyzerContext context)
|
||||
public static IReadOnlyList<RubyPackage> CollectPackages(
|
||||
RubyLockData lockData,
|
||||
LanguageAnalyzerContext context,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var packages = new List<RubyPackage>();
|
||||
var seen = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var bundlerConfig = RubyBundlerConfig.Load(context.RootPath);
|
||||
var vendorArtifacts = RubyVendorArtifactCollector.Collect(context, bundlerConfig, cancellationToken);
|
||||
var builders = new Dictionary<string, RubyPackageBuilder>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (!lockData.IsEmpty)
|
||||
foreach (var artifact in vendorArtifacts)
|
||||
{
|
||||
var relativeLockPath = lockData.LockFilePath is null
|
||||
? "Gemfile.lock"
|
||||
: context.GetRelativePath(lockData.LockFilePath);
|
||||
|
||||
if (string.IsNullOrWhiteSpace(relativeLockPath))
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var key = RubyPackageBuilder.BuildKey(artifact.Name, artifact.Version, artifact.Platform);
|
||||
if (!builders.TryGetValue(key, out var builder))
|
||||
{
|
||||
relativeLockPath = "Gemfile.lock";
|
||||
builder = new RubyPackageBuilder(artifact.Name, artifact.Version, artifact.Platform);
|
||||
builders[key] = builder;
|
||||
}
|
||||
|
||||
foreach (var entry in lockData.Entries)
|
||||
{
|
||||
var key = $"{entry.Name}@{entry.Version}";
|
||||
if (!seen.Add(key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
packages.Add(RubyPackage.From(entry, relativeLockPath));
|
||||
}
|
||||
builder.ApplyVendorArtifact(artifact);
|
||||
}
|
||||
|
||||
CollectVendorCachePackages(context, packages, seen);
|
||||
|
||||
return packages;
|
||||
}
|
||||
|
||||
private static void CollectVendorCachePackages(LanguageAnalyzerContext context, List<RubyPackage> packages, HashSet<string> seen)
|
||||
{
|
||||
var vendorCache = Path.Combine(context.RootPath, "vendor", "cache");
|
||||
if (!Directory.Exists(vendorCache))
|
||||
foreach (var entry in lockData.Entries)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var gemPath in Directory.EnumerateFiles(vendorCache, "*.gem", SearchOption.AllDirectories))
|
||||
{
|
||||
if (!TryParseGemArchive(gemPath, out var name, out var version, out var platform))
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var key = RubyPackageBuilder.BuildKey(entry.Name, entry.Version, entry.Platform);
|
||||
if (!builders.TryGetValue(key, out var builder))
|
||||
{
|
||||
continue;
|
||||
builder = new RubyPackageBuilder(entry.Name, entry.Version, entry.Platform);
|
||||
builders[key] = builder;
|
||||
}
|
||||
|
||||
var key = $"{name}@{version}";
|
||||
if (!seen.Add(key))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var locator = context.GetRelativePath(gemPath);
|
||||
packages.Add(RubyPackage.FromVendor(name, version, source: "vendor-cache", platform, locator));
|
||||
builder.ApplyLockEntry(entry);
|
||||
}
|
||||
}
|
||||
|
||||
private static bool TryParseGemArchive(string gemPath, out string name, out string version, out string? platform)
|
||||
{
|
||||
name = string.Empty;
|
||||
version = string.Empty;
|
||||
platform = null;
|
||||
|
||||
var fileName = Path.GetFileNameWithoutExtension(gemPath);
|
||||
if (string.IsNullOrWhiteSpace(fileName))
|
||||
if (builders.Count == 0)
|
||||
{
|
||||
return false;
|
||||
return Array.Empty<RubyPackage>();
|
||||
}
|
||||
|
||||
var segments = fileName.Split('-', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (segments.Length < 2)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var versionIndex = -1;
|
||||
for (var i = 1; i < segments.Length; i++)
|
||||
{
|
||||
if (char.IsDigit(segments[i][0]))
|
||||
{
|
||||
versionIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (versionIndex <= 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
name = string.Join('-', segments[..versionIndex]);
|
||||
version = segments[versionIndex];
|
||||
platform = segments.Length > versionIndex + 1
|
||||
? string.Join('-', segments[(versionIndex + 1)..])
|
||||
: null;
|
||||
|
||||
return !string.IsNullOrWhiteSpace(name) && !string.IsNullOrWhiteSpace(version);
|
||||
return builders.Values
|
||||
.Select(builder => builder.Build())
|
||||
.OrderBy(static package => package.ComponentKey, StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class RubyPackageBuilder
|
||||
{
|
||||
private readonly string _name;
|
||||
private readonly string _version;
|
||||
private readonly string? _platform;
|
||||
|
||||
private readonly HashSet<string> _groups = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private string? _lockSource;
|
||||
private string? _lockLocator;
|
||||
private string? _lockEvidenceSource;
|
||||
|
||||
private string? _artifactSource;
|
||||
private string? _artifactLocator;
|
||||
private string? _artifactEvidenceSource;
|
||||
|
||||
private bool _hasVendor;
|
||||
|
||||
public RubyPackageBuilder(string name, string version, string? platform)
|
||||
{
|
||||
_name = name;
|
||||
_version = version;
|
||||
_platform = platform;
|
||||
}
|
||||
|
||||
public static string BuildKey(string name, string version, string? platform)
|
||||
{
|
||||
return platform is null
|
||||
? $"{name}::{version}"
|
||||
: $"{name}::{version}::{platform}";
|
||||
}
|
||||
|
||||
public void ApplyLockEntry(RubyLockEntry entry)
|
||||
{
|
||||
_lockSource ??= entry.Source;
|
||||
_lockLocator ??= NormalizeLocator(entry.LockFileRelativePath);
|
||||
_lockEvidenceSource ??= Path.GetFileName(entry.LockFileRelativePath);
|
||||
|
||||
foreach (var group in entry.Groups)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(group))
|
||||
{
|
||||
_groups.Add(group);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void ApplyVendorArtifact(RubyVendorArtifact artifact)
|
||||
{
|
||||
_hasVendor = true;
|
||||
_artifactLocator ??= NormalizeLocator(artifact.RelativePath);
|
||||
_artifactSource ??= artifact.SourceLabel;
|
||||
_artifactEvidenceSource ??= Path.GetFileName(artifact.RelativePath);
|
||||
}
|
||||
|
||||
public RubyPackage Build()
|
||||
{
|
||||
var groups = _groups.Count == 0
|
||||
? RubyPackage.DefaultGroups
|
||||
: _groups
|
||||
.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase)
|
||||
.ToArray();
|
||||
|
||||
var source = _lockSource ?? _artifactSource ?? "unknown";
|
||||
var evidenceSource = _hasVendor
|
||||
? _artifactEvidenceSource ?? "vendor"
|
||||
: _lockEvidenceSource ?? "Gemfile.lock";
|
||||
|
||||
return new RubyPackage(
|
||||
_name,
|
||||
_version,
|
||||
source,
|
||||
_platform,
|
||||
groups,
|
||||
lockfileLocator: _lockLocator,
|
||||
artifactLocator: _artifactLocator,
|
||||
evidenceSource,
|
||||
declaredOnly: !_hasVendor);
|
||||
}
|
||||
|
||||
private static string NormalizeLocator(string? path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
return path.Replace('\\', '/');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,290 @@
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
|
||||
|
||||
internal static class RubyVendorArtifactCollector
|
||||
{
|
||||
private static readonly string[] DefaultVendorRoots =
|
||||
{
|
||||
Path.Combine("vendor", "cache"),
|
||||
Path.Combine(".bundle", "cache")
|
||||
};
|
||||
|
||||
private static readonly string[] DirectoryBlockList =
|
||||
{
|
||||
".git",
|
||||
".hg",
|
||||
".svn",
|
||||
"node_modules",
|
||||
"tmp",
|
||||
"log",
|
||||
"coverage"
|
||||
};
|
||||
|
||||
public static IReadOnlyList<RubyVendorArtifact> Collect(LanguageAnalyzerContext context, RubyBundlerConfig config, CancellationToken cancellationToken)
|
||||
{
|
||||
var roots = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
var normalizedRoot = EnsureTrailingSeparator(Path.GetFullPath(context.RootPath));
|
||||
|
||||
void TryAdd(string path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var absolute = Path.IsPathRooted(path) ? path : Path.Combine(context.RootPath, path);
|
||||
try
|
||||
{
|
||||
absolute = Path.GetFullPath(absolute);
|
||||
}
|
||||
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or ArgumentException or NotSupportedException)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!absolute.StartsWith(normalizedRoot, OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (Directory.Exists(absolute))
|
||||
{
|
||||
roots.Add(absolute);
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var root in DefaultVendorRoots)
|
||||
{
|
||||
TryAdd(root);
|
||||
}
|
||||
|
||||
TryAdd(Path.Combine(context.RootPath, "vendor", "bundle"));
|
||||
|
||||
foreach (var bundlePath in config.BundlePaths)
|
||||
{
|
||||
TryAdd(bundlePath);
|
||||
TryAdd(Path.Combine(bundlePath, "cache"));
|
||||
}
|
||||
|
||||
var artifacts = new List<RubyVendorArtifact>();
|
||||
foreach (var root in roots.OrderBy(static value => value, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
TraverseDirectory(context, root, artifacts, cancellationToken);
|
||||
}
|
||||
|
||||
return artifacts;
|
||||
}
|
||||
|
||||
private static void TraverseDirectory(
|
||||
LanguageAnalyzerContext context,
|
||||
string root,
|
||||
List<RubyVendorArtifact> artifacts,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var stack = new Stack<string>();
|
||||
stack.Push(root);
|
||||
|
||||
while (stack.Count > 0)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
var current = stack.Pop();
|
||||
|
||||
IEnumerable<string>? files = null;
|
||||
try
|
||||
{
|
||||
files = Directory.EnumerateFiles(current);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
files = null;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
files = null;
|
||||
}
|
||||
|
||||
if (files is not null)
|
||||
{
|
||||
foreach (var file in files)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
ProcessFile(context, artifacts, file);
|
||||
}
|
||||
}
|
||||
|
||||
IEnumerable<string>? directories = null;
|
||||
try
|
||||
{
|
||||
directories = Directory.EnumerateDirectories(current);
|
||||
}
|
||||
catch (IOException)
|
||||
{
|
||||
directories = null;
|
||||
}
|
||||
catch (UnauthorizedAccessException)
|
||||
{
|
||||
directories = null;
|
||||
}
|
||||
|
||||
if (directories is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
cancellationToken.ThrowIfCancellationRequested();
|
||||
|
||||
if (ShouldSkip(directory))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
ProcessDirectoryArtifact(context, artifacts, directory);
|
||||
stack.Push(directory);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static void ProcessFile(LanguageAnalyzerContext context, List<RubyVendorArtifact> artifacts, string filePath)
|
||||
{
|
||||
var extension = Path.GetExtension(filePath);
|
||||
if (!extension.Equals(".gem", StringComparison.OrdinalIgnoreCase)
|
||||
&& !extension.Equals(".gemspec", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var fileName = Path.GetFileNameWithoutExtension(filePath);
|
||||
if (!RubyPackageNameParser.TryParse(fileName, out var name, out var version, out var platform))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!TryGetRelativePath(context, filePath, out var relative))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sourceLabel = DescribeSource(relative);
|
||||
artifacts.Add(new RubyVendorArtifact(name, version, platform, relative, sourceLabel));
|
||||
}
|
||||
|
||||
private static void ProcessDirectoryArtifact(LanguageAnalyzerContext context, List<RubyVendorArtifact> artifacts, string directoryPath)
|
||||
{
|
||||
var parent = Path.GetFileName(Path.GetDirectoryName(directoryPath)?.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) ?? string.Empty);
|
||||
if (!parent.Equals("gems", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var directoryName = Path.GetFileName(directoryPath.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar));
|
||||
if (!RubyPackageNameParser.TryParse(directoryName, out var name, out var version, out var platform))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!TryGetRelativePath(context, directoryPath, out var relative))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var sourceLabel = DescribeSource(relative);
|
||||
artifacts.Add(new RubyVendorArtifact(name, version, platform, relative, sourceLabel));
|
||||
}
|
||||
|
||||
private static bool TryGetRelativePath(LanguageAnalyzerContext context, string absolutePath, out string relative)
|
||||
{
|
||||
relative = context.GetRelativePath(absolutePath);
|
||||
if (string.IsNullOrWhiteSpace(relative) || relative.StartsWith("..", StringComparison.Ordinal))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
relative = relative.Replace('\\', '/');
|
||||
return true;
|
||||
}
|
||||
|
||||
private static bool ShouldSkip(string directoryPath)
|
||||
{
|
||||
var name = Path.GetFileName(directoryPath);
|
||||
if (string.IsNullOrEmpty(name))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return DirectoryBlockList.Contains(name, StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
private static string DescribeSource(string relativePath)
|
||||
{
|
||||
var normalized = relativePath.Replace('\\', '/');
|
||||
var segments = normalized.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (segments.Length == 0)
|
||||
{
|
||||
return normalized;
|
||||
}
|
||||
|
||||
return segments[0];
|
||||
}
|
||||
|
||||
private static string EnsureTrailingSeparator(string path)
|
||||
{
|
||||
if (path.EndsWith(Path.DirectorySeparatorChar) || path.EndsWith(Path.AltDirectorySeparatorChar))
|
||||
{
|
||||
return path;
|
||||
}
|
||||
|
||||
return path + Path.DirectorySeparatorChar;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record RubyVendorArtifact(
|
||||
string Name,
|
||||
string Version,
|
||||
string? Platform,
|
||||
string RelativePath,
|
||||
string SourceLabel);
|
||||
|
||||
internal static class RubyPackageNameParser
|
||||
{
|
||||
public static bool TryParse(string value, out string name, out string version, out string? platform)
|
||||
{
|
||||
name = string.Empty;
|
||||
version = string.Empty;
|
||||
platform = null;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var segments = value.Split('-', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (segments.Length < 2)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
var versionIndex = -1;
|
||||
for (var i = segments.Length - 1; i >= 0; i--)
|
||||
{
|
||||
if (char.IsDigit(segments[i][0]))
|
||||
{
|
||||
versionIndex = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (versionIndex <= 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
name = string.Join('-', segments[..versionIndex]);
|
||||
version = segments[versionIndex];
|
||||
platform = versionIndex < segments.Length - 1
|
||||
? string.Join('-', segments[(versionIndex + 1)..])
|
||||
: null;
|
||||
|
||||
return !string.IsNullOrWhiteSpace(name) && !string.IsNullOrWhiteSpace(version);
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,6 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang.Ruby.Internal;
|
||||
using StellaOps.Scanner.Surface.Env;
|
||||
using StellaOps.Scanner.Surface.Validation;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Ruby;
|
||||
|
||||
@@ -13,14 +15,16 @@ public sealed class RubyLanguageAnalyzer : ILanguageAnalyzer
|
||||
ArgumentNullException.ThrowIfNull(context);
|
||||
ArgumentNullException.ThrowIfNull(writer);
|
||||
|
||||
var lockData = await RubyLockData.LoadAsync(context.RootPath, cancellationToken).ConfigureAwait(false);
|
||||
if (lockData.IsEmpty)
|
||||
await EnsureSurfaceValidationAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var lockData = await RubyLockData.LoadAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
var packages = RubyPackageCollector.CollectPackages(lockData, context, cancellationToken);
|
||||
if (packages.Count == 0)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var capabilities = await RubyCapabilityDetector.DetectAsync(context, cancellationToken).ConfigureAwait(false);
|
||||
var packages = RubyPackageCollector.CollectPackages(lockData, context);
|
||||
var runtimeGraph = await RubyRuntimeGraphBuilder.BuildAsync(context, packages, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
foreach (var package in packages.OrderBy(static p => p.ComponentKey, StringComparer.Ordinal))
|
||||
@@ -40,4 +44,32 @@ public sealed class RubyLanguageAnalyzer : ILanguageAnalyzer
|
||||
usedByEntrypoint: runtimeUsage?.UsedByEntrypoint ?? false);
|
||||
}
|
||||
}
|
||||
|
||||
private static async ValueTask EnsureSurfaceValidationAsync(LanguageAnalyzerContext context, CancellationToken cancellationToken)
|
||||
{
|
||||
if (context.Services is null)
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
if (!context.TryGetService<ISurfaceValidatorRunner>(out var validatorRunner)
|
||||
|| !context.TryGetService<ISurfaceEnvironment>(out var environment))
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
var properties = new Dictionary<string, object?>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
["analyzerId"] = \"ruby\",
|
||||
[\"rootPath\"] = context.RootPath
|
||||
};
|
||||
|
||||
var validationContext = SurfaceValidationContext.Create(
|
||||
context.Services,
|
||||
\"StellaOps.Scanner.Analyzers.Lang.Ruby\",
|
||||
environment.Settings,
|
||||
properties);
|
||||
|
||||
await validatorRunner.EnsureAsync(validationContext, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,5 +2,6 @@
|
||||
|
||||
| Task ID | State | Notes |
|
||||
| --- | --- | --- |
|
||||
| `SCANNER-ENG-0016` | DOING (2025-11-10) | Building RubyLockCollector + multi-source vendor ingestion per design §4.1–4.3 (Codex agent). |
|
||||
| `SCANNER-ENG-0017` | DONE (2025-11-09) | Build runtime require/autoload graph builder with tree-sitter Ruby per design §4.4, feed EntryTrace hints. |
|
||||
| `SCANNER-ENG-0018` | DONE (2025-11-09) | Emit Ruby capability + framework surface signals, align with design §4.5 / Sprint 138. |
|
||||
|
||||
@@ -62,8 +62,8 @@ public sealed class BundleInspectorTests
|
||||
var scan = DenoBundleScanner.Scan(temp, CancellationToken.None);
|
||||
var observations = DenoBundleScanner.ToObservations(scan);
|
||||
|
||||
Assert.Equal(1, scan.EszipBundles.Length);
|
||||
Assert.Equal(1, scan.CompiledBundles.Length);
|
||||
Assert.Single(scan.EszipBundles);
|
||||
Assert.Single(scan.CompiledBundles);
|
||||
Assert.Equal(2, observations.Length);
|
||||
Assert.Contains(observations, obs => obs.BundleType == "eszip");
|
||||
Assert.Contains(observations, obs => obs.BundleType == "deno-compile");
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Linq;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Internal;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
|
||||
@@ -21,9 +22,24 @@ public sealed class ContainerAdapterTests
|
||||
var inputs = DenoContainerAdapter.CollectInputs(workspace, observations);
|
||||
|
||||
Assert.NotEmpty(inputs);
|
||||
Assert.Contains(inputs, input => input.Kind == DenoContainerSourceKind.Cache);
|
||||
Assert.Contains(inputs, input => input.Kind == DenoContainerSourceKind.Vendor);
|
||||
Assert.Contains(inputs, input => input.Kind == DenoContainerSourceKind.Bundle);
|
||||
|
||||
var cacheInputs = inputs.Where(input => input.Kind == DenoContainerSourceKind.Cache).ToArray();
|
||||
Assert.Equal(3, cacheInputs.Length);
|
||||
Assert.All(cacheInputs, cache => Assert.True(cache.Metadata.ContainsKey("path")));
|
||||
|
||||
var vendorInputs = inputs.Where(input => input.Kind == DenoContainerSourceKind.Vendor).ToArray();
|
||||
Assert.Equal(2, vendorInputs.Length);
|
||||
Assert.Contains(vendorInputs, vendor => vendor.Metadata.TryGetValue("alias", out var alias) && alias?.Contains("vendor", StringComparison.OrdinalIgnoreCase) == true);
|
||||
|
||||
var bundleInputs = inputs.Where(input => input.Kind == DenoContainerSourceKind.Bundle).ToArray();
|
||||
Assert.Equal(2, bundleInputs.Length);
|
||||
Assert.Contains(bundleInputs, bundle => string.Equals(bundle.Bundle?.BundleType, "eszip", StringComparison.OrdinalIgnoreCase));
|
||||
Assert.Contains(bundleInputs, bundle => string.Equals(bundle.Bundle?.BundleType, "deno-compile", StringComparison.OrdinalIgnoreCase));
|
||||
Assert.All(bundleInputs, bundle =>
|
||||
{
|
||||
Assert.True(bundle.Metadata.ContainsKey("entrypoint"));
|
||||
Assert.True(bundle.Metadata.ContainsKey("moduleCount"));
|
||||
});
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -70,6 +70,7 @@ public sealed class DenoWorkspaceNormalizerTests
|
||||
|
||||
var graph = DenoModuleGraphResolver.Resolve(workspace, CancellationToken.None);
|
||||
var compatibility = DenoNpmCompatibilityAdapter.Analyze(workspace, graph, CancellationToken.None);
|
||||
var cacheSummary = string.Join(";", workspace.CacheLocations.Select(cache => $"{cache.Kind}:{cache.AbsolutePath}"));
|
||||
|
||||
Assert.NotEmpty(graph.Nodes);
|
||||
Assert.NotEmpty(graph.Edges);
|
||||
@@ -108,7 +109,7 @@ public sealed class DenoWorkspaceNormalizerTests
|
||||
usage => usage.Specifier == "node:fs");
|
||||
|
||||
var npmResolution = compatibility.NpmResolutions.First(res => res.Specifier == "npm:dayjs@1");
|
||||
Assert.True(npmResolution.ExistsOnDisk);
|
||||
Assert.True(npmResolution.ExistsOnDisk, $"ResolvedPath={npmResolution.ResolvedPath ?? "(null)"}");
|
||||
Assert.Equal("deno", npmResolution.Condition);
|
||||
Assert.True(npmResolution.ResolvedPath?.EndsWith("deno.mod.ts", StringComparison.OrdinalIgnoreCase));
|
||||
}
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.Harness;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.Golden;
|
||||
|
||||
public sealed class DenoAnalyzerGoldenTests
|
||||
{
|
||||
[Fact]
|
||||
public async Task AnalyzerMatchesGoldenSnapshotAsync()
|
||||
{
|
||||
var fixture = TestPaths.ResolveFixture("lang", "deno", "full");
|
||||
var golden = Path.Combine(fixture, "expected.json");
|
||||
var analyzers = new ILanguageAnalyzer[] { new DenoLanguageAnalyzer() };
|
||||
var cancellationToken = TestContext.Current.CancellationToken;
|
||||
|
||||
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(fixture, analyzers, cancellationToken).ConfigureAwait(false);
|
||||
var normalized = Normalize(json, fixture);
|
||||
var expected = await File.ReadAllTextAsync(golden, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
normalized = normalized.TrimEnd();
|
||||
expected = expected.TrimEnd();
|
||||
|
||||
if (!string.Equals(expected, normalized, StringComparison.Ordinal))
|
||||
{
|
||||
var actualPath = golden + ".actual";
|
||||
await File.WriteAllTextAsync(actualPath, normalized, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
Assert.Equal(expected, normalized);
|
||||
}
|
||||
|
||||
private static string Normalize(string json, string workspaceRoot)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(json))
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
var normalizedRoot = workspaceRoot.Replace("\\", "/", StringComparison.Ordinal);
|
||||
var builder = json.Replace(normalizedRoot, "<workspace>", StringComparison.Ordinal);
|
||||
var altRoot = workspaceRoot.Replace("/", "\\", StringComparison.Ordinal);
|
||||
builder = builder.Replace(altRoot, "<workspace>", StringComparison.Ordinal);
|
||||
return builder;
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
@@ -37,6 +39,27 @@ public sealed class DenoLanguageAnalyzerObservationTests
|
||||
Assert.True(payload.Content.Length > 0);
|
||||
Assert.NotNull(payload.Metadata);
|
||||
Assert.True(payload.Metadata!.ContainsKey("deno.observation.hash"));
|
||||
|
||||
using var document = JsonDocument.Parse(payload.Content.Span);
|
||||
var root = document.RootElement;
|
||||
|
||||
var entrypoints = root.GetProperty("entrypoints").EnumerateArray().Select(element => element.GetString()).ToArray();
|
||||
Assert.Contains("src/main.ts", entrypoints);
|
||||
|
||||
var capabilities = root.GetProperty("capabilities").EnumerateArray().ToArray();
|
||||
Assert.Contains(capabilities, capability => capability.GetProperty("reason").GetString() == "builtin.deno.ffi");
|
||||
Assert.Contains(capabilities, capability => capability.GetProperty("reason").GetString() == "builtin.node.worker_threads");
|
||||
Assert.Contains(capabilities, capability => capability.GetProperty("reason").GetString() == "builtin.node.fs");
|
||||
|
||||
var dynamicImports = root.GetProperty("dynamicImports").EnumerateArray().Select(element => element.GetProperty("specifier").GetString()).ToArray();
|
||||
Assert.Contains("https://cdn.example.com/dynamic/mod.ts", dynamicImports);
|
||||
|
||||
var literalFetches = root.GetProperty("literalFetches").EnumerateArray().Select(element => element.GetProperty("url").GetString()).ToArray();
|
||||
Assert.Contains("https://api.example.com/data.json", literalFetches);
|
||||
|
||||
var bundles = root.GetProperty("bundles").EnumerateArray().ToArray();
|
||||
Assert.Contains(bundles, bundle => bundle.GetProperty("type").GetString() == "eszip");
|
||||
Assert.Contains(bundles, bundle => bundle.GetProperty("type").GetString() == "deno-compile");
|
||||
}
|
||||
finally
|
||||
{
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Internal;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Fixtures;
|
||||
|
||||
internal static class BundleFixtureBuilder
|
||||
{
|
||||
@@ -8,6 +10,10 @@ internal static class BundleFixtureBuilder
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
var bundlePath = Path.Combine(directory, "sample.eszip");
|
||||
if (File.Exists(bundlePath))
|
||||
{
|
||||
File.Delete(bundlePath);
|
||||
}
|
||||
|
||||
using (var archive = ZipFile.Open(bundlePath, ZipArchiveMode.Create))
|
||||
{
|
||||
@@ -52,6 +58,10 @@ internal static class BundleFixtureBuilder
|
||||
{
|
||||
Directory.CreateDirectory(directory);
|
||||
var binaryPath = Path.Combine(directory, "sample.deno");
|
||||
if (File.Exists(binaryPath))
|
||||
{
|
||||
File.Delete(binaryPath);
|
||||
}
|
||||
var eszipPath = CreateSampleEszip(directory);
|
||||
var eszipBytes = File.ReadAllBytes(eszipPath);
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using StellaOps.Scanner.Analyzers.Lang.Deno.Fixtures;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
|
||||
|
||||
namespace StellaOps.Scanner.Analyzers.Lang.Deno.Tests.TestFixtures;
|
||||
@@ -19,106 +20,7 @@ internal static class DenoWorkspaceTestFixture
|
||||
private static void CreateDenoFixture(string root, out string envDenoDir)
|
||||
{
|
||||
Directory.CreateDirectory(root);
|
||||
|
||||
File.WriteAllText(Path.Combine(root, "deno.jsonc"), """
|
||||
// sample deno config
|
||||
{
|
||||
"importMap": "./import_map.json",
|
||||
"lock": {
|
||||
"enabled": true,
|
||||
"path": "./deno.lock"
|
||||
},
|
||||
"vendor": {
|
||||
"enabled": true,
|
||||
"path": "./vendor"
|
||||
},
|
||||
"nodeModulesDir": false,
|
||||
"imports": {
|
||||
"$std/": "https://deno.land/std@0.207.0/",
|
||||
"app/": "./src/app/",
|
||||
"data": "./data/data.json",
|
||||
"npmDynamic": "npm:dayjs@1",
|
||||
"nodeFs": "node:fs"
|
||||
},
|
||||
"scopes": {
|
||||
"https://deno.land/": {
|
||||
"fmt/": "https://deno.land/std@0.207.0/fmt/"
|
||||
}
|
||||
}
|
||||
}
|
||||
""");
|
||||
|
||||
Directory.CreateDirectory(Path.Combine(root, "src", "app"));
|
||||
Directory.CreateDirectory(Path.Combine(root, "data"));
|
||||
File.WriteAllText(Path.Combine(root, "data", "data.json"), "{ \"ok\": true }");
|
||||
File.WriteAllText(
|
||||
Path.Combine(root, "import_map.json"),
|
||||
"""
|
||||
{
|
||||
"imports": {
|
||||
"app/": "./src/app/main.ts",
|
||||
"vendor/": "https://deno.land/std@0.207.0/"
|
||||
}
|
||||
}
|
||||
""");
|
||||
|
||||
File.WriteAllText(
|
||||
Path.Combine(root, "deno.lock"),
|
||||
"""
|
||||
{
|
||||
"version": "4",
|
||||
"remote": {
|
||||
"https://deno.land/std@0.207.0/http/server.ts": "sha256-deadbeef",
|
||||
"https://example.com/mod.ts": "sha256-feedface",
|
||||
"node:fs": "builtin"
|
||||
},
|
||||
"redirects": {
|
||||
"https://deno.land/std/http/server.ts": "https://deno.land/std@0.207.0/http/server.ts"
|
||||
},
|
||||
"npm": {
|
||||
"specifiers": {
|
||||
"npm:dayjs@1": "dayjs@1.11.12"
|
||||
},
|
||||
"packages": {
|
||||
"dayjs@1.11.12": {
|
||||
"integrity": "sha512-sample",
|
||||
"dependencies": {
|
||||
"tslib": "tslib@2.6.3"
|
||||
}
|
||||
},
|
||||
"tslib@2.6.3": {
|
||||
"integrity": "sha512-tslib",
|
||||
"dependencies": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
""");
|
||||
|
||||
var vendorRoot = Path.Combine(root, "vendor", "https", "deno.land", "std@0.207.0", "http");
|
||||
Directory.CreateDirectory(vendorRoot);
|
||||
File.WriteAllText(Path.Combine(vendorRoot, "server.ts"), "export const vendor = true;");
|
||||
|
||||
var vendorBase = Path.Combine(root, "vendor");
|
||||
File.WriteAllText(
|
||||
Path.Combine(vendorBase, "import_map.json"),
|
||||
"""
|
||||
{
|
||||
"imports": {
|
||||
"std/http/server.ts": "https://deno.land/std@0.207.0/http/server.ts"
|
||||
}
|
||||
}
|
||||
""");
|
||||
File.WriteAllText(
|
||||
Path.Combine(vendorBase, "deno.lock"),
|
||||
"""
|
||||
{
|
||||
"version": "1",
|
||||
"remote": {}
|
||||
}
|
||||
""");
|
||||
|
||||
CreateDenoDir(Path.Combine(root, ".deno"), "workspace.ts", includeRegistry: true);
|
||||
CopyTemplateWorkspace(root);
|
||||
|
||||
envDenoDir = Path.Combine(root, "env-deno");
|
||||
CreateDenoDir(envDenoDir, "env.ts");
|
||||
@@ -130,6 +32,39 @@ internal static class DenoWorkspaceTestFixture
|
||||
var layerVendor = Path.Combine(layerFs, "vendor", "https", "layer.example");
|
||||
Directory.CreateDirectory(layerVendor);
|
||||
File.WriteAllText(Path.Combine(layerFs, "vendor", "import_map.json"), "{\"imports\":{\"layer/\": \"https://layer.example/\"}}");
|
||||
|
||||
var bundlesRoot = Path.Combine(root, "bundles");
|
||||
Directory.CreateDirectory(bundlesRoot);
|
||||
BundleFixtureBuilder.CreateSampleEszip(bundlesRoot);
|
||||
BundleFixtureBuilder.CreateSampleCompiledBinary(bundlesRoot);
|
||||
}
|
||||
|
||||
private static void CopyTemplateWorkspace(string destination)
|
||||
{
|
||||
var template = TestPaths.ResolveFixture("lang", "deno", "full");
|
||||
CopyDirectory(template, destination);
|
||||
}
|
||||
|
||||
private static void CopyDirectory(string source, string destination)
|
||||
{
|
||||
foreach (var directory in Directory.EnumerateDirectories(source, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var relative = Path.GetRelativePath(source, directory);
|
||||
Directory.CreateDirectory(Path.Combine(destination, relative));
|
||||
}
|
||||
|
||||
foreach (var file in Directory.EnumerateFiles(source, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
if (file.EndsWith(".actual", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var relative = Path.GetRelativePath(source, file);
|
||||
var target = Path.Combine(destination, relative);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(target)!);
|
||||
File.Copy(file, target, overwrite: true);
|
||||
}
|
||||
}
|
||||
|
||||
private static void CreateDenoDir(string root, string fileName, bool includeRegistry = false)
|
||||
@@ -142,7 +77,7 @@ internal static class DenoWorkspaceTestFixture
|
||||
Directory.CreateDirectory(npm);
|
||||
|
||||
File.WriteAllText(Path.Combine(deps, fileName), "export const cache = true;");
|
||||
File.WriteAllText(Path.Combine(gen, $"{Path.GetFileNameWithoutExtension(fileName)}.js"), "console.log('gen');");
|
||||
File.WriteAllText(Path.Combine(gen, $"{Path.GetFileNameWithoutExtension(fileName)}.js"), "console.log(gen);");
|
||||
File.WriteAllText(Path.Combine(npm, "package.json"), "{}");
|
||||
|
||||
if (includeRegistry)
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
export const dynamicValue = 42;
|
||||
@@ -0,0 +1 @@
|
||||
export const dayjs = () => ({ iso: () => "2024-09-01" });
|
||||
@@ -0,0 +1,3 @@
|
||||
export function dayjs() {
|
||||
return { iso: () => "2024-09-01" };
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
module.exports = function dayjs() {
|
||||
return { iso: () => "2024-09-01" };
|
||||
};
|
||||
@@ -0,0 +1,11 @@
|
||||
{
|
||||
"name": "dayjs",
|
||||
"version": "1.11.12",
|
||||
"exports": {
|
||||
".": {
|
||||
"deno": "./deno.mod.ts",
|
||||
"import": "./esm/index.js",
|
||||
"default": "./lib/index.js"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"name": "tslib",
|
||||
"version": "2.6.3"
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"ok": true
|
||||
}
|
||||
@@ -0,0 +1,24 @@
|
||||
// Deterministic Deno workspace exercising vendor, npm, FFI, worker, and fetch flows.
|
||||
{
|
||||
"importMap": "./import_map.json",
|
||||
"lock": {
|
||||
"enabled": true,
|
||||
"path": "./deno.lock"
|
||||
},
|
||||
"nodeModulesDir": false,
|
||||
"tasks": {
|
||||
"serve": "deno run --allow-net ./src/main.ts"
|
||||
},
|
||||
"compilerOptions": {
|
||||
"strict": true,
|
||||
"noUncheckedIndexedAccess": true
|
||||
},
|
||||
"vendor": {
|
||||
"enabled": true,
|
||||
"path": "./vendor"
|
||||
},
|
||||
"fmt": {
|
||||
"useTabs": false,
|
||||
"lineWidth": 100
|
||||
}
|
||||
}
|
||||
28
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/deno/full/deno.lock
generated
Normal file
28
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Tests/Fixtures/lang/deno/full/deno.lock
generated
Normal file
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"version": "4",
|
||||
"remote": {
|
||||
"https://deno.land/std@0.207.0/http/server.ts": "sha256-deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
|
||||
"https://cdn.example.com/dynamic/mod.ts": "sha256-feedfacefeedfacefeedfacefeedfacefeedfacefeedfacefeedfacefeedface",
|
||||
"https://api.example.com/data.json": "sha256-0102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f20"
|
||||
},
|
||||
"redirects": {
|
||||
"https://deno.land/std/http/server.ts": "https://deno.land/std@0.207.0/http/server.ts"
|
||||
},
|
||||
"npm": {
|
||||
"specifiers": {
|
||||
"npm:dayjs@1": "dayjs@1.11.12"
|
||||
},
|
||||
"packages": {
|
||||
"dayjs@1.11.12": {
|
||||
"integrity": "sha512-sample-dayjs-integrity",
|
||||
"dependencies": {
|
||||
"tslib": "tslib@2.6.3"
|
||||
}
|
||||
},
|
||||
"tslib@2.6.3": {
|
||||
"integrity": "sha512-sample-tslib",
|
||||
"dependencies": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
"pending"
|
||||
@@ -0,0 +1,12 @@
|
||||
{
|
||||
"imports": {
|
||||
"app/": "./src/",
|
||||
"ffi/": "./src/ffi/",
|
||||
"workers/": "./src/workers/",
|
||||
"npmDynamic": "npm:dayjs@1",
|
||||
"nodeFs": "node:fs",
|
||||
"nodeCrypto": "node:crypto",
|
||||
"nodeWorker": "node:worker_threads",
|
||||
"denoFfi": "deno:ffi"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,6 @@
|
||||
export function openBridge() {
|
||||
const lib = Deno.dlopen("./ffi/libexample.so", {
|
||||
add: { parameters: ["i32", "i32"], result: "i32" }
|
||||
});
|
||||
lib.close();
|
||||
}
|
||||
@@ -0,0 +1 @@
|
||||
FAKEELF
|
||||
@@ -0,0 +1,41 @@
|
||||
import dayjs from "npmDynamic";
|
||||
import { serve } from "https://deno.land/std@0.207.0/http/server.ts";
|
||||
import { Worker } from "nodeWorker";
|
||||
import { dlopen } from "denoFfi";
|
||||
import "workers/metrics.ts";
|
||||
|
||||
const dynamicTarget = "https://cdn.example.com/dynamic/mod.ts";
|
||||
const fetchTarget = "https://api.example.com/data.json";
|
||||
|
||||
async function spinWorkers() {
|
||||
const worker = new Worker(new URL("./workers/child.ts", import.meta.url), { type: "module" });
|
||||
worker.postMessage({ kind: "child", payload: "ping" });
|
||||
|
||||
const shared = new SharedWorker(new URL("./workers/shared.ts", import.meta.url), { type: "module" });
|
||||
shared.port.postMessage({ kind: "shared", payload: "metrics" });
|
||||
}
|
||||
|
||||
function loadFfi() {
|
||||
const lib = dlopen("./ffi/libexample.so", {
|
||||
add: { parameters: ["i32", "i32"], result: "i32" }
|
||||
});
|
||||
try {
|
||||
return lib.symbols;
|
||||
} finally {
|
||||
lib.close();
|
||||
}
|
||||
}
|
||||
|
||||
export async function main() {
|
||||
await spinWorkers();
|
||||
loadFfi();
|
||||
|
||||
await import(dynamicTarget);
|
||||
await fetch(fetchTarget);
|
||||
|
||||
await serve(() => new Response(dayjs().format()), { hostname: "127.0.0.1", port: 8088 });
|
||||
}
|
||||
|
||||
if (import.meta.main) {
|
||||
await main();
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
self.onmessage = (event) => {
|
||||
const payload = event.data ?? {};
|
||||
self.postMessage({ ...payload, worker: "child" });
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
addEventListener("message", (event) => {
|
||||
console.log("metric", event.data);
|
||||
});
|
||||
@@ -0,0 +1,6 @@
|
||||
onconnect = (event) => {
|
||||
const [port] = event.ports;
|
||||
port.onmessage = (message) => {
|
||||
port.postMessage({ kind: "shared", payload: message.data });
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,3 @@
|
||||
export async function serve(handler: () => Response, _options?: { hostname?: string; port?: number }) {
|
||||
return handler();
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
---
|
||||
BUNDLE_GEMFILE: "apps/api/Gemfile"
|
||||
BUNDLE_PATH: "apps/api/vendor/bundle"
|
||||
@@ -0,0 +1,12 @@
|
||||
source "https://rubygems.org"
|
||||
|
||||
gem "rails", "~> 7.1.0"
|
||||
|
||||
group :development, :test do
|
||||
gem "pry"
|
||||
gem "rubocop", require: false
|
||||
end
|
||||
|
||||
group :production, :console do
|
||||
gem "puma", "~> 6.4"
|
||||
end
|
||||
@@ -0,0 +1,19 @@
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
pry (1.0.0)
|
||||
puma (6.4.2)
|
||||
rails (7.1.3)
|
||||
rubocop (1.60.0)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
|
||||
DEPENDENCIES
|
||||
pry
|
||||
puma (~> 6.4)
|
||||
rails (~> 7.1.0)
|
||||
rubocop
|
||||
|
||||
BUNDLED WITH
|
||||
2.5.10
|
||||
@@ -0,0 +1,6 @@
|
||||
require "rails"
|
||||
require "puma"
|
||||
require "bootsnap"
|
||||
require "sidekiq"
|
||||
|
||||
puts "workspace"
|
||||
@@ -0,0 +1,7 @@
|
||||
source "https://rubygems.org"
|
||||
|
||||
group :jobs do
|
||||
gem "sidekiq"
|
||||
end
|
||||
|
||||
gem "bootsnap"
|
||||
@@ -0,0 +1,15 @@
|
||||
GEM
|
||||
remote: https://rubygems.org/
|
||||
specs:
|
||||
bootsnap (1.18.4)
|
||||
sidekiq (7.2.4)
|
||||
|
||||
PLATFORMS
|
||||
ruby
|
||||
|
||||
DEPENDENCIES
|
||||
bootsnap
|
||||
sidekiq
|
||||
|
||||
BUNDLED WITH
|
||||
2.5.10
|
||||
@@ -0,0 +1 @@
|
||||
[]
|
||||
@@ -21,4 +21,17 @@ public sealed class RubyLanguageAnalyzerTests
|
||||
cancellationToken: TestContext.Current.CancellationToken,
|
||||
usageHints: usageHints);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task WorkspaceLockfilesAndVendorArtifactsAsync()
|
||||
{
|
||||
var fixture = TestPaths.ResolveFixture("lang", "ruby", "workspace");
|
||||
var golden = Path.Combine(fixture, "expected.json");
|
||||
|
||||
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
||||
fixture,
|
||||
golden,
|
||||
new ILanguageAnalyzer[] { new RubyLanguageAnalyzer() },
|
||||
cancellationToken: TestContext.Current.CancellationToken);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,7 @@ public static class JavaClassFileFactory
|
||||
using var buffer = new MemoryStream();
|
||||
using var writer = new BigEndianWriter(buffer);
|
||||
|
||||
WriteClassFileHeader(writer, constantPoolCount: 18);
|
||||
WriteClassFileHeader(writer, constantPoolCount: 20);
|
||||
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8(internalClassName); // #1
|
||||
writer.WriteByte((byte)ConstantTag.Class); writer.WriteUInt16(1); // #2
|
||||
@@ -59,14 +59,16 @@ public static class JavaClassFileFactory
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("Code"); // #7
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8(resourcePath); // #8
|
||||
writer.WriteByte((byte)ConstantTag.String); writer.WriteUInt16(8); // #9
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("java/lang/Class"); // #10
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("java/lang/ClassLoader"); // #10
|
||||
writer.WriteByte((byte)ConstantTag.Class); writer.WriteUInt16(10); // #11
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("getResource"); // #12
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("(Ljava/lang/String;)Ljava/net/URL;"); // #13
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("getSystemClassLoader"); // #12
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("()Ljava/lang/ClassLoader;"); // #13
|
||||
writer.WriteByte((byte)ConstantTag.NameAndType); writer.WriteUInt16(12); writer.WriteUInt16(13); // #14
|
||||
writer.WriteByte((byte)ConstantTag.Methodref); writer.WriteUInt16(11); writer.WriteUInt16(14); // #15
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("dummy"); // #16
|
||||
writer.WriteByte((byte)ConstantTag.String); writer.WriteUInt16(16); // #17
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("getResource"); // #16
|
||||
writer.WriteByte((byte)ConstantTag.Utf8); writer.WriteUtf8("(Ljava/lang/String;)Ljava/net/URL;"); // #17
|
||||
writer.WriteByte((byte)ConstantTag.NameAndType); writer.WriteUInt16(16); writer.WriteUInt16(17); // #18
|
||||
writer.WriteByte((byte)ConstantTag.Methodref); writer.WriteUInt16(11); writer.WriteUInt16(18); // #19
|
||||
|
||||
writer.WriteUInt16(0x0001); // public
|
||||
writer.WriteUInt16(2); // this class
|
||||
@@ -76,7 +78,7 @@ public static class JavaClassFileFactory
|
||||
writer.WriteUInt16(0); // fields
|
||||
writer.WriteUInt16(1); // methods
|
||||
|
||||
WriteResourceLookupMethod(writer, methodNameIndex: 5, descriptorIndex: 6, classConstantIndex: 4, stringIndex: 9, methodRefIndex: 15);
|
||||
WriteResourceLookupMethod(writer, methodNameIndex: 5, descriptorIndex: 6, systemLoaderMethodRefIndex: 15, stringIndex: 9, getResourceMethodRefIndex: 19);
|
||||
|
||||
writer.WriteUInt16(0); // class attributes
|
||||
|
||||
@@ -188,7 +190,13 @@ public static class JavaClassFileFactory
|
||||
writer.WriteBytes(codeBytes);
|
||||
}
|
||||
|
||||
private static void WriteResourceLookupMethod(BigEndianWriter writer, ushort methodNameIndex, ushort descriptorIndex, ushort classConstantIndex, ushort stringIndex, ushort methodRefIndex)
|
||||
private static void WriteResourceLookupMethod(
|
||||
BigEndianWriter writer,
|
||||
ushort methodNameIndex,
|
||||
ushort descriptorIndex,
|
||||
ushort systemLoaderMethodRefIndex,
|
||||
ushort stringIndex,
|
||||
ushort getResourceMethodRefIndex)
|
||||
{
|
||||
writer.WriteUInt16(0x0009);
|
||||
writer.WriteUInt16(methodNameIndex);
|
||||
@@ -201,13 +209,13 @@ public static class JavaClassFileFactory
|
||||
{
|
||||
codeWriter.WriteUInt16(2);
|
||||
codeWriter.WriteUInt16(0);
|
||||
codeWriter.WriteUInt32(8);
|
||||
codeWriter.WriteByte(0x13); // ldc_w for class literal
|
||||
codeWriter.WriteUInt16(classConstantIndex);
|
||||
codeWriter.WriteByte(0x12);
|
||||
codeWriter.WriteUInt32(10);
|
||||
codeWriter.WriteByte(0xB8); // invokestatic
|
||||
codeWriter.WriteUInt16(systemLoaderMethodRefIndex);
|
||||
codeWriter.WriteByte(0x12); // ldc
|
||||
codeWriter.WriteByte((byte)stringIndex);
|
||||
codeWriter.WriteByte(0xB6);
|
||||
codeWriter.WriteUInt16(methodRefIndex);
|
||||
codeWriter.WriteByte(0xB6); // invokevirtual
|
||||
codeWriter.WriteUInt16(getResourceMethodRefIndex);
|
||||
codeWriter.WriteByte(0x57);
|
||||
codeWriter.WriteByte(0xB1);
|
||||
codeWriter.WriteUInt16(0);
|
||||
|
||||
@@ -20,6 +20,7 @@ using StellaOps.Scanner.Surface.Validation;
|
||||
using StellaOps.Scanner.Worker.Options;
|
||||
using StellaOps.Scanner.Worker.Processing;
|
||||
using StellaOps.Scanner.Worker.Tests.TestInfrastructure;
|
||||
using StellaOps.Cryptography;
|
||||
using Xunit;
|
||||
|
||||
namespace StellaOps.Scanner.Worker.Tests;
|
||||
|
||||
@@ -8,7 +8,9 @@ using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using System.Security.Cryptography;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using Microsoft.Extensions.Options;
|
||||
using StellaOps.Scanner.Core.Contracts;
|
||||
using StellaOps.Scanner.EntryTrace;
|
||||
using StellaOps.Scanner.Surface.Env;
|
||||
@@ -35,7 +37,7 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
using var listener = new WorkerMeterListener();
|
||||
listener.Start();
|
||||
|
||||
var hash = new DefaultCryptoHash();
|
||||
var hash = CreateCryptoHash();
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
cache,
|
||||
@@ -71,7 +73,7 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
using var listener = new WorkerMeterListener();
|
||||
listener.Start();
|
||||
|
||||
var hash = new DefaultCryptoHash();
|
||||
var hash = CreateCryptoHash();
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
cache,
|
||||
@@ -163,7 +165,7 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
var publisher = new TestSurfaceManifestPublisher("tenant-a");
|
||||
var cache = new RecordingSurfaceCache();
|
||||
var environment = new TestSurfaceEnvironment("tenant-a");
|
||||
var hash = new DefaultCryptoHash();
|
||||
var hash = CreateCryptoHash();
|
||||
var executor = new SurfaceManifestStageExecutor(
|
||||
publisher,
|
||||
cache,
|
||||
@@ -193,7 +195,8 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
await executor.ExecuteAsync(context, CancellationToken.None);
|
||||
|
||||
Assert.Equal(1, publisher.PublishCalls);
|
||||
var request = Assert.NotNull(publisher.LastRequest);
|
||||
Assert.NotNull(publisher.LastRequest);
|
||||
var request = publisher.LastRequest!;
|
||||
Assert.Contains(request.Payloads, payload => payload.Kind == "deno.observation");
|
||||
Assert.Contains(cache.Entries.Keys, key => key.Namespace == "surface.artifacts.deno.observation");
|
||||
}
|
||||
@@ -361,6 +364,32 @@ public sealed class SurfaceManifestStageExecutorTests
|
||||
public IReadOnlyDictionary<string, string> RawVariables { get; } = new Dictionary<string, string>();
|
||||
}
|
||||
|
||||
private static ICryptoHash CreateCryptoHash()
|
||||
=> new DefaultCryptoHash(new StaticOptionsMonitor<CryptoHashOptions>(new CryptoHashOptions()), NullLogger<DefaultCryptoHash>.Instance);
|
||||
|
||||
private sealed class StaticOptionsMonitor<T> : IOptionsMonitor<T>
|
||||
{
|
||||
public StaticOptionsMonitor(T value)
|
||||
{
|
||||
CurrentValue = value;
|
||||
}
|
||||
|
||||
public T CurrentValue { get; }
|
||||
|
||||
public T Get(string? name) => CurrentValue;
|
||||
|
||||
public IDisposable OnChange(Action<T, string?> listener) => Disposable.Instance;
|
||||
|
||||
private sealed class Disposable : IDisposable
|
||||
{
|
||||
public static readonly Disposable Instance = new();
|
||||
|
||||
public void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private sealed class FakeJobLease : IScanJobLease
|
||||
{
|
||||
private readonly Dictionary<string, string> _metadata = new()
|
||||
|
||||
@@ -16,6 +16,15 @@ public sealed class CallgraphArtifactMetadata
|
||||
[BsonElement("casUri")]
|
||||
public string CasUri { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("manifestPath")]
|
||||
public string ManifestPath { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("manifestCasUri")]
|
||||
public string ManifestCasUri { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("graphHash")]
|
||||
public string GraphHash { get; set; } = string.Empty;
|
||||
|
||||
[BsonElement("contentType")]
|
||||
public string ContentType { get; set; } = string.Empty;
|
||||
|
||||
|
||||
@@ -35,7 +35,10 @@ public sealed class CallgraphDocument
|
||||
[BsonElement("edges")]
|
||||
public List<CallgraphEdge> Edges { get; set; } = new();
|
||||
|
||||
[BsonElement("metadata")]
|
||||
[BsonIgnoreIfNull]
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
}
|
||||
[BsonElement("metadata")]
|
||||
[BsonIgnoreIfNull]
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
|
||||
[BsonElement("graphHash")]
|
||||
public string GraphHash { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
@@ -7,4 +7,6 @@ public sealed record CallgraphIngestResponse(
|
||||
string CallgraphId,
|
||||
string ArtifactPath,
|
||||
string ArtifactHash,
|
||||
string CasUri);
|
||||
string CasUri,
|
||||
string GraphHash,
|
||||
string ManifestCasUri);
|
||||
|
||||
31
src/Signals/StellaOps.Signals/Models/CallgraphManifest.cs
Normal file
31
src/Signals/StellaOps.Signals/Models/CallgraphManifest.cs
Normal file
@@ -0,0 +1,31 @@
|
||||
using System;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
public sealed class CallgraphManifest
|
||||
{
|
||||
[JsonPropertyName("language")]
|
||||
public string Language { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("component")]
|
||||
public string Component { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("graphHash")]
|
||||
public string GraphHash { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("artifactHash")]
|
||||
public string ArtifactHash { get; set; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("nodeCount")]
|
||||
public int NodeCount { get; set; }
|
||||
|
||||
[JsonPropertyName("edgeCount")]
|
||||
public int EdgeCount { get; set; }
|
||||
|
||||
[JsonPropertyName("createdAt")]
|
||||
public DateTimeOffset CreatedAt { get; set; }
|
||||
}
|
||||
@@ -114,6 +114,26 @@ public sealed class RuntimeFactDocument
|
||||
[BsonIgnoreIfNull]
|
||||
public string? LoaderBase { get; set; }
|
||||
|
||||
[BsonElement("processId")]
|
||||
[BsonIgnoreIfNull]
|
||||
public int? ProcessId { get; set; }
|
||||
|
||||
[BsonElement("processName")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? ProcessName { get; set; }
|
||||
|
||||
[BsonElement("socketAddress")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? SocketAddress { get; set; }
|
||||
|
||||
[BsonElement("containerId")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? ContainerId { get; set; }
|
||||
|
||||
[BsonElement("evidenceUri")]
|
||||
[BsonIgnoreIfNull]
|
||||
public string? EvidenceUri { get; set; }
|
||||
|
||||
[BsonElement("hitCount")]
|
||||
public int HitCount { get; set; }
|
||||
|
||||
|
||||
@@ -26,6 +26,16 @@ public sealed class RuntimeFactEvent
|
||||
|
||||
public string? LoaderBase { get; set; }
|
||||
|
||||
public int? ProcessId { get; set; }
|
||||
|
||||
public string? ProcessName { get; set; }
|
||||
|
||||
public string? SocketAddress { get; set; }
|
||||
|
||||
public string? ContainerId { get; set; }
|
||||
|
||||
public string? EvidenceUri { get; set; }
|
||||
|
||||
public int HitCount { get; set; } = 1;
|
||||
|
||||
public Dictionary<string, string?>? Metadata { get; set; }
|
||||
|
||||
@@ -1,14 +1,29 @@
|
||||
using Microsoft.AspNetCore.Mvc;
|
||||
|
||||
namespace StellaOps.Signals.Models;
|
||||
|
||||
public sealed class RuntimeFactsStreamMetadata
|
||||
{
|
||||
[FromQuery(Name = "callgraphId")]
|
||||
public string CallgraphId { get; set; } = string.Empty;
|
||||
|
||||
[FromQuery(Name = "scanId")]
|
||||
public string? ScanId { get; set; }
|
||||
|
||||
[FromQuery(Name = "imageDigest")]
|
||||
public string? ImageDigest { get; set; }
|
||||
|
||||
[FromQuery(Name = "component")]
|
||||
public string? Component { get; set; }
|
||||
|
||||
[FromQuery(Name = "version")]
|
||||
public string? Version { get; set; }
|
||||
|
||||
public ReachabilitySubject ToSubject() => new()
|
||||
{
|
||||
ScanId = ScanId,
|
||||
ImageDigest = ImageDigest,
|
||||
Component = Component,
|
||||
Version = Version
|
||||
};
|
||||
}
|
||||
|
||||
@@ -206,15 +206,22 @@ app.MapGet("/readyz", (SignalsStartupState state, SignalsSealedModeMonitor seale
|
||||
: Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}).AllowAnonymous();
|
||||
|
||||
var fallbackAllowed = !bootstrap.Authority.Enabled || bootstrap.Authority.AllowAnonymousFallback;
|
||||
|
||||
var signalsGroup = app.MapGroup("/signals");
|
||||
|
||||
signalsGroup.MapGet("/ping", (HttpContext context, SignalsOptions options, SignalsSealedModeMonitor sealedModeMonitor) =>
|
||||
Program.TryAuthorize(context, requiredScope: SignalsPolicies.Read, fallbackAllowed: options.Authority.AllowAnonymousFallback, out var authFailure) &&
|
||||
Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure)
|
||||
? Results.NoContent()
|
||||
: authFailure ?? sealedFailure ?? Results.Unauthorized()).WithName("SignalsPing");
|
||||
{
|
||||
if (!Program.TryAuthorize(context, requiredScope: SignalsPolicies.Read, fallbackAllowed: options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
return Results.NoContent();
|
||||
}).WithName("SignalsPing");
|
||||
|
||||
signalsGroup.MapGet("/status", (HttpContext context, SignalsOptions options, SignalsSealedModeMonitor sealedModeMonitor) =>
|
||||
{
|
||||
@@ -245,33 +252,37 @@ signalsGroup.MapPost("/callgraphs", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var result = await ingestionService.IngestAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Accepted($"/signals/callgraphs/{result.CallgraphId}", result);
|
||||
}
|
||||
catch (CallgraphIngestionValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
catch (CallgraphParserNotFoundException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
catch (CallgraphParserValidationException ex)
|
||||
{
|
||||
return Results.UnprocessableEntity(new { error = ex.Message });
|
||||
}
|
||||
catch (FormatException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var result = await ingestionService.IngestAsync(request, cancellationToken).ConfigureAwait(false);
|
||||
return Results.Accepted($"/signals/callgraphs/{result.CallgraphId}", result);
|
||||
}
|
||||
catch (CallgraphIngestionValidationException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
catch (CallgraphParserNotFoundException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
catch (CallgraphParserValidationException ex)
|
||||
{
|
||||
return Results.UnprocessableEntity(new { error = ex.Message });
|
||||
}
|
||||
catch (FormatException ex)
|
||||
{
|
||||
return Results.BadRequest(new { error = ex.Message });
|
||||
}
|
||||
}).WithName("SignalsCallgraphIngest");
|
||||
|
||||
signalsGroup.MapGet("/callgraphs/{callgraphId}", async Task<IResult> (
|
||||
@@ -282,10 +293,14 @@ signalsGroup.MapGet("/callgraphs/{callgraphId}", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(callgraphId))
|
||||
@@ -296,7 +311,46 @@ signalsGroup.MapGet("/callgraphs/{callgraphId}", async Task<IResult> (
|
||||
var document = await callgraphRepository.GetByIdAsync(callgraphId.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
return document is null ? Results.NotFound() : Results.Ok(document);
|
||||
}).WithName("SignalsCallgraphGet");
|
||||
|
||||
|
||||
signalsGroup.MapGet("/callgraphs/{callgraphId}/manifest", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
string callgraphId,
|
||||
ICallgraphRepository callgraphRepository,
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(callgraphId))
|
||||
{
|
||||
return Results.BadRequest(new { error = "callgraphId is required." });
|
||||
}
|
||||
|
||||
var document = await callgraphRepository.GetByIdAsync(callgraphId.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
if (document is null || string.IsNullOrWhiteSpace(document.Artifact.ManifestPath))
|
||||
{
|
||||
return Results.NotFound();
|
||||
}
|
||||
|
||||
var manifestPath = Path.Combine(options.Storage.RootPath, document.Artifact.ManifestPath);
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
return Results.NotFound(new { error = "manifest not found" });
|
||||
}
|
||||
|
||||
var bytes = await File.ReadAllBytesAsync(manifestPath, cancellationToken).ConfigureAwait(false);
|
||||
return Results.File(bytes, "application/json");
|
||||
}).WithName("SignalsCallgraphManifestGet");
|
||||
|
||||
signalsGroup.MapPost("/runtime-facts", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
@@ -305,10 +359,14 @@ signalsGroup.MapPost("/runtime-facts", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
try
|
||||
@@ -325,15 +383,19 @@ signalsGroup.MapPost("/runtime-facts", async Task<IResult> (
|
||||
signalsGroup.MapPost("/runtime-facts/ndjson", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
RuntimeFactsStreamMetadata metadata,
|
||||
[AsParameters] RuntimeFactsStreamMetadata metadata,
|
||||
IRuntimeFactsIngestionService ingestionService,
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Write, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (metadata is null || string.IsNullOrWhiteSpace(metadata.CallgraphId))
|
||||
@@ -341,13 +403,7 @@ signalsGroup.MapPost("/runtime-facts/ndjson", async Task<IResult> (
|
||||
return Results.BadRequest(new { error = "callgraphId is required." });
|
||||
}
|
||||
|
||||
var subject = new ReachabilitySubject
|
||||
{
|
||||
ScanId = metadata.ScanId,
|
||||
ImageDigest = metadata.ImageDigest,
|
||||
Component = metadata.Component,
|
||||
Version = metadata.Version
|
||||
};
|
||||
var subject = metadata.ToSubject();
|
||||
|
||||
var isGzip = string.Equals(context.Request.Headers.ContentEncoding, "gzip", StringComparison.OrdinalIgnoreCase);
|
||||
var events = await RuntimeFactsNdjsonReader.ReadAsync(context.Request.Body, isGzip, cancellationToken).ConfigureAwait(false);
|
||||
@@ -382,10 +438,14 @@ signalsGroup.MapGet("/facts/{subjectKey}", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
@@ -405,10 +465,14 @@ signalsGroup.MapPost("/reachability/recompute", async Task<IResult> (
|
||||
SignalsSealedModeMonitor sealedModeMonitor,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Admin, options.Authority.AllowAnonymousFallback, out var authFailure) ||
|
||||
!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Admin, options.Authority.AllowAnonymousFallback, out var authFailure))
|
||||
{
|
||||
return authFailure ?? sealedFailure ?? Results.Unauthorized();
|
||||
return authFailure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (!Program.TryEnsureSealedMode(sealedModeMonitor, out var sealedFailure))
|
||||
{
|
||||
return sealedFailure ?? Results.StatusCode(StatusCodes.Status503ServiceUnavailable);
|
||||
}
|
||||
|
||||
try
|
||||
@@ -434,26 +498,6 @@ signalsGroup.MapPost("/reachability/recompute", async Task<IResult> (
|
||||
}
|
||||
}).WithName("SignalsReachabilityRecompute");
|
||||
|
||||
signalsGroup.MapGet("/facts/{subjectKey}", async Task<IResult> (
|
||||
HttpContext context,
|
||||
SignalsOptions options,
|
||||
string subjectKey,
|
||||
IReachabilityFactRepository factRepository,
|
||||
CancellationToken cancellationToken) =>
|
||||
{
|
||||
if (!Program.TryAuthorize(context, SignalsPolicies.Read, options.Authority.AllowAnonymousFallback, out var failure))
|
||||
{
|
||||
return failure ?? Results.Unauthorized();
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
return Results.BadRequest(new { error = "subjectKey is required." });
|
||||
}
|
||||
|
||||
var fact = await factRepository.GetBySubjectAsync(subjectKey.Trim(), cancellationToken).ConfigureAwait(false);
|
||||
return fact is null ? Results.NotFound() : Results.Ok(fact);
|
||||
}).WithName("SignalsFactsGet");
|
||||
|
||||
app.Run();
|
||||
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Linq;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using Microsoft.Extensions.Logging;
|
||||
@@ -26,10 +28,11 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
|
||||
private readonly ICallgraphParserResolver parserResolver;
|
||||
private readonly ICallgraphArtifactStore artifactStore;
|
||||
private readonly ICallgraphRepository repository;
|
||||
private readonly ILogger<CallgraphIngestionService> logger;
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private readonly ICallgraphRepository repository;
|
||||
private readonly ILogger<CallgraphIngestionService> logger;
|
||||
private readonly SignalsOptions options;
|
||||
private readonly TimeProvider timeProvider;
|
||||
private static readonly JsonSerializerOptions ManifestSerializerOptions = new(JsonSerializerDefaults.Web);
|
||||
|
||||
public CallgraphIngestionService(
|
||||
ICallgraphParserResolver parserResolver,
|
||||
@@ -53,23 +56,42 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
|
||||
var parser = parserResolver.Resolve(request.Language);
|
||||
|
||||
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
|
||||
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
|
||||
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
parseStream.Position = 0;
|
||||
var hash = ComputeSha256(artifactBytes);
|
||||
|
||||
var artifactMetadata = await artifactStore.SaveAsync(
|
||||
new CallgraphArtifactSaveRequest(
|
||||
request.Language,
|
||||
request.Component,
|
||||
request.Version,
|
||||
request.ArtifactFileName,
|
||||
request.ArtifactContentType,
|
||||
hash),
|
||||
parseStream,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
var artifactBytes = Convert.FromBase64String(request.ArtifactContentBase64);
|
||||
await using var parseStream = new MemoryStream(artifactBytes, writable: false);
|
||||
var parseResult = await parser.ParseAsync(parseStream, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
parseStream.Position = 0;
|
||||
var artifactHash = ComputeSha256(artifactBytes);
|
||||
var graphHash = ComputeGraphHash(parseResult);
|
||||
|
||||
var manifest = new CallgraphManifest
|
||||
{
|
||||
Language = request.Language,
|
||||
Component = request.Component,
|
||||
Version = request.Version,
|
||||
ArtifactHash = artifactHash,
|
||||
GraphHash = graphHash,
|
||||
NodeCount = parseResult.Nodes.Count,
|
||||
EdgeCount = parseResult.Edges.Count,
|
||||
CreatedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
await using var manifestStream = new MemoryStream();
|
||||
await JsonSerializer.SerializeAsync(manifestStream, manifest, ManifestSerializerOptions, cancellationToken).ConfigureAwait(false);
|
||||
manifestStream.Position = 0;
|
||||
|
||||
parseStream.Position = 0;
|
||||
var artifactMetadata = await artifactStore.SaveAsync(
|
||||
new CallgraphArtifactSaveRequest(
|
||||
request.Language,
|
||||
request.Component,
|
||||
request.Version,
|
||||
request.ArtifactFileName,
|
||||
request.ArtifactContentType,
|
||||
artifactHash,
|
||||
manifestStream),
|
||||
parseStream,
|
||||
cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var document = new CallgraphDocument
|
||||
{
|
||||
@@ -81,21 +103,25 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
Metadata = request.Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string?>(request.Metadata, StringComparer.OrdinalIgnoreCase),
|
||||
Artifact = new CallgraphArtifactMetadata
|
||||
{
|
||||
Path = artifactMetadata.Path,
|
||||
Hash = artifactMetadata.Hash,
|
||||
CasUri = artifactMetadata.CasUri,
|
||||
ContentType = artifactMetadata.ContentType,
|
||||
Length = artifactMetadata.Length
|
||||
},
|
||||
IngestedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
document.Metadata["formatVersion"] = parseResult.FormatVersion;
|
||||
|
||||
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
Artifact = new CallgraphArtifactMetadata
|
||||
{
|
||||
Path = artifactMetadata.Path,
|
||||
Hash = artifactMetadata.Hash,
|
||||
CasUri = artifactMetadata.CasUri,
|
||||
ManifestPath = artifactMetadata.ManifestPath,
|
||||
ManifestCasUri = artifactMetadata.ManifestCasUri,
|
||||
GraphHash = graphHash,
|
||||
ContentType = artifactMetadata.ContentType,
|
||||
Length = artifactMetadata.Length
|
||||
},
|
||||
IngestedAt = timeProvider.GetUtcNow()
|
||||
};
|
||||
|
||||
document.Metadata ??= new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
document.Metadata["formatVersion"] = parseResult.FormatVersion;
|
||||
document.GraphHash = graphHash;
|
||||
|
||||
document = await repository.UpsertAsync(document, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
logger.LogInformation(
|
||||
"Ingested callgraph {Language}:{Component}:{Version} (id={Id}) with {NodeCount} nodes and {EdgeCount} edges.",
|
||||
@@ -106,7 +132,13 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
document.Nodes.Count,
|
||||
document.Edges.Count);
|
||||
|
||||
return new CallgraphIngestResponse(document.Id, document.Artifact.Path, document.Artifact.Hash, document.Artifact.CasUri);
|
||||
return new CallgraphIngestResponse(
|
||||
document.Id,
|
||||
document.Artifact.Path,
|
||||
document.Artifact.Hash,
|
||||
document.Artifact.CasUri,
|
||||
graphHash,
|
||||
document.Artifact.ManifestCasUri);
|
||||
}
|
||||
|
||||
private static void ValidateRequest(CallgraphIngestRequest request)
|
||||
@@ -144,13 +176,29 @@ internal sealed class CallgraphIngestionService : ICallgraphIngestionService
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
SHA256.HashData(buffer, hash);
|
||||
return Convert.ToHexString(hash);
|
||||
}
|
||||
}
|
||||
private static string ComputeSha256(ReadOnlySpan<byte> buffer)
|
||||
{
|
||||
Span<byte> hash = stackalloc byte[SHA256.HashSizeInBytes];
|
||||
SHA256.HashData(buffer, hash);
|
||||
return Convert.ToHexString(hash);
|
||||
}
|
||||
|
||||
private static string ComputeGraphHash(CallgraphParseResult result)
|
||||
{
|
||||
var builder = new StringBuilder();
|
||||
foreach (var node in result.Nodes.OrderBy(n => n.Id, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append(node.Id).Append('|').Append(node.Name).AppendLine();
|
||||
}
|
||||
|
||||
foreach (var edge in result.Edges.OrderBy(e => e.SourceId, StringComparer.Ordinal).ThenBy(e => e.TargetId, StringComparer.Ordinal))
|
||||
{
|
||||
builder.Append(edge.SourceId).Append("->").Append(edge.TargetId).AppendLine();
|
||||
}
|
||||
|
||||
return ComputeSha256(Encoding.UTF8.GetBytes(builder.ToString()));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Exception thrown when the ingestion request is invalid.
|
||||
|
||||
@@ -40,6 +40,14 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
|
||||
ValidateRequest(request);
|
||||
|
||||
var subjectKey = request.Subject?.ToSubjectKey();
|
||||
if (string.IsNullOrWhiteSpace(subjectKey))
|
||||
{
|
||||
throw new ReachabilityScoringValidationException("Subject must include scanId, imageDigest, or component/version.");
|
||||
}
|
||||
|
||||
var existingFact = await factRepository.GetBySubjectAsync(subjectKey, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var callgraph = await callgraphRepository.GetByIdAsync(request.CallgraphId, cancellationToken).ConfigureAwait(false);
|
||||
if (callgraph is null)
|
||||
{
|
||||
@@ -57,10 +65,24 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
throw new ReachabilityScoringValidationException("At least one target symbol is required.");
|
||||
}
|
||||
|
||||
var runtimeHits = request.RuntimeHits?.Where(hit => !string.IsNullOrWhiteSpace(hit))
|
||||
.Select(hit => hit.Trim())
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToList() ?? new List<string>();
|
||||
var runtimeHitSet = new HashSet<string>(StringComparer.Ordinal);
|
||||
if (existingFact?.RuntimeFacts is { Count: > 0 })
|
||||
{
|
||||
foreach (var fact in existingFact.RuntimeFacts.Where(f => !string.IsNullOrWhiteSpace(f.SymbolId)))
|
||||
{
|
||||
runtimeHitSet.Add(fact.SymbolId);
|
||||
}
|
||||
}
|
||||
|
||||
if (request.RuntimeHits is { Count: > 0 })
|
||||
{
|
||||
foreach (var hit in request.RuntimeHits.Where(h => !string.IsNullOrWhiteSpace(h)))
|
||||
{
|
||||
runtimeHitSet.Add(hit.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
var runtimeHits = runtimeHitSet.ToList();
|
||||
|
||||
var states = new List<ReachabilityStateDocument>(targets.Count);
|
||||
foreach (var target in targets)
|
||||
@@ -95,12 +117,13 @@ public sealed class ReachabilityScoringService : IReachabilityScoringService
|
||||
var document = new ReachabilityFactDocument
|
||||
{
|
||||
CallgraphId = request.CallgraphId,
|
||||
Subject = request.Subject,
|
||||
Subject = request.Subject!,
|
||||
EntryPoints = entryPoints,
|
||||
States = states,
|
||||
Metadata = request.Metadata,
|
||||
ComputedAt = timeProvider.GetUtcNow(),
|
||||
SubjectKey = request.Subject.ToSubjectKey()
|
||||
SubjectKey = subjectKey,
|
||||
RuntimeFacts = existingFact?.RuntimeFacts
|
||||
};
|
||||
|
||||
logger.LogInformation("Computed reachability fact for subject {SubjectKey} with {StateCount} targets.", document.SubjectKey, states.Count);
|
||||
|
||||
@@ -115,6 +115,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
SymbolId = key.SymbolId,
|
||||
CodeId = key.CodeId,
|
||||
LoaderBase = key.LoaderBase,
|
||||
ProcessId = evt.ProcessId,
|
||||
ProcessName = Normalize(evt.ProcessName),
|
||||
SocketAddress = Normalize(evt.SocketAddress),
|
||||
ContainerId = Normalize(evt.ContainerId),
|
||||
EvidenceUri = Normalize(evt.EvidenceUri),
|
||||
Metadata = evt.Metadata != null
|
||||
? new Dictionary<string, string?>(evt.Metadata, StringComparer.Ordinal)
|
||||
: null
|
||||
@@ -152,9 +157,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
|
||||
if (incoming != null)
|
||||
{
|
||||
foreach (var (key, value) in incoming)
|
||||
foreach (var (metaKey, metaValue) in incoming)
|
||||
{
|
||||
merged[key] = value;
|
||||
merged[metaKey] = metaValue;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -177,6 +182,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
SymbolId = fact.SymbolId,
|
||||
CodeId = fact.CodeId,
|
||||
LoaderBase = fact.LoaderBase,
|
||||
ProcessId = fact.ProcessId,
|
||||
ProcessName = fact.ProcessName,
|
||||
SocketAddress = fact.SocketAddress,
|
||||
ContainerId = fact.ContainerId,
|
||||
EvidenceUri = fact.EvidenceUri,
|
||||
HitCount = fact.HitCount,
|
||||
Metadata = fact.Metadata is null
|
||||
? null
|
||||
@@ -197,6 +207,11 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
SymbolId = fact.SymbolId,
|
||||
CodeId = fact.CodeId,
|
||||
LoaderBase = fact.LoaderBase,
|
||||
ProcessId = fact.ProcessId,
|
||||
ProcessName = fact.ProcessName,
|
||||
SocketAddress = fact.SocketAddress,
|
||||
ContainerId = fact.ContainerId,
|
||||
EvidenceUri = fact.EvidenceUri,
|
||||
HitCount = fact.HitCount,
|
||||
Metadata = fact.Metadata is null
|
||||
? null
|
||||
@@ -206,12 +221,17 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
}
|
||||
|
||||
existingFact.HitCount = Math.Clamp(existingFact.HitCount + fact.HitCount, 1, int.MaxValue);
|
||||
existingFact.ProcessId ??= fact.ProcessId;
|
||||
existingFact.ProcessName ??= fact.ProcessName;
|
||||
existingFact.SocketAddress ??= fact.SocketAddress;
|
||||
existingFact.ContainerId ??= fact.ContainerId;
|
||||
existingFact.EvidenceUri ??= fact.EvidenceUri;
|
||||
if (fact.Metadata != null && fact.Metadata.Count > 0)
|
||||
{
|
||||
existingFact.Metadata ??= new Dictionary<string, string?>(StringComparer.Ordinal);
|
||||
foreach (var (key, value) in fact.Metadata)
|
||||
foreach (var (metaKey, metaValue) in fact.Metadata)
|
||||
{
|
||||
existingFact.Metadata[key] = value;
|
||||
existingFact.Metadata[metaKey] = metaValue;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -224,6 +244,9 @@ public sealed class RuntimeFactsIngestionService : IRuntimeFactsIngestionService
|
||||
.ToList();
|
||||
}
|
||||
|
||||
private static string? Normalize(string? value) =>
|
||||
string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
|
||||
private readonly record struct RuntimeFactKey(string SymbolId, string? CodeId, string? LoaderBase);
|
||||
|
||||
private sealed class RuntimeFactKeyComparer : IEqualityComparer<RuntimeFactKey>
|
||||
|
||||
@@ -48,15 +48,29 @@ internal sealed class FileSystemCallgraphArtifactStore : ICallgraphArtifactStore
|
||||
await content.CopyToAsync(fileStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var manifestPath = Path.Combine(casDirectory, "manifest.json");
|
||||
if (request.ManifestContent != null)
|
||||
{
|
||||
await using var manifestStream = File.Create(manifestPath);
|
||||
request.ManifestContent.Position = 0;
|
||||
await request.ManifestContent.CopyToAsync(manifestStream, cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
else if (!File.Exists(manifestPath))
|
||||
{
|
||||
await File.WriteAllTextAsync(manifestPath, "{}", cancellationToken).ConfigureAwait(false);
|
||||
}
|
||||
|
||||
var fileInfo = new FileInfo(destinationPath);
|
||||
logger.LogInformation("Stored callgraph artifact at {Path} (length={Length}).", destinationPath, fileInfo.Length);
|
||||
|
||||
return new StoredCallgraphArtifact(
|
||||
Path.GetRelativePath(root, destinationPath),
|
||||
fileInfo.Length,
|
||||
request.Hash,
|
||||
hash,
|
||||
request.ContentType,
|
||||
$"cas://reachability/graphs/{hash}");
|
||||
$"cas://reachability/graphs/{hash}",
|
||||
Path.GetRelativePath(root, manifestPath),
|
||||
$"cas://reachability/graphs/{hash}/manifest");
|
||||
}
|
||||
|
||||
private static string SanitizeFileName(string value)
|
||||
|
||||
@@ -1,12 +1,15 @@
|
||||
namespace StellaOps.Signals.Storage.Models;
|
||||
using System.IO;
|
||||
|
||||
namespace StellaOps.Signals.Storage.Models;
|
||||
|
||||
/// <summary>
|
||||
/// Context required to persist a callgraph artifact.
|
||||
/// </summary>
|
||||
public sealed record CallgraphArtifactSaveRequest(
|
||||
string Language,
|
||||
string Component,
|
||||
string Version,
|
||||
string FileName,
|
||||
string ContentType,
|
||||
string Hash);
|
||||
public sealed record CallgraphArtifactSaveRequest(
|
||||
string Language,
|
||||
string Component,
|
||||
string Version,
|
||||
string FileName,
|
||||
string ContentType,
|
||||
string Hash,
|
||||
Stream? ManifestContent);
|
||||
|
||||
@@ -8,4 +8,6 @@ public sealed record StoredCallgraphArtifact(
|
||||
long Length,
|
||||
string Hash,
|
||||
string ContentType,
|
||||
string CasUri);
|
||||
string CasUri,
|
||||
string ManifestPath,
|
||||
string ManifestCasUri);
|
||||
|
||||
@@ -28,21 +28,21 @@ public class CallgraphIngestionTests : IClassFixture<SignalsTestFactory>
|
||||
[InlineData("go")]
|
||||
public async Task Ingest_Callgraph_PersistsDocumentAndArtifact(string language)
|
||||
{
|
||||
using var client = factory.CreateClient();
|
||||
client.DefaultRequestHeaders.Add("X-Scopes", "signals:write");
|
||||
using var client = factory.CreateClient();
|
||||
client.DefaultRequestHeaders.Add("X-Scopes", "signals:write signals:read");
|
||||
|
||||
var component = $"demo-{language}";
|
||||
var request = CreateRequest(language, component: component);
|
||||
var response = await client.PostAsJsonAsync("/signals/callgraphs", request);
|
||||
|
||||
Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
|
||||
var body = await response.Content.ReadFromJsonAsync<CallgraphIngestResponse>();
|
||||
Assert.NotNull(body);
|
||||
|
||||
var database = new MongoClient(factory.MongoRunner.ConnectionString).GetDatabase("signals-tests");
|
||||
var collection = database.GetCollection<CallgraphDocument>("callgraphs");
|
||||
var doc = await collection.Find(d => d.Id == body!.CallgraphId).FirstOrDefaultAsync();
|
||||
|
||||
Assert.Equal(HttpStatusCode.Accepted, response.StatusCode);
|
||||
var body = await response.Content.ReadFromJsonAsync<CallgraphIngestResponse>();
|
||||
Assert.NotNull(body);
|
||||
|
||||
var database = new MongoClient(factory.MongoRunner.ConnectionString).GetDatabase("signals-tests");
|
||||
var collection = database.GetCollection<CallgraphDocument>("callgraphs");
|
||||
var doc = await collection.Find(d => d.Id == body!.CallgraphId).FirstOrDefaultAsync();
|
||||
|
||||
Assert.NotNull(doc);
|
||||
Assert.Equal(language, doc!.Language);
|
||||
Assert.Equal(component, doc.Component);
|
||||
@@ -50,10 +50,20 @@ public class CallgraphIngestionTests : IClassFixture<SignalsTestFactory>
|
||||
Assert.Equal(2, doc.Nodes.Count);
|
||||
Assert.Equal(1, doc.Edges.Count);
|
||||
|
||||
var artifactPath = Path.Combine(factory.StoragePath, body.ArtifactPath);
|
||||
Assert.True(File.Exists(artifactPath));
|
||||
Assert.False(string.IsNullOrWhiteSpace(body.ArtifactHash));
|
||||
}
|
||||
var artifactPath = Path.Combine(factory.StoragePath, body.ArtifactPath);
|
||||
Assert.True(File.Exists(artifactPath));
|
||||
Assert.False(string.IsNullOrWhiteSpace(body.ArtifactHash));
|
||||
Assert.False(string.IsNullOrWhiteSpace(body.CasUri));
|
||||
Assert.False(string.IsNullOrWhiteSpace(body.GraphHash));
|
||||
Assert.False(string.IsNullOrWhiteSpace(body.ManifestCasUri));
|
||||
Assert.Equal(body.GraphHash, doc.GraphHash);
|
||||
|
||||
var manifestResponse = await client.GetAsync($"/signals/callgraphs/{body.CallgraphId}/manifest");
|
||||
Assert.Equal(HttpStatusCode.OK, manifestResponse.StatusCode);
|
||||
var manifest = await manifestResponse.Content.ReadFromJsonAsync<CallgraphManifest>();
|
||||
Assert.NotNull(manifest);
|
||||
Assert.Equal(body.GraphHash, manifest!.GraphHash);
|
||||
}
|
||||
|
||||
[Fact]
|
||||
public async Task Ingest_UnsupportedLanguage_ReturnsBadRequest()
|
||||
|
||||
@@ -202,11 +202,17 @@ public sealed class ScannerToSignalsReachabilityTests
|
||||
throw new InvalidOperationException($"Hash mismatch for {request.FileName}: expected {request.Hash} but computed {computedHash}.");
|
||||
}
|
||||
|
||||
var casUri = $"cas://fixtures/{request.Component}/{request.Version}/{computedHash}";
|
||||
var manifestPath = $"cas://fixtures/{request.Component}/{request.Version}/{computedHash}/manifest";
|
||||
|
||||
return new StoredCallgraphArtifact(
|
||||
Path: $"cas://fixtures/{request.Component}/{request.Version}/{request.FileName}",
|
||||
Path: $"fixtures/{request.Component}/{request.Version}/{request.FileName}",
|
||||
Length: bytes.Length,
|
||||
Hash: computedHash,
|
||||
ContentType: request.ContentType);
|
||||
ContentType: request.ContentType,
|
||||
CasUri: casUri,
|
||||
ManifestPath: manifestPath,
|
||||
ManifestCasUri: manifestPath);
|
||||
}
|
||||
}
|
||||
private static string LocateRepoRoot()
|
||||
|
||||
@@ -36,6 +36,10 @@ public sealed class RuntimeFactsIngestionServiceTests
|
||||
{
|
||||
SymbolId = "symbol::foo",
|
||||
HitCount = 3,
|
||||
ProcessId = 100,
|
||||
ProcessName = "worker",
|
||||
ContainerId = "ctr-1",
|
||||
SocketAddress = "10.0.0.5:443",
|
||||
Metadata = new Dictionary<string, string?> { ["thread"] = "main" }
|
||||
},
|
||||
new()
|
||||
@@ -68,6 +72,8 @@ public sealed class RuntimeFactsIngestionServiceTests
|
||||
repository.LastUpsert!.RuntimeFacts![0].SymbolId.Should().Be("symbol::bar");
|
||||
repository.LastUpsert!.RuntimeFacts![0].HitCount.Should().Be(1);
|
||||
repository.LastUpsert!.RuntimeFacts![1].SymbolId.Should().Be("symbol::foo");
|
||||
repository.LastUpsert!.RuntimeFacts![1].ProcessId.Should().Be(100);
|
||||
repository.LastUpsert!.RuntimeFacts![1].ContainerId.Should().Be("ctr-1");
|
||||
repository.LastUpsert!.RuntimeFacts![1].HitCount.Should().Be(5);
|
||||
repository.LastUpsert!.Metadata.Should().ContainKey("source");
|
||||
}
|
||||
@@ -95,8 +101,8 @@ public sealed class RuntimeFactsIngestionServiceTests
|
||||
CallgraphId = "cg-new",
|
||||
Events = new List<RuntimeFactEvent>
|
||||
{
|
||||
new() { SymbolId = "new::symbol", HitCount = 2 },
|
||||
new() { SymbolId = "old::symbol", HitCount = 3, Metadata = new Dictionary<string, string?> { ["thread"] = "main" } }
|
||||
new() { SymbolId = "new::symbol", HitCount = 2, ProcessName = "svc" },
|
||||
new() { SymbolId = "old::symbol", HitCount = 3, ProcessId = 200, Metadata = new Dictionary<string, string?> { ["thread"] = "main" } }
|
||||
}
|
||||
};
|
||||
|
||||
@@ -107,6 +113,7 @@ public sealed class RuntimeFactsIngestionServiceTests
|
||||
repository.LastUpsert!.RuntimeFacts![0].SymbolId.Should().Be("new::symbol");
|
||||
repository.LastUpsert!.RuntimeFacts![1].SymbolId.Should().Be("old::symbol");
|
||||
repository.LastUpsert!.RuntimeFacts![1].HitCount.Should().Be(4);
|
||||
repository.LastUpsert!.RuntimeFacts![1].ProcessId.Should().Be(200);
|
||||
repository.LastUpsert!.RuntimeFacts![1].Metadata.Should().ContainKey("thread").WhoseValue.Should().Be("main");
|
||||
}
|
||||
|
||||
|
||||
@@ -15,9 +15,8 @@ public sealed class RuntimeFactsNdjsonReaderTests
|
||||
public async Task ReadAsync_ParsesLines()
|
||||
{
|
||||
var ndjson = """
|
||||
{"symbolId":"sym::foo","hitCount":2}
|
||||
{"symbolId":"sym::foo","hitCount":2,"processId":10,"processName":"api"}
|
||||
{"symbolId":"sym::bar","codeId":"elf:abcd","loaderBase":"0x1000","metadata":{"thread":"bg"}}
|
||||
|
||||
""";
|
||||
|
||||
await using var stream = new MemoryStream(Encoding.UTF8.GetBytes(ndjson));
|
||||
@@ -25,6 +24,8 @@ public sealed class RuntimeFactsNdjsonReaderTests
|
||||
|
||||
events.Should().HaveCount(2);
|
||||
events[0].SymbolId.Should().Be("sym::foo");
|
||||
events[0].ProcessId.Should().Be(10);
|
||||
events[0].ProcessName.Should().Be("api");
|
||||
events[1].LoaderBase.Should().Be("0x1000");
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user