up

src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityGraphBuilder.cs

@@ -54,6 +54,21 @@ public sealed class ReachabilityGraphBuilder
         return JsonSerializer.Serialize(payload, options);
     }
 
+    public ReachabilityUnionGraph ToUnionGraph(string language)
+    {
+        ArgumentException.ThrowIfNullOrWhiteSpace(language);
+
+        var nodeList = nodes
+            .Select(id => new ReachabilityUnionNode(id, language, "symbol"))
+            .ToList();
+
+        var edgeList = edges
+            .Select(edge => new ReachabilityUnionEdge(edge.From, edge.To, edge.Kind))
+            .ToList();
+
+        return new ReachabilityUnionGraph(nodeList, edgeList);
+    }
+
     public static ReachabilityGraphBuilder FromFixture(string variantPath)
     {
         ArgumentException.ThrowIfNullOrWhiteSpace(variantPath);

src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisher.cs

@@ -0,0 +1,82 @@
+using System;
+using System.IO;
+using System.IO.Compression;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Threading;
+using System.Threading.Tasks;
+using StellaOps.Scanner.Cache.Abstractions;
+
+namespace StellaOps.Scanner.Reachability;
+
+/// <summary>
+/// Packages a reachability union graph into a deterministic zip, stores it in CAS, and returns the CAS reference.
+/// </summary>
+public sealed class ReachabilityUnionPublisher
+{
+    private readonly ReachabilityUnionWriter writer;
+
+    public ReachabilityUnionPublisher(ReachabilityUnionWriter writer)
+    {
+        this.writer = writer ?? throw new ArgumentNullException(nameof(writer));
+    }
+
+    public async Task<ReachabilityUnionPublishResult> PublishAsync(
+        ReachabilityUnionGraph graph,
+        IFileContentAddressableStore cas,
+        string workRoot,
+        string analysisId,
+        CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(graph);
+        ArgumentNullException.ThrowIfNull(cas);
+        ArgumentException.ThrowIfNullOrWhiteSpace(workRoot);
+        ArgumentException.ThrowIfNullOrWhiteSpace(analysisId);
+
+        var result = await writer.WriteAsync(graph, workRoot, analysisId, cancellationToken).ConfigureAwait(false);
+
+        var folder = Path.GetDirectoryName(result.MetaPath)!;
+        var zipPath = Path.Combine(folder, "reachability.zip");
+        CreateZip(folder, zipPath);
+
+        var sha = ComputeSha256(zipPath);
+        await using var zipStream = File.OpenRead(zipPath);
+        var casEntry = await cas.PutAsync(new FileCasPutRequest(sha, zipStream, leaveOpen: false), cancellationToken).ConfigureAwait(false);
+
+        return new ReachabilityUnionPublishResult(
+            Sha256: sha,
+            RelativePath: casEntry.RelativePath,
+            Records: result.Nodes.RecordCount + result.Edges.RecordCount + (result.Facts?.RecordCount ?? 0));
+    }
+
+    private static void CreateZip(string sourceDir, string destinationZip)
+    {
+        if (File.Exists(destinationZip))
+        {
+            File.Delete(destinationZip);
+        }
+
+        var files = Directory.EnumerateFiles(sourceDir, "*", SearchOption.TopDirectoryOnly)
+            .OrderBy(f => f, StringComparer.Ordinal)
+            .ToList();
+
+        using var zip = ZipFile.Open(destinationZip, ZipArchiveMode.Create);
+        foreach (var file in files)
+        {
+            var entryName = Path.GetFileName(file);
+            zip.CreateEntryFromFile(file, entryName, CompressionLevel.Optimal);
+        }
+    }
+
+    private static string ComputeSha256(string path)
+    {
+        using var sha = SHA256.Create();
+        using var stream = File.OpenRead(path);
+        return Convert.ToHexString(sha.ComputeHash(stream)).ToLowerInvariant();
+    }
+}
+
+public sealed record ReachabilityUnionPublishResult(
+    string Sha256,
+    string RelativePath,
+    int Records);

src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionPublisherService.cs

@@ -0,0 +1,40 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using StellaOps.Scanner.Cache.Abstractions;
+using StellaOps.Scanner.Surface.Env;
+
+namespace StellaOps.Scanner.Reachability;
+
+public interface IReachabilityUnionPublisherService
+{
+    Task<ReachabilityUnionPublishResult> PublishAsync(ReachabilityUnionGraph graph, string analysisId, CancellationToken cancellationToken = default);
+}
+
+/// <summary>
+/// Default service that writes a union graph to CAS using the worker surface cache root.
+/// </summary>
+public sealed class ReachabilityUnionPublisherService : IReachabilityUnionPublisherService
+{
+    private readonly ISurfaceEnvironment environment;
+    private readonly IFileContentAddressableStore cas;
+    private readonly ReachabilityUnionPublisher publisher;
+
+    public ReachabilityUnionPublisherService(
+        ISurfaceEnvironment environment,
+        IFileContentAddressableStore cas,
+        ReachabilityUnionPublisher publisher)
+    {
+        this.environment = environment ?? throw new ArgumentNullException(nameof(environment));
+        this.cas = cas ?? throw new ArgumentNullException(nameof(cas));
+        this.publisher = publisher ?? throw new ArgumentNullException(nameof(publisher));
+    }
+
+    public Task<ReachabilityUnionPublishResult> PublishAsync(ReachabilityUnionGraph graph, string analysisId, CancellationToken cancellationToken = default)
+    {
+        var workRoot = Path.Combine(environment.Settings.CacheRoot.FullName, "reachability");
+        Directory.CreateDirectory(workRoot);
+        return publisher.PublishAsync(graph, cas, workRoot, analysisId, cancellationToken);
+    }
+}

src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionSchemas.cs

@@ -0,0 +1,6 @@
+namespace StellaOps.Scanner.Reachability;
+
+public static class ReachabilityUnionSchemas
+{
+    public const string UnionSchema = "reachability-union@0.1";
+}

src/Scanner/__Libraries/StellaOps.Scanner.Reachability/ReachabilityUnionWriter.cs

@@ -0,0 +1,390 @@
+using System;
+using System.Collections.Generic;
+using System.Collections.Immutable;
+using System.IO;
+using System.Linq;
+using System.Security.Cryptography;
+using System.Text;
+using System.Text.Json;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace StellaOps.Scanner.Reachability;
+
+/// <summary>
+/// Serializes reachability graphs (static + runtime) into the union NDJSON layout
+/// described in docs/reachability/runtime-static-union-schema.md.
+/// </summary>
+public sealed class ReachabilityUnionWriter
+{
+    private static readonly JsonWriterOptions JsonOptions = new()
+    {
+        Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping,
+        Indented = false,
+        SkipValidation = false
+    };
+
+    public async Task<ReachabilityUnionWriteResult> WriteAsync(
+        ReachabilityUnionGraph graph,
+        string outputRoot,
+        string analysisId,
+        CancellationToken cancellationToken = default)
+    {
+        ArgumentNullException.ThrowIfNull(graph);
+        ArgumentException.ThrowIfNullOrWhiteSpace(outputRoot);
+        ArgumentException.ThrowIfNullOrWhiteSpace(analysisId);
+
+        var root = Path.Combine(outputRoot, "reachability_graphs", analysisId);
+        Directory.CreateDirectory(root);
+
+        var normalized = Normalize(graph);
+
+        var nodesPath = Path.Combine(root, "nodes.ndjson");
+        var edgesPath = Path.Combine(root, "edges.ndjson");
+        var factsPath = Path.Combine(root, "facts_runtime.ndjson");
+        var metaPath = Path.Combine(root, "meta.json");
+
+        var nodesInfo = await WriteNdjsonAsync(nodesPath, normalized.Nodes, WriteNodeAsync, cancellationToken).ConfigureAwait(false);
+        var edgesInfo = await WriteNdjsonAsync(edgesPath, normalized.Edges, WriteEdgeAsync, cancellationToken).ConfigureAwait(false);
+        FileHashInfo? factsInfo = null;
+
+        if (normalized.RuntimeFacts.Count > 0)
+        {
+            factsInfo = await WriteNdjsonAsync(factsPath, normalized.RuntimeFacts, WriteRuntimeFactAsync, cancellationToken).ConfigureAwait(false);
+        }
+        else if (File.Exists(factsPath))
+        {
+            File.Delete(factsPath);
+        }
+
+        await WriteMetaAsync(metaPath, nodesInfo, edgesInfo, factsInfo, cancellationToken).ConfigureAwait(false);
+
+        return new ReachabilityUnionWriteResult(nodesInfo.ToPublic(), edgesInfo.ToPublic(), factsInfo?.ToPublic(), metaPath);
+    }
+
+    private static NormalizedGraph Normalize(ReachabilityUnionGraph graph)
+    {
+        var nodes = graph.Nodes
+            .Where(n => !string.IsNullOrWhiteSpace(n.SymbolId))
+            .Select(n => n with
+            {
+                SymbolId = Trim(n.SymbolId) ?? string.Empty,
+                Lang = Trim(n.Lang) ?? string.Empty,
+                Kind = Trim(n.Kind) ?? string.Empty,
+                Display = Trim(n.Display),
+                Source = n.Source?.Trimmed(),
+                Attributes = (n.Attributes ?? ImmutableDictionary<string, string>.Empty)
+                    .Where(kv => !string.IsNullOrWhiteSpace(kv.Key) && kv.Value is not null)
+                    .ToImmutableSortedDictionary(kv => kv.Key.Trim(), kv => kv.Value!.Trim())
+            })
+            .OrderBy(n => n.SymbolId, StringComparer.Ordinal)
+            .ThenBy(n => n.Kind, StringComparer.Ordinal)
+            .ToList();
+
+        var edges = graph.Edges
+            .Where(e => !string.IsNullOrWhiteSpace(e.From) && !string.IsNullOrWhiteSpace(e.To))
+            .Select(e => e with
+            {
+                From = Trim(e.From)!,
+                To = Trim(e.To)!,
+                EdgeType = Trim(e.EdgeType) ?? "call",
+                Confidence = Trim(e.Confidence) ?? "certain",
+                Source = e.Source?.Trimmed()
+            })
+            .OrderBy(e => e.From, StringComparer.Ordinal)
+            .ThenBy(e => e.To, StringComparer.Ordinal)
+            .ThenBy(e => e.EdgeType, StringComparer.Ordinal)
+            .ToList();
+
+        var facts = (graph.RuntimeFacts ?? Enumerable.Empty<ReachabilityRuntimeFact>())
+            .Where(f => !string.IsNullOrWhiteSpace(f.SymbolId))
+            .Select(f => f with
+            {
+                SymbolId = Trim(f.SymbolId)!,
+                Samples = f.Samples?.Trimmed() ?? new ReachabilityRuntimeSamples(0, null, null),
+                Env = f.Env?.Trimmed() ?? ReachabilityRuntimeEnv.Empty
+            })
+            .OrderBy(f => f.SymbolId, StringComparer.Ordinal)
+            .ToList();
+
+        return new NormalizedGraph(nodes, edges, facts);
+    }
+
+    private static async Task<FileHashInfo> WriteNdjsonAsync<T>(
+        string path,
+        IReadOnlyCollection<T> items,
+        Func<T, StreamWriter, Task> writer,
+        CancellationToken cancellationToken)
+    {
+        await using (var stream = File.Create(path))
+        await using (var textWriter = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)))
+        {
+            foreach (var item in items)
+            {
+                await writer(item, textWriter).ConfigureAwait(false);
+                await textWriter.WriteLineAsync().ConfigureAwait(false);
+                cancellationToken.ThrowIfCancellationRequested();
+            }
+        }
+
+        var sha = ComputeSha256(path);
+        return new FileHashInfo(path, sha, items.Count);
+    }
+
+    private static async Task WriteNodeAsync(ReachabilityUnionNode node, StreamWriter writer)
+    {
+        await using var json = new MemoryStream();
+        await using (var jw = new Utf8JsonWriter(json, JsonOptions))
+        {
+            jw.WriteStartObject();
+            jw.WriteString("symbol_id", node.SymbolId);
+            jw.WriteString("lang", node.Lang);
+            jw.WriteString("kind", node.Kind);
+            if (!string.IsNullOrWhiteSpace(node.Display))
+            {
+                jw.WriteString("display", node.Display);
+            }
+
+            if (node.Source is not null)
+            {
+                jw.WritePropertyName("source");
+                WriteSource(jw, node.Source);
+            }
+
+            if (node.Attributes is not null && node.Attributes.Count > 0)
+            {
+                jw.WritePropertyName("attributes");
+                jw.WriteStartObject();
+                foreach (var kv in node.Attributes)
+                {
+                    jw.WriteString(kv.Key, kv.Value);
+                }
+
+                jw.WriteEndObject();
+            }
+
+            jw.WriteEndObject();
+        }
+
+        await writer.WriteAsync(Encoding.UTF8.GetString(json.ToArray())).ConfigureAwait(false);
+    }
+
+    private static async Task WriteEdgeAsync(ReachabilityUnionEdge edge, StreamWriter writer)
+    {
+        await using var json = new MemoryStream();
+        await using (var jw = new Utf8JsonWriter(json, JsonOptions))
+        {
+            jw.WriteStartObject();
+            jw.WriteString("from", edge.From);
+            jw.WriteString("to", edge.To);
+            jw.WriteString("edge_type", edge.EdgeType);
+            jw.WriteString("confidence", edge.Confidence);
+
+            if (edge.Source is not null)
+            {
+                jw.WritePropertyName("source");
+                WriteSource(jw, edge.Source);
+            }
+
+            jw.WriteEndObject();
+        }
+
+        await writer.WriteAsync(Encoding.UTF8.GetString(json.ToArray())).ConfigureAwait(false);
+    }
+
+    private static async Task WriteRuntimeFactAsync(ReachabilityRuntimeFact fact, StreamWriter writer)
+    {
+        await using var json = new MemoryStream();
+        await using (var jw = new Utf8JsonWriter(json, JsonOptions))
+        {
+            jw.WriteStartObject();
+            jw.WriteString("symbol_id", fact.SymbolId);
+
+            jw.WritePropertyName("samples");
+            jw.WriteStartObject();
+            jw.WriteNumber("call_count", fact.Samples?.CallCount ?? 0);
+            if (fact.Samples?.FirstSeenUtc is not null)
+            {
+                jw.WriteString("first_seen_utc", fact.Samples.FirstSeenUtc.Value.ToUniversalTime().ToString("O"));
+            }
+            if (fact.Samples?.LastSeenUtc is not null)
+            {
+                jw.WriteString("last_seen_utc", fact.Samples.LastSeenUtc.Value.ToUniversalTime().ToString("O"));
+            }
+            jw.WriteEndObject();
+
+            jw.WritePropertyName("env");
+            jw.WriteStartObject();
+            if (fact.Env?.Pid is not null)
+            {
+                jw.WriteNumber("pid", fact.Env.Pid.Value);
+            }
+            if (!string.IsNullOrWhiteSpace(fact.Env?.Image))
+            {
+                jw.WriteString("image", fact.Env!.Image);
+            }
+            if (!string.IsNullOrWhiteSpace(fact.Env?.Entrypoint))
+            {
+                jw.WriteString("entrypoint", fact.Env!.Entrypoint);
+            }
+            if (fact.Env?.Tags is { Count: > 0 })
+            {
+                jw.WritePropertyName("tags");
+                jw.WriteStartArray();
+                foreach (var tag in fact.Env!.Tags)
+                {
+                    jw.WriteStringValue(tag);
+                }
+                jw.WriteEndArray();
+            }
+            jw.WriteEndObject();
+
+            jw.WriteEndObject();
+        }
+
+        await writer.WriteAsync(Encoding.UTF8.GetString(json.ToArray())).ConfigureAwait(false);
+    }
+
+    private static void WriteSource(Utf8JsonWriter jw, ReachabilitySource source)
+    {
+        jw.WriteStartObject();
+        jw.WriteString("origin", source.Origin ?? "static");
+        if (!string.IsNullOrWhiteSpace(source.Provenance))
+        {
+            jw.WriteString("provenance", source.Provenance);
+        }
+        if (!string.IsNullOrWhiteSpace(source.Evidence))
+        {
+            jw.WriteString("evidence", source.Evidence);
+        }
+        jw.WriteEndObject();
+    }
+
+    private static string ComputeSha256(string path)
+    {
+        using var sha = SHA256.Create();
+        using var stream = File.OpenRead(path);
+        var hash = sha.ComputeHash(stream);
+        return Convert.ToHexString(hash).ToLowerInvariant();
+    }
+
+    private static string? Trim(string? value) => string.IsNullOrWhiteSpace(value) ? null : value.Trim();
+
+    private sealed record FileHashInfo(string Path, string Sha256, int RecordCount)
+    {
+        public ReachabilityUnionFileInfo ToPublic() => new(Path, Sha256, RecordCount);
+    }
+
+    private sealed record NormalizedGraph(
+        IReadOnlyList<ReachabilityUnionNode> Nodes,
+        IReadOnlyList<ReachabilityUnionEdge> Edges,
+        IReadOnlyList<ReachabilityRuntimeFact> RuntimeFacts);
+
+    private static async Task WriteMetaAsync(
+        string path,
+        FileHashInfo nodes,
+        FileHashInfo edges,
+        FileHashInfo? facts,
+        CancellationToken cancellationToken)
+    {
+        await using var stream = File.Create(path);
+        await using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = true });
+
+        writer.WriteStartObject();
+        writer.WriteString("schema", "reachability-union@0.1");
+        writer.WriteString("generated_at", DateTimeOffset.UtcNow.ToString("O"));
+        writer.WritePropertyName("files");
+        writer.WriteStartArray();
+        WriteMetaFile(writer, nodes);
+        WriteMetaFile(writer, edges);
+        if (facts is not null)
+        {
+            WriteMetaFile(writer, facts);
+        }
+        writer.WriteEndArray();
+        writer.WriteEndObject();
+        await writer.FlushAsync(cancellationToken).ConfigureAwait(false);
+    }
+
+    private static void WriteMetaFile(Utf8JsonWriter writer, FileHashInfo info)
+    {
+        writer.WriteStartObject();
+        writer.WriteString("path", info.Path);
+        writer.WriteString("sha256", info.Sha256);
+        writer.WriteNumber("records", info.RecordCount);
+        writer.WriteEndObject();
+    }
+}
+
+public sealed record ReachabilityUnionGraph(
+    IReadOnlyCollection<ReachabilityUnionNode> Nodes,
+    IReadOnlyCollection<ReachabilityUnionEdge> Edges,
+    IReadOnlyCollection<ReachabilityRuntimeFact>? RuntimeFacts = null);
+
+public sealed record ReachabilityUnionNode(
+    string SymbolId,
+    string Lang,
+    string Kind,
+    string? Display = null,
+    ReachabilitySource? Source = null,
+    IReadOnlyDictionary<string, string>? Attributes = null);
+
+public sealed record ReachabilityUnionEdge(
+    string From,
+    string To,
+    string EdgeType,
+    string? Confidence = "certain",
+    ReachabilitySource? Source = null);
+
+public sealed record ReachabilityRuntimeFact(
+    string SymbolId,
+    ReachabilityRuntimeSamples? Samples,
+    ReachabilityRuntimeEnv? Env);
+
+public sealed record ReachabilityRuntimeSamples(
+    long CallCount,
+    DateTimeOffset? FirstSeenUtc,
+    DateTimeOffset? LastSeenUtc)
+{
+    public ReachabilityRuntimeSamples Trimmed()
+        => new(CallCount, FirstSeenUtc?.ToUniversalTime(), LastSeenUtc?.ToUniversalTime());
+}
+
+public sealed record ReachabilityRuntimeEnv(
+    int? Pid,
+    string? Image,
+    string? Entrypoint,
+    IReadOnlyList<string> Tags)
+{
+    public static ReachabilityRuntimeEnv Empty { get; } = new(null, null, null, Array.Empty<string>());
+
+    public ReachabilityRuntimeEnv Trimmed()
+        => new(
+            Pid,
+            string.IsNullOrWhiteSpace(Image) ? null : Image.Trim(),
+            string.IsNullOrWhiteSpace(Entrypoint) ? null : Entrypoint.Trim(),
+            (Tags ?? Array.Empty<string>()).Where(t => !string.IsNullOrWhiteSpace(t)).Select(t => t.Trim()).OrderBy(t => t, StringComparer.Ordinal).ToArray());
+}
+
+public sealed record ReachabilitySource(
+    string? Origin,
+    string? Provenance,
+    string? Evidence)
+{
+    public ReachabilitySource Trimmed()
+        => new(
+            string.IsNullOrWhiteSpace(Origin) ? "static" : Origin.Trim(),
+            string.IsNullOrWhiteSpace(Provenance) ? null : Provenance.Trim(),
+            string.IsNullOrWhiteSpace(Evidence) ? null : Evidence.Trim());
+}
+
+public sealed record ReachabilityUnionWriteResult(
+    ReachabilityUnionFileInfo Nodes,
+    ReachabilityUnionFileInfo Edges,
+    ReachabilityUnionFileInfo? Facts,
+    string MetaPath);
+
+public sealed record ReachabilityUnionFileInfo(
+    string Path,
+    string Sha256,
+    int RecordCount);

src/Scanner/__Libraries/StellaOps.Scanner.Reachability/StellaOps.Scanner.Reachability.csproj

@@ -5,9 +5,8 @@
     <Nullable>enable</Nullable>
   </PropertyGroup>
   <ItemGroup>
-    <PackageReference Include="System.Text.Json" Version="10.0.0-preview.7.25380.108" />
-  </ItemGroup>
-  <ItemGroup>
+    <ProjectReference Include="..\StellaOps.Scanner.Cache\StellaOps.Scanner.Cache.csproj" />
+    <ProjectReference Include="..\StellaOps.Scanner.Surface.Env\StellaOps.Scanner.Surface.Env.csproj" />
     <ProjectReference Include="..\..\..\__Libraries\StellaOps.Replay.Core\StellaOps.Replay.Core.csproj" />
   </ItemGroup>
 </Project>