using System.Text.Json; using Xunit; using StellaOps.Scanner.Analyzers.Lang.Node; using StellaOps.Scanner.Analyzers.Lang.Tests.Harness; namespace StellaOps.Scanner.Analyzers.Lang.Node.Tests.Node; /// /// Tests to verify deterministic output from the Node analyzer. /// Output must be reproducible across multiple runs. /// public sealed class NodeDeterminismTests : IDisposable { private readonly string _tempDir; public NodeDeterminismTests() { _tempDir = Path.Combine(Path.GetTempPath(), "node-determinism-tests-" + Guid.NewGuid().ToString("N")[..8]); Directory.CreateDirectory(_tempDir); } public void Dispose() { try { if (Directory.Exists(_tempDir)) { Directory.Delete(_tempDir, recursive: true); } } catch { // Ignore cleanup errors } } private void WriteFile(string relativePath, string content) { var fullPath = Path.Combine(_tempDir, relativePath); Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!); File.WriteAllText(fullPath, content); } #region Multiple Runs Determinism [Fact] public async Task MultipleRuns_ProduceIdenticalOutput() { // Arrange SetupComplexProject(); // Act - Run analyzer multiple times var run1 = await RunAnalyzerAsync(); var run2 = await RunAnalyzerAsync(); var run3 = await RunAnalyzerAsync(); // Assert - All runs should produce identical output Assert.Equal(run1, run2); Assert.Equal(run2, run3); } [Fact] public async Task MultipleRuns_PackageOrderIsStable() { // Arrange WriteFile("package.json", JsonSerializer.Serialize(new { name = "root", version = "1.0.0" })); // Create packages in non-alphabetical order WriteFile("node_modules/zebra/package.json", JsonSerializer.Serialize(new { name = "zebra", version = "1.0.0" })); WriteFile("node_modules/alpha/package.json", JsonSerializer.Serialize(new { name = "alpha", version = "1.0.0" })); WriteFile("node_modules/mike/package.json", JsonSerializer.Serialize(new { name = "mike", version = "1.0.0" })); WriteFile("node_modules/beta/package.json", JsonSerializer.Serialize(new { name = "beta", version = "1.0.0" })); // Act var result1 = await RunAnalyzerAsync(); var result2 = await RunAnalyzerAsync(); // Assert var order1 = ExtractPackageNames(result1); var order2 = ExtractPackageNames(result2); Assert.Equal(order1, order2); } #endregion #region Package Ordering [Fact] public async Task PackageOrdering_IsSortedByPurl() { // Arrange WriteFile("package.json", JsonSerializer.Serialize(new { name = "root", version = "1.0.0" })); WriteFile("node_modules/z-pkg/package.json", JsonSerializer.Serialize(new { name = "z-pkg", version = "1.0.0" })); WriteFile("node_modules/a-pkg/package.json", JsonSerializer.Serialize(new { name = "a-pkg", version = "1.0.0" })); WriteFile("node_modules/m-pkg/package.json", JsonSerializer.Serialize(new { name = "m-pkg", version = "1.0.0" })); // Act var result = await RunAnalyzerAsync(); // Assert - Packages should be sorted var names = ExtractPackageNames(result); var sortedNames = names.OrderBy(n => n, StringComparer.Ordinal).ToList(); Assert.Equal(sortedNames, names); } [Fact] public async Task ScopedPackageOrdering_IsConsistent() { // Arrange WriteFile("package.json", JsonSerializer.Serialize(new { name = "root", version = "1.0.0" })); WriteFile("node_modules/@z-scope/pkg/package.json", JsonSerializer.Serialize(new { name = "@z-scope/pkg", version = "1.0.0" })); WriteFile("node_modules/@a-scope/pkg/package.json", JsonSerializer.Serialize(new { name = "@a-scope/pkg", version = "1.0.0" })); WriteFile("node_modules/regular-pkg/package.json", JsonSerializer.Serialize(new { name = "regular-pkg", version = "1.0.0" })); // Act var result1 = await RunAnalyzerAsync(); var result2 = await RunAnalyzerAsync(); // Assert Assert.Equal(result1, result2); } #endregion [Fact] public async Task LockOnlyProject_EmitsDeclaredOnlyComponents_WithoutRangeAsPurl() { WriteFile("package.json", JsonSerializer.Serialize(new { name = "root", version = "1.0.0", dependencies = new Dictionary { ["express"] = "^4.18.2", ["left-pad"] = "^1.3.0" } })); WriteFile("package-lock.json", JsonSerializer.Serialize(new { name = "root", version = "1.0.0", lockfileVersion = 3, packages = new Dictionary { [""] = new { name = "root", version = "1.0.0" }, ["node_modules/express"] = new { version = "4.18.2", resolved = "https://registry.npmjs.org/express/-/express-4.18.2.tgz", integrity = "sha512-deadbeef" } } })); var json = await RunAnalyzerAsync(); using var document = JsonDocument.Parse(json); var components = document.RootElement.EnumerateArray().ToArray(); var express = components.Single(static element => element.TryGetProperty("purl", out var purl) && purl.ValueKind == JsonValueKind.String && purl.GetString() == "pkg:npm/express@4.18.2"); var expressMeta = express.GetProperty("metadata"); Assert.Equal("true", expressMeta.GetProperty("declaredOnly").GetString()); Assert.Equal("package-lock.json", expressMeta.GetProperty("declared.source").GetString()); Assert.Equal("package-lock.json:node_modules/express", expressMeta.GetProperty("declared.locator").GetString()); Assert.Equal("^4.18.2", expressMeta.GetProperty("declared.versionSpec").GetString()); Assert.Equal("4.18.2", expressMeta.GetProperty("declared.resolvedVersion").GetString()); var leftPad = components.Single(static element => element.GetProperty("name").GetString() == "left-pad"); Assert.False(leftPad.TryGetProperty("purl", out _)); Assert.StartsWith("explicit::node::npm::left-pad::sha256:", leftPad.GetProperty("componentKey").GetString(), StringComparison.Ordinal); var leftPadMeta = leftPad.GetProperty("metadata"); Assert.Equal("true", leftPadMeta.GetProperty("declaredOnly").GetString()); Assert.Equal("package.json", leftPadMeta.GetProperty("declared.source").GetString()); Assert.Equal("^1.3.0", leftPadMeta.GetProperty("declared.versionSpec").GetString()); } [Fact] public async Task PnpmLock_IntegrityMissing_EmitsDeclaredOnlyMetadata() { WriteFile("package.json", JsonSerializer.Serialize(new { name = "root", version = "1.0.0", dependencies = new Dictionary { ["local-file"] = "file:../local-file-1.0.0.tgz" } })); var pnpmLock = "lockfileVersion: '6.0'\n" + "packages:\n" + " /local-file/1.0.0:\n" + " resolution: {tarball: file:../local-file-1.0.0.tgz}\n"; WriteFile("pnpm-lock.yaml", pnpmLock); var json = await RunAnalyzerAsync(); using var document = JsonDocument.Parse(json); var components = document.RootElement.EnumerateArray().ToArray(); var localFile = components.Single(static element => element.TryGetProperty("purl", out var purl) && purl.ValueKind == JsonValueKind.String && purl.GetString() == "pkg:npm/local-file@1.0.0"); var meta = localFile.GetProperty("metadata"); Assert.Equal("true", meta.GetProperty("declaredOnly").GetString()); Assert.Equal("pnpm-lock.yaml", meta.GetProperty("declared.source").GetString()); Assert.Equal("pnpm-lock.yaml:local-file/1.0.0", meta.GetProperty("declared.locator").GetString()); Assert.Equal("file:../local-file-1.0.0.tgz", meta.GetProperty("declared.versionSpec").GetString()); Assert.Equal("1.0.0", meta.GetProperty("declared.resolvedVersion").GetString()); Assert.Equal("true", meta.GetProperty("lockIntegrityMissing").GetString()); Assert.Equal("file", meta.GetProperty("lockIntegrityMissingReason").GetString()); } #region Entrypoint Ordering [Fact] public async Task EntrypointOrdering_IsStable() { // Arrange - Multiple entrypoints in various fields var packageJson = new { name = "multi-entry-pkg", version = "1.0.0", main = "./dist/main.js", module = "./dist/module.mjs", bin = new { cli1 = "./bin/cli1.js", cli2 = "./bin/cli2.js" } }; WriteFile("package.json", JsonSerializer.Serialize(packageJson)); WriteFile("dist/main.js", "// main"); WriteFile("dist/module.mjs", "// module"); WriteFile("bin/cli1.js", "// cli1"); WriteFile("bin/cli2.js", "// cli2"); // Act var result1 = await RunAnalyzerAsync(); var result2 = await RunAnalyzerAsync(); // Assert Assert.Equal(result1, result2); } [Fact] public async Task ExportsOrdering_IsSortedAlphabetically() { // Arrange - Exports with conditions in non-alphabetical order var packageJsonContent = @"{ ""name"": ""exports-pkg"", ""version"": ""1.0.0"", ""exports"": { ""."": { ""require"": ""./dist/index.cjs"", ""import"": ""./dist/index.mjs"", ""default"": ""./dist/index.js"" } } }"; WriteFile("package.json", packageJsonContent); WriteFile("dist/index.cjs", "// cjs"); WriteFile("dist/index.mjs", "// mjs"); WriteFile("dist/index.js", "// js"); // Act var result1 = await RunAnalyzerAsync(); var result2 = await RunAnalyzerAsync(); // Assert - Order should be consistent Assert.Equal(result1, result2); } #endregion #region Evidence Ordering [Fact] public async Task EvidenceOrdering_IsStable() { // Arrange var packageJson = new { name = "evidence-pkg", version = "1.0.0", main = "./index.js", license = "MIT", scripts = new { postinstall = "node setup.js" } }; WriteFile("package.json", JsonSerializer.Serialize(packageJson)); WriteFile("index.js", "// index"); // Act var result1 = await RunAnalyzerAsync(); var result2 = await RunAnalyzerAsync(); // Assert Assert.Equal(result1, result2); } #endregion #region Dependency Resolution Ordering [Fact] public async Task DependencyIndex_ProducesDeterministicScopes() { // Arrange var packageJson = new { name = "deps-pkg", version = "1.0.0", dependencies = new { dep1 = "^1.0.0", dep2 = "^2.0.0" }, devDependencies = new { devDep1 = "^3.0.0", devDep2 = "^4.0.0" }, peerDependencies = new { peerDep1 = "^5.0.0" }, optionalDependencies = new { optDep1 = "^6.0.0" } }; WriteFile("package.json", JsonSerializer.Serialize(packageJson)); WriteFile("node_modules/dep1/package.json", JsonSerializer.Serialize(new { name = "dep1", version = "1.0.0" })); WriteFile("node_modules/dep2/package.json", JsonSerializer.Serialize(new { name = "dep2", version = "2.0.0" })); WriteFile("node_modules/devDep1/package.json", JsonSerializer.Serialize(new { name = "devDep1", version = "3.0.0" })); WriteFile("node_modules/devDep2/package.json", JsonSerializer.Serialize(new { name = "devDep2", version = "4.0.0" })); // Act var result1 = await RunAnalyzerAsync(); var result2 = await RunAnalyzerAsync(); // Assert Assert.Equal(result1, result2); } #endregion #region Lockfile Ordering [Fact] public async Task LockfilePackages_ProduceDeterministicOutput() { // Arrange WriteFile("package.json", JsonSerializer.Serialize(new { name = "lock-pkg", version = "1.0.0" })); WriteFile("package-lock.json", @"{ ""name"": ""lock-pkg"", ""version"": ""1.0.0"", ""lockfileVersion"": 3, ""packages"": { """": { ""name"": ""lock-pkg"", ""version"": ""1.0.0"" }, ""node_modules/z-dep"": { ""version"": ""3.0.0"", ""resolved"": ""https://r.example/z"", ""integrity"": ""sha512-Z"" }, ""node_modules/a-dep"": { ""version"": ""1.0.0"", ""resolved"": ""https://r.example/a"", ""integrity"": ""sha512-A"" }, ""node_modules/m-dep"": { ""version"": ""2.0.0"", ""resolved"": ""https://r.example/m"", ""integrity"": ""sha512-M"" } } }"); // Act var result1 = await RunAnalyzerAsync(); var result2 = await RunAnalyzerAsync(); // Assert Assert.Equal(result1, result2); } #endregion private void SetupComplexProject() { // Root package var rootPackage = new { name = "complex-app", version = "1.0.0", dependencies = new { lodash = "^4.17.21", express = "^4.18.0" }, devDependencies = new { typescript = "^5.0.0" } }; WriteFile("package.json", JsonSerializer.Serialize(rootPackage)); // Dependencies WriteFile("node_modules/lodash/package.json", JsonSerializer.Serialize(new { name = "lodash", version = "4.17.21" })); WriteFile("node_modules/express/package.json", JsonSerializer.Serialize(new { name = "express", version = "4.18.2" })); WriteFile("node_modules/typescript/package.json", JsonSerializer.Serialize(new { name = "typescript", version = "5.2.2" })); // Nested dependencies WriteFile("node_modules/express/node_modules/accepts/package.json", JsonSerializer.Serialize(new { name = "accepts", version = "1.3.8" })); WriteFile("node_modules/express/node_modules/body-parser/package.json", JsonSerializer.Serialize(new { name = "body-parser", version = "1.20.1" })); } private async Task RunAnalyzerAsync() { var analyzers = new ILanguageAnalyzer[] { new NodeLanguageAnalyzer() }; return await LanguageAnalyzerTestHarness.RunToJsonAsync( _tempDir, analyzers, TestContext.Current.CancellationToken); } private static List ExtractPackageNames(string json) { var doc = JsonDocument.Parse(json); return doc.RootElement.EnumerateArray() .Select(el => el.GetProperty("name").GetString()!) .ToList(); } }