using System.IO.Compression; using System.Security.Cryptography; using System.Text; using System.Text.Json; using StellaOps.Scanner.Analyzers.Lang.Python; using StellaOps.Scanner.Analyzers.Lang.Tests.Harness; using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities; namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests; public sealed class PythonLanguageAnalyzerTests { [Fact] public async Task SimpleVenvFixtureProducesDeterministicOutputAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "simple-venv"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var usageHints = new LanguageUsageHints(new[] { Path.Combine(fixturePath, "bin", "simple-tool") }); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken, usageHints); } [Fact] public async Task PipCacheFixtureProducesDeterministicOutputAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "pip-cache"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var usageHints = new LanguageUsageHints(new[] { Path.Combine(fixturePath, "lib", "python3.11", "site-packages", "cache_pkg-1.2.3.data", "scripts", "cache-tool") }); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken, usageHints); } [Fact] public async Task LayeredEditableFixtureMergesAcrossLayersAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "layered-editable"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var usageHints = new LanguageUsageHints(new[] { Path.Combine(fixturePath, "layer1", "usr", "bin", "layered-cli") }); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken, usageHints); } [Fact] public async Task CondaEnvFixtureProducesDeterministicOutputAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "conda-env"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken); } [Fact] public async Task RequirementsIncludesEditableFixtureProducesDeterministicOutputAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "requirements-includes-editable"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken); } [Fact] public async Task PipfileLockDefaultDevelopFixtureProducesDeterministicOutputAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "pipfile-lock-default-develop"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken); } [Fact] public async Task WheelWorkspaceFixtureProducesDeterministicOutputAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "wheel-workspace"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var distDir = Path.Combine(fixturePath, "dist"); Directory.CreateDirectory(distDir); var wheelPath = Path.Combine(distDir, "wheelpkg-1.0.0-py3-none-any.whl"); CreateWheelpkgWheel(wheelPath); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken); } [Fact] public async Task ZipappEmbeddedRequirementsFixtureProducesDeterministicOutputAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "zipapp-embedded-requirements"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var zipappPath = Path.Combine(fixturePath, "myapp.pyz"); CreateZipappWithEmbeddedRequirements(zipappPath); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken); } [Fact] public async Task ContainerWhiteoutsFixtureProducesDeterministicOutputAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "container-whiteouts"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken); } [Fact] public async Task VendoredDirectoryFixtureProducesDeterministicOutputAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "vendored-directory"); var goldenPath = Path.Combine(fixturePath, "expected.json"); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; await LanguageAnalyzerTestHarness.AssertDeterministicAsync( fixturePath, goldenPath, analyzers, cancellationToken); } [Fact] public async Task LockfileCollectorEmitsDeclaredOnlyComponentsAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { await CreatePythonPackageAsync(fixturePath, "locked", "1.0.0", cancellationToken); await CreatePythonPackageAsync(fixturePath, "runtime-only", "2.0.0", cancellationToken); var requirementsPath = Path.Combine(fixturePath, "requirements.txt"); var requirements = new StringBuilder() .AppendLine("locked==1.0.0") .AppendLine("declared-only==3.0.0") .ToString(); await File.WriteAllTextAsync(requirementsPath, requirements, cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; Assert.True(ComponentHasMetadata(root, "declared-only", "declaredOnly", "true")); Assert.True(ComponentHasMetadata(root, "declared-only", "lockSource", "requirements.txt")); Assert.True(ComponentHasMetadata(root, "locked", "lockSource", "requirements.txt")); Assert.True(ComponentHasMetadata(root, "runtime-only", "lockMissing", "true")); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task EditableRequirementsUseExplicitKeyWithoutHostPathLeakAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { var editableDir = Path.Combine(fixturePath, "editable-src"); Directory.CreateDirectory(editableDir); var requirementsPath = Path.Combine(fixturePath, "requirements.txt"); await File.WriteAllTextAsync(requirementsPath, $"--editable {editableDir}{Environment.NewLine}", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; foreach (var component in root.EnumerateArray()) { if (component.TryGetProperty("purl", out var purlElement) && purlElement.ValueKind == JsonValueKind.String) { Assert.DoesNotContain("@editable", purlElement.GetString(), StringComparison.OrdinalIgnoreCase); } } var editableComponent = root.EnumerateArray().Single(static component => component.TryGetProperty("name", out var nameElement) && string.Equals("editable-src", nameElement.GetString(), StringComparison.OrdinalIgnoreCase)); Assert.True(!editableComponent.TryGetProperty("purl", out var purlValue) || purlValue.ValueKind == JsonValueKind.Null); var componentKey = editableComponent.GetProperty("componentKey").GetString(); Assert.StartsWith("explicit::python::pypi::editable-src::sha256:", componentKey, StringComparison.Ordinal); var metadata = editableComponent.GetProperty("metadata"); Assert.Equal("true", metadata.GetProperty("declaredOnly").GetString()); Assert.Equal("editable", metadata.GetProperty("declared.sourceType").GetString()); Assert.Equal("requirements.txt", metadata.GetProperty("declared.source").GetString()); Assert.Equal("requirements.txt", metadata.GetProperty("declared.locator").GetString()); var editableSpec = metadata.GetProperty("lockEditablePath").GetString(); Assert.Equal("editable-src", editableSpec); Assert.DoesNotContain(fixturePath, editableSpec, StringComparison.OrdinalIgnoreCase); Assert.DoesNotContain(":", editableSpec, StringComparison.Ordinal); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task WheelArchiveDistInfo_IsVerifiedFromRecordAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { var distDir = Path.Combine(fixturePath, "dist"); Directory.CreateDirectory(distDir); var wheelPath = Path.Combine(distDir, "archivepkg-1.0.0-py3-none-any.whl"); var initBytes = Encoding.UTF8.GetBytes("__version__ = \"1.0.0\"\n"); var metadataBytes = Encoding.UTF8.GetBytes( $"Metadata-Version: 2.1\nName: archivepkg\nVersion: 1.0.0\n{Environment.NewLine}"); var wheelBytes = Encoding.UTF8.GetBytes( "Wheel-Version: 1.0\nGenerator: test\nRoot-Is-Purelib: true\nTag: py3-none-any\n"); var recordContent = new StringBuilder() .AppendLine($"archivepkg/__init__.py,sha256={ComputeSha256Base64(initBytes)},{initBytes.Length}") .AppendLine($"archivepkg-1.0.0.dist-info/METADATA,sha256={ComputeSha256Base64(metadataBytes)},{metadataBytes.Length}") .AppendLine($"archivepkg-1.0.0.dist-info/WHEEL,sha256={ComputeSha256Base64(wheelBytes)},{wheelBytes.Length}") .AppendLine("archivepkg-1.0.0.dist-info/RECORD,,") .ToString(); var recordBytes = Encoding.UTF8.GetBytes(recordContent); using (var stream = File.Create(wheelPath)) using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: false)) { WriteEntry(archive, "archivepkg/__init__.py", initBytes); WriteEntry(archive, "archivepkg-1.0.0.dist-info/METADATA", metadataBytes); WriteEntry(archive, "archivepkg-1.0.0.dist-info/WHEEL", wheelBytes); WriteEntry(archive, "archivepkg-1.0.0.dist-info/RECORD", recordBytes); } var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; Assert.True(ComponentHasMetadata(root, "archivepkg", "record.totalEntries", "4")); Assert.True(ComponentHasMetadata(root, "archivepkg", "record.hashedEntries", "3")); Assert.True(ComponentHasMetadata(root, "archivepkg", "record.missingFiles", "0")); Assert.True(ComponentHasMetadata(root, "archivepkg", "record.hashMismatches", "0")); Assert.True(ComponentHasMetadata(root, "archivepkg", "record.ioErrors", "0")); } finally { Directory.Delete(fixturePath, recursive: true); } static void WriteEntry(ZipArchive archive, string entryName, byte[] content) { var entry = archive.CreateEntry(entryName); using var entryStream = entry.Open(); entryStream.Write(content, 0, content.Length); } static string ComputeSha256Base64(byte[] content) => Convert.ToBase64String(SHA256.HashData(content)); } private static async Task CreatePythonPackageAsync(string root, string name, string version, CancellationToken cancellationToken) { var sitePackages = Path.Combine(root, "lib", "python3.11", "site-packages"); Directory.CreateDirectory(sitePackages); var packageDir = Path.Combine(sitePackages, name); Directory.CreateDirectory(packageDir); var modulePath = Path.Combine(packageDir, "__init__.py"); var moduleContent = $"__version__ = \"{version}\"{Environment.NewLine}"; await File.WriteAllTextAsync(modulePath, moduleContent, cancellationToken); var distInfoDir = Path.Combine(sitePackages, $"{name}-{version}.dist-info"); Directory.CreateDirectory(distInfoDir); var metadataPath = Path.Combine(distInfoDir, "METADATA"); var metadataContent = $"Metadata-Version: 2.1{Environment.NewLine}Name: {name}{Environment.NewLine}Version: {version}{Environment.NewLine}"; await File.WriteAllTextAsync(metadataPath, metadataContent, cancellationToken); var wheelPath = Path.Combine(distInfoDir, "WHEEL"); await File.WriteAllTextAsync(wheelPath, "Wheel-Version: 1.0", cancellationToken); var entryPointsPath = Path.Combine(distInfoDir, "entry_points.txt"); await File.WriteAllTextAsync(entryPointsPath, string.Empty, cancellationToken); var recordPath = Path.Combine(distInfoDir, "RECORD"); var recordContent = new StringBuilder() .AppendLine($"{name}/__init__.py,sha256={ComputeSha256Base64(modulePath)},{new FileInfo(modulePath).Length}") .AppendLine($"{name}-{version}.dist-info/METADATA,sha256={ComputeSha256Base64(metadataPath)},{new FileInfo(metadataPath).Length}") .AppendLine($"{name}-{version}.dist-info/RECORD,,") .AppendLine($"{name}-{version}.dist-info/WHEEL,sha256={ComputeSha256Base64(wheelPath)},{new FileInfo(wheelPath).Length}") .AppendLine($"{name}-{version}.dist-info/entry_points.txt,sha256={ComputeSha256Base64(entryPointsPath)},{new FileInfo(entryPointsPath).Length}") .ToString(); await File.WriteAllTextAsync(recordPath, recordContent, cancellationToken); } private static bool ComponentHasMetadata(JsonElement root, string componentName, string key, string? expectedValue) { foreach (var component in root.EnumerateArray()) { if (!component.TryGetProperty("name", out var nameElement) || !string.Equals(nameElement.GetString(), componentName, StringComparison.OrdinalIgnoreCase)) { continue; } if (!component.TryGetProperty("metadata", out var metadataElement) || metadataElement.ValueKind != JsonValueKind.Object) { continue; } if (!metadataElement.TryGetProperty(key, out var valueElement)) { continue; } var actual = valueElement.GetString(); if (string.Equals(actual, expectedValue, StringComparison.Ordinal)) { return true; } } return false; } private static string ComputeSha256Base64(string path) { using var sha = SHA256.Create(); using var stream = File.OpenRead(path); var hash = sha.ComputeHash(stream); return Convert.ToBase64String(hash); } [Fact] public async Task DetectsSitecustomizeStartupHooksAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { // Create site-packages with sitecustomize.py var sitePackages = Path.Combine(fixturePath, "lib", "python3.11", "site-packages"); Directory.CreateDirectory(sitePackages); var sitecustomizePath = Path.Combine(sitePackages, "sitecustomize.py"); await File.WriteAllTextAsync(sitecustomizePath, "# Site customization\nprint('startup hook')", cancellationToken); // Create a package await CreatePythonPackageAsync(fixturePath, "test-pkg", "1.0.0", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // Verify startup hooks metadata is present Assert.True(ComponentHasMetadata(root, "test-pkg", "startupHooks.detected", "true")); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task DetectsPthFilesWithImportDirectivesAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { var sitePackages = Path.Combine(fixturePath, "lib", "python3.11", "site-packages"); Directory.CreateDirectory(sitePackages); // Create a .pth file with import directive var pthPath = Path.Combine(sitePackages, "test-hooks.pth"); await File.WriteAllTextAsync(pthPath, "import some_module\n/some/path", cancellationToken); // Create a package await CreatePythonPackageAsync(fixturePath, "test-pkg", "1.0.0", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // Verify .pth import warning metadata is present Assert.True(ComponentHasMetadata(root, "test-pkg", "pthFiles.withImports.detected", "true")); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task DetectsOciLayerSitePackagesAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { // Create OCI layer structure with packages var layersDir = Path.Combine(fixturePath, "layers", "layer1", "fs"); var sitePackages = Path.Combine(layersDir, "usr", "lib", "python3.11", "site-packages"); Directory.CreateDirectory(sitePackages); // Create a package in the layer var packageDir = Path.Combine(sitePackages, "layered-pkg"); Directory.CreateDirectory(packageDir); var modulePath = Path.Combine(packageDir, "__init__.py"); await File.WriteAllTextAsync(modulePath, "__version__ = \"1.0.0\"", cancellationToken); var distInfoDir = Path.Combine(sitePackages, "layered-pkg-1.0.0.dist-info"); Directory.CreateDirectory(distInfoDir); var metadataPath = Path.Combine(distInfoDir, "METADATA"); await File.WriteAllTextAsync(metadataPath, "Metadata-Version: 2.1\nName: layered-pkg\nVersion: 1.0.0", cancellationToken); var wheelPath = Path.Combine(distInfoDir, "WHEEL"); await File.WriteAllTextAsync(wheelPath, "Wheel-Version: 1.0", cancellationToken); var recordPath = Path.Combine(distInfoDir, "RECORD"); await File.WriteAllTextAsync(recordPath, "", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // Verify the package from OCI layers was discovered var found = false; foreach (var component in root.EnumerateArray()) { if (component.TryGetProperty("name", out var nameElement) && string.Equals(nameElement.GetString(), "layered-pkg", StringComparison.OrdinalIgnoreCase)) { found = true; break; } } Assert.True(found, "Package from OCI layer should be discovered"); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task EggInfoPackagesAreDetectedAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = TestPaths.ResolveFixture("lang", "python", "egg-info"); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; var expectedPath = Path.Combine("lib", "python3.11", "site-packages", "egg_info_pkg-1.2.3.egg-info") .Replace('\\', '/'); Assert.True(ComponentHasMetadata(root, "egg-info-pkg", "provenance", "egg-info")); Assert.True(ComponentHasMetadata(root, "egg-info-pkg", "record.totalEntries", "4")); Assert.True(ComponentHasMetadata(root, "egg-info-pkg", "distInfoPath", expectedPath)); } [Fact] public async Task DetectsPythonEnvironmentVariablesAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { // Create environment file with PYTHONPATH var envPath = Path.Combine(fixturePath, ".env"); await File.WriteAllTextAsync(envPath, "PYTHONPATH=/app/lib:/app/vendor", cancellationToken); // Create a package await CreatePythonPackageAsync(fixturePath, "test-pkg", "1.0.0", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // Verify PYTHONPATH warning metadata is present Assert.True(ComponentHasMetadata(root, "test-pkg", "env.pythonpath", "/app/lib:/app/vendor")); Assert.True(ComponentHasMetadata(root, "test-pkg", "env.pythonpath.warning", "PYTHONPATH is set; may affect module resolution")); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task DetectsPyvenvConfigAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { // Create pyvenv.cfg file var pyvenvPath = Path.Combine(fixturePath, "pyvenv.cfg"); await File.WriteAllTextAsync(pyvenvPath, "home = /usr/local/bin\ninclude-system-site-packages = false", cancellationToken); // Create a package await CreatePythonPackageAsync(fixturePath, "test-pkg", "1.0.0", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // Verify PYTHONHOME warning metadata is present (from pyvenv.cfg home) Assert.True(ComponentHasMetadata(root, "test-pkg", "env.pythonhome", "/usr/local/bin")); } finally { Directory.Delete(fixturePath, recursive: true); } } private static string CreateTemporaryWorkspace() { var path = Path.Combine(Path.GetTempPath(), $"stellaops-python-{Guid.NewGuid():N}"); Directory.CreateDirectory(path); return path; } private static void CreateWheelpkgWheel(string wheelPath) { Directory.CreateDirectory(Path.GetDirectoryName(wheelPath)!); var initBytes = Encoding.UTF8.GetBytes("__version__ = \"1.0.0\"\n"); var metadataBytes = Encoding.UTF8.GetBytes( $"Metadata-Version: 2.1\nName: wheelpkg\nVersion: 1.0.0\nSummary: Wheel fixture\n{Environment.NewLine}"); var wheelBytes = Encoding.UTF8.GetBytes( "Wheel-Version: 1.0\nGenerator: stellaops-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n"); var recordContent = new StringBuilder() .AppendLine($"wheelpkg/__init__.py,sha256={ComputeSha256Base64(initBytes)},{initBytes.Length}") .AppendLine($"wheelpkg-1.0.0.dist-info/METADATA,sha256={ComputeSha256Base64(metadataBytes)},{metadataBytes.Length}") .AppendLine($"wheelpkg-1.0.0.dist-info/WHEEL,sha256={ComputeSha256Base64(wheelBytes)},{wheelBytes.Length}") .AppendLine("wheelpkg-1.0.0.dist-info/RECORD,,") .ToString(); var recordBytes = Encoding.UTF8.GetBytes(recordContent); if (File.Exists(wheelPath)) { File.Delete(wheelPath); } using (var stream = File.Create(wheelPath)) using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: false)) { WriteEntry(archive, "wheelpkg/__init__.py", initBytes); WriteEntry(archive, "wheelpkg-1.0.0.dist-info/METADATA", metadataBytes); WriteEntry(archive, "wheelpkg-1.0.0.dist-info/WHEEL", wheelBytes); WriteEntry(archive, "wheelpkg-1.0.0.dist-info/RECORD", recordBytes); } static void WriteEntry(ZipArchive archive, string entryName, byte[] content) { var entry = archive.CreateEntry(entryName); entry.LastWriteTime = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero); using var entryStream = entry.Open(); entryStream.Write(content, 0, content.Length); } static string ComputeSha256Base64(byte[] content) => Convert.ToBase64String(SHA256.HashData(content)); } private static void CreateZipappWithEmbeddedRequirements(string zipappPath) { if (File.Exists(zipappPath)) { File.Delete(zipappPath); } using var fileStream = File.Create(zipappPath); var shebangBytes = Encoding.UTF8.GetBytes("#!/usr/bin/python3.11\n"); fileStream.Write(shebangBytes); using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, leaveOpen: true); WriteTextEntry(archive, "__main__.py", "print('hello')\n"); WriteTextEntry(archive, "requirements.txt", "requests==2.28.0\nflask==2.1.0\n"); static void WriteTextEntry(ZipArchive archive, string name, string content) { var entry = archive.CreateEntry(name, CompressionLevel.NoCompression); entry.LastWriteTime = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero); using var stream = entry.Open(); using var writer = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false)); writer.Write(content); } } // ===== SCAN-PY-405-007 Fixtures ===== [Fact] public async Task RequirementsWithIncludesAreFollowedAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { // Create main requirements.txt that includes another file var requirementsPath = Path.Combine(fixturePath, "requirements.txt"); await File.WriteAllTextAsync(requirementsPath, $"requests==2.28.0{Environment.NewLine}-r requirements-base.txt{Environment.NewLine}", cancellationToken); // Create included requirements file var baseRequirementsPath = Path.Combine(fixturePath, "requirements-base.txt"); await File.WriteAllTextAsync(baseRequirementsPath, $"urllib3==1.26.0{Environment.NewLine}certifi==2022.12.7{Environment.NewLine}", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // All three packages should be found (from both files) Assert.True(ComponentHasMetadata(root, "requests", "declaredOnly", "true")); Assert.True(ComponentHasMetadata(root, "urllib3", "declaredOnly", "true")); Assert.True(ComponentHasMetadata(root, "certifi", "declaredOnly", "true")); // urllib3 and certifi should come from the included file Assert.True(ComponentHasMetadata(root, "urllib3", "lockSource", "requirements-base.txt")); Assert.True(ComponentHasMetadata(root, "certifi", "lockSource", "requirements-base.txt")); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task PipfileLockDevelopSectionIsParsedAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { // Create Pipfile.lock with default and develop sections var pipfileLockPath = Path.Combine(fixturePath, "Pipfile.lock"); var pipfileLock = """ { "_meta": { "sources": [] }, "default": { "requests": { "version": "==2.28.0" } }, "develop": { "pytest": { "version": "==7.0.0" } } } """; await File.WriteAllTextAsync(pipfileLockPath, pipfileLock, cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // Both packages should be found Assert.True(ComponentHasMetadata(root, "requests", "declaredOnly", "true")); Assert.True(ComponentHasMetadata(root, "pytest", "declaredOnly", "true")); // requests should be prod scope, pytest should be dev scope Assert.True(ComponentHasMetadata(root, "requests", "scope", "prod")); Assert.True(ComponentHasMetadata(root, "pytest", "scope", "dev")); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task RequirementsDevTxtGetsScopeDevAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { // Create requirements.txt for prod var requirementsPath = Path.Combine(fixturePath, "requirements.txt"); await File.WriteAllTextAsync(requirementsPath, $"flask==2.0.0{Environment.NewLine}", cancellationToken); // Create requirements-dev.txt for dev dependencies var requirementsDevPath = Path.Combine(fixturePath, "requirements-dev.txt"); await File.WriteAllTextAsync(requirementsDevPath, $"pytest==7.0.0{Environment.NewLine}", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // flask should be prod scope (from requirements.txt) Assert.True(ComponentHasMetadata(root, "flask", "scope", "prod")); // pytest should be dev scope (from requirements-dev.txt) Assert.True(ComponentHasMetadata(root, "pytest", "scope", "dev")); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task Pep508DirectReferenceIsParsedAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { // Create requirements.txt with direct reference var requirementsPath = Path.Combine(fixturePath, "requirements.txt"); await File.WriteAllTextAsync(requirementsPath, $"mypackage @ https://example.com/packages/mypackage-1.0.0.whl{Environment.NewLine}", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // Package should be found with URL reference Assert.True(ComponentHasMetadata(root, "mypackage", "declaredOnly", "true")); Assert.True(ComponentHasMetadata(root, "mypackage", "lockDirectUrl", "https://example.com/packages/mypackage-1.0.0.whl")); } finally { Directory.Delete(fixturePath, recursive: true); } } [Fact] public async Task RequirementsCycleIsDetectedAndHandledAsync() { var cancellationToken = TestContext.Current.CancellationToken; var fixturePath = CreateTemporaryWorkspace(); try { // Create requirements.txt that includes base var requirementsPath = Path.Combine(fixturePath, "requirements.txt"); await File.WriteAllTextAsync(requirementsPath, $"requests==2.28.0{Environment.NewLine}-r requirements-base.txt{Environment.NewLine}", cancellationToken); // Create requirements-base.txt that includes back to main (cycle) var baseRequirementsPath = Path.Combine(fixturePath, "requirements-base.txt"); await File.WriteAllTextAsync(baseRequirementsPath, $"urllib3==1.26.0{Environment.NewLine}-r requirements.txt{Environment.NewLine}", cancellationToken); var analyzers = new ILanguageAnalyzer[] { new PythonLanguageAnalyzer() }; // Should not throw due to infinite loop var json = await LanguageAnalyzerTestHarness.RunToJsonAsync( fixturePath, analyzers, cancellationToken); using var document = JsonDocument.Parse(json); var root = document.RootElement; // Both packages should still be found (cycle handled gracefully) Assert.True(ComponentHasMetadata(root, "requests", "declaredOnly", "true")); Assert.True(ComponentHasMetadata(root, "urllib3", "declaredOnly", "true")); } finally { Directory.Delete(fixturePath, recursive: true); } } }