292a6e94e8
- Updated TASKS.md to reflect changes in test fixtures for SCAN-PY-405-007. - Added multiple test cases to ensure deterministic output for various Python package scenarios, including conda environments, requirements files, and vendored directories. - Created new expected output files for conda packages (numpy, requests) and updated existing test fixtures for container whiteouts, wheel workspaces, and zipapp embedded requirements. - Introduced helper methods to create wheel and zipapp packages for testing purposes. - Added metadata files for new test fixtures to validate package detection and dependencies.
1005 lines
39 KiB
C#
1005 lines
39 KiB
C#
using System.IO.Compression;
|
|
using System.Security.Cryptography;
|
|
using System.Text;
|
|
using System.Text.Json;
|
|
using StellaOps.Scanner.Analyzers.Lang.Python;
|
|
using StellaOps.Scanner.Analyzers.Lang.Tests.Harness;
|
|
using StellaOps.Scanner.Analyzers.Lang.Tests.TestUtilities;
|
|
|
|
namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests;
|
|
|
|
public sealed class PythonLanguageAnalyzerTests
|
|
{
|
|
[Fact]
|
|
public async Task SimpleVenvFixtureProducesDeterministicOutputAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "simple-venv");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var usageHints = new LanguageUsageHints(new[]
|
|
{
|
|
Path.Combine(fixturePath, "bin", "simple-tool")
|
|
});
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken,
|
|
usageHints);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PipCacheFixtureProducesDeterministicOutputAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "pip-cache");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var usageHints = new LanguageUsageHints(new[]
|
|
{
|
|
Path.Combine(fixturePath, "lib", "python3.11", "site-packages", "cache_pkg-1.2.3.data", "scripts", "cache-tool")
|
|
});
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken,
|
|
usageHints);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task LayeredEditableFixtureMergesAcrossLayersAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "layered-editable");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var usageHints = new LanguageUsageHints(new[]
|
|
{
|
|
Path.Combine(fixturePath, "layer1", "usr", "bin", "layered-cli")
|
|
});
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken,
|
|
usageHints);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task CondaEnvFixtureProducesDeterministicOutputAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "conda-env");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RequirementsIncludesEditableFixtureProducesDeterministicOutputAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "requirements-includes-editable");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PipfileLockDefaultDevelopFixtureProducesDeterministicOutputAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "pipfile-lock-default-develop");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task WheelWorkspaceFixtureProducesDeterministicOutputAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "wheel-workspace");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var distDir = Path.Combine(fixturePath, "dist");
|
|
Directory.CreateDirectory(distDir);
|
|
|
|
var wheelPath = Path.Combine(distDir, "wheelpkg-1.0.0-py3-none-any.whl");
|
|
CreateWheelpkgWheel(wheelPath);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ZipappEmbeddedRequirementsFixtureProducesDeterministicOutputAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "zipapp-embedded-requirements");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var zipappPath = Path.Combine(fixturePath, "myapp.pyz");
|
|
CreateZipappWithEmbeddedRequirements(zipappPath);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task ContainerWhiteoutsFixtureProducesDeterministicOutputAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "container-whiteouts");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task VendoredDirectoryFixtureProducesDeterministicOutputAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "vendored-directory");
|
|
var goldenPath = Path.Combine(fixturePath, "expected.json");
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
|
|
fixturePath,
|
|
goldenPath,
|
|
analyzers,
|
|
cancellationToken);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task LockfileCollectorEmitsDeclaredOnlyComponentsAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
await CreatePythonPackageAsync(fixturePath, "locked", "1.0.0", cancellationToken);
|
|
await CreatePythonPackageAsync(fixturePath, "runtime-only", "2.0.0", cancellationToken);
|
|
|
|
var requirementsPath = Path.Combine(fixturePath, "requirements.txt");
|
|
var requirements = new StringBuilder()
|
|
.AppendLine("locked==1.0.0")
|
|
.AppendLine("declared-only==3.0.0")
|
|
.ToString();
|
|
await File.WriteAllTextAsync(requirementsPath, requirements, cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
Assert.True(ComponentHasMetadata(root, "declared-only", "declaredOnly", "true"));
|
|
Assert.True(ComponentHasMetadata(root, "declared-only", "lockSource", "requirements.txt"));
|
|
Assert.True(ComponentHasMetadata(root, "locked", "lockSource", "requirements.txt"));
|
|
Assert.True(ComponentHasMetadata(root, "runtime-only", "lockMissing", "true"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task EditableRequirementsUseExplicitKeyWithoutHostPathLeakAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
|
|
try
|
|
{
|
|
var editableDir = Path.Combine(fixturePath, "editable-src");
|
|
Directory.CreateDirectory(editableDir);
|
|
|
|
var requirementsPath = Path.Combine(fixturePath, "requirements.txt");
|
|
await File.WriteAllTextAsync(requirementsPath, $"--editable {editableDir}{Environment.NewLine}", cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
foreach (var component in root.EnumerateArray())
|
|
{
|
|
if (component.TryGetProperty("purl", out var purlElement) && purlElement.ValueKind == JsonValueKind.String)
|
|
{
|
|
Assert.DoesNotContain("@editable", purlElement.GetString(), StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
}
|
|
|
|
var editableComponent = root.EnumerateArray().Single(static component =>
|
|
component.TryGetProperty("name", out var nameElement)
|
|
&& string.Equals("editable-src", nameElement.GetString(), StringComparison.OrdinalIgnoreCase));
|
|
|
|
Assert.True(!editableComponent.TryGetProperty("purl", out var purlValue) || purlValue.ValueKind == JsonValueKind.Null);
|
|
|
|
var componentKey = editableComponent.GetProperty("componentKey").GetString();
|
|
Assert.StartsWith("explicit::python::pypi::editable-src::sha256:", componentKey, StringComparison.Ordinal);
|
|
|
|
var metadata = editableComponent.GetProperty("metadata");
|
|
Assert.Equal("true", metadata.GetProperty("declaredOnly").GetString());
|
|
Assert.Equal("editable", metadata.GetProperty("declared.sourceType").GetString());
|
|
Assert.Equal("requirements.txt", metadata.GetProperty("declared.source").GetString());
|
|
Assert.Equal("requirements.txt", metadata.GetProperty("declared.locator").GetString());
|
|
|
|
var editableSpec = metadata.GetProperty("lockEditablePath").GetString();
|
|
Assert.Equal("editable-src", editableSpec);
|
|
Assert.DoesNotContain(fixturePath, editableSpec, StringComparison.OrdinalIgnoreCase);
|
|
Assert.DoesNotContain(":", editableSpec, StringComparison.Ordinal);
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task WheelArchiveDistInfo_IsVerifiedFromRecordAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
var distDir = Path.Combine(fixturePath, "dist");
|
|
Directory.CreateDirectory(distDir);
|
|
|
|
var wheelPath = Path.Combine(distDir, "archivepkg-1.0.0-py3-none-any.whl");
|
|
|
|
var initBytes = Encoding.UTF8.GetBytes("__version__ = \"1.0.0\"\n");
|
|
var metadataBytes = Encoding.UTF8.GetBytes(
|
|
$"Metadata-Version: 2.1\nName: archivepkg\nVersion: 1.0.0\n{Environment.NewLine}");
|
|
var wheelBytes = Encoding.UTF8.GetBytes(
|
|
"Wheel-Version: 1.0\nGenerator: test\nRoot-Is-Purelib: true\nTag: py3-none-any\n");
|
|
|
|
var recordContent = new StringBuilder()
|
|
.AppendLine($"archivepkg/__init__.py,sha256={ComputeSha256Base64(initBytes)},{initBytes.Length}")
|
|
.AppendLine($"archivepkg-1.0.0.dist-info/METADATA,sha256={ComputeSha256Base64(metadataBytes)},{metadataBytes.Length}")
|
|
.AppendLine($"archivepkg-1.0.0.dist-info/WHEEL,sha256={ComputeSha256Base64(wheelBytes)},{wheelBytes.Length}")
|
|
.AppendLine("archivepkg-1.0.0.dist-info/RECORD,,")
|
|
.ToString();
|
|
var recordBytes = Encoding.UTF8.GetBytes(recordContent);
|
|
|
|
using (var stream = File.Create(wheelPath))
|
|
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: false))
|
|
{
|
|
WriteEntry(archive, "archivepkg/__init__.py", initBytes);
|
|
WriteEntry(archive, "archivepkg-1.0.0.dist-info/METADATA", metadataBytes);
|
|
WriteEntry(archive, "archivepkg-1.0.0.dist-info/WHEEL", wheelBytes);
|
|
WriteEntry(archive, "archivepkg-1.0.0.dist-info/RECORD", recordBytes);
|
|
}
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
Assert.True(ComponentHasMetadata(root, "archivepkg", "record.totalEntries", "4"));
|
|
Assert.True(ComponentHasMetadata(root, "archivepkg", "record.hashedEntries", "3"));
|
|
Assert.True(ComponentHasMetadata(root, "archivepkg", "record.missingFiles", "0"));
|
|
Assert.True(ComponentHasMetadata(root, "archivepkg", "record.hashMismatches", "0"));
|
|
Assert.True(ComponentHasMetadata(root, "archivepkg", "record.ioErrors", "0"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
|
|
static void WriteEntry(ZipArchive archive, string entryName, byte[] content)
|
|
{
|
|
var entry = archive.CreateEntry(entryName);
|
|
using var entryStream = entry.Open();
|
|
entryStream.Write(content, 0, content.Length);
|
|
}
|
|
|
|
static string ComputeSha256Base64(byte[] content)
|
|
=> Convert.ToBase64String(SHA256.HashData(content));
|
|
}
|
|
|
|
private static async Task CreatePythonPackageAsync(string root, string name, string version, CancellationToken cancellationToken)
|
|
{
|
|
var sitePackages = Path.Combine(root, "lib", "python3.11", "site-packages");
|
|
Directory.CreateDirectory(sitePackages);
|
|
|
|
var packageDir = Path.Combine(sitePackages, name);
|
|
Directory.CreateDirectory(packageDir);
|
|
|
|
var modulePath = Path.Combine(packageDir, "__init__.py");
|
|
var moduleContent = $"__version__ = \"{version}\"{Environment.NewLine}";
|
|
await File.WriteAllTextAsync(modulePath, moduleContent, cancellationToken);
|
|
|
|
var distInfoDir = Path.Combine(sitePackages, $"{name}-{version}.dist-info");
|
|
Directory.CreateDirectory(distInfoDir);
|
|
|
|
var metadataPath = Path.Combine(distInfoDir, "METADATA");
|
|
var metadataContent = $"Metadata-Version: 2.1{Environment.NewLine}Name: {name}{Environment.NewLine}Version: {version}{Environment.NewLine}";
|
|
await File.WriteAllTextAsync(metadataPath, metadataContent, cancellationToken);
|
|
|
|
var wheelPath = Path.Combine(distInfoDir, "WHEEL");
|
|
await File.WriteAllTextAsync(wheelPath, "Wheel-Version: 1.0", cancellationToken);
|
|
|
|
var entryPointsPath = Path.Combine(distInfoDir, "entry_points.txt");
|
|
await File.WriteAllTextAsync(entryPointsPath, string.Empty, cancellationToken);
|
|
|
|
var recordPath = Path.Combine(distInfoDir, "RECORD");
|
|
var recordContent = new StringBuilder()
|
|
.AppendLine($"{name}/__init__.py,sha256={ComputeSha256Base64(modulePath)},{new FileInfo(modulePath).Length}")
|
|
.AppendLine($"{name}-{version}.dist-info/METADATA,sha256={ComputeSha256Base64(metadataPath)},{new FileInfo(metadataPath).Length}")
|
|
.AppendLine($"{name}-{version}.dist-info/RECORD,,")
|
|
.AppendLine($"{name}-{version}.dist-info/WHEEL,sha256={ComputeSha256Base64(wheelPath)},{new FileInfo(wheelPath).Length}")
|
|
.AppendLine($"{name}-{version}.dist-info/entry_points.txt,sha256={ComputeSha256Base64(entryPointsPath)},{new FileInfo(entryPointsPath).Length}")
|
|
.ToString();
|
|
await File.WriteAllTextAsync(recordPath, recordContent, cancellationToken);
|
|
}
|
|
|
|
private static bool ComponentHasMetadata(JsonElement root, string componentName, string key, string? expectedValue)
|
|
{
|
|
foreach (var component in root.EnumerateArray())
|
|
{
|
|
if (!component.TryGetProperty("name", out var nameElement) ||
|
|
!string.Equals(nameElement.GetString(), componentName, StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!component.TryGetProperty("metadata", out var metadataElement) || metadataElement.ValueKind != JsonValueKind.Object)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
if (!metadataElement.TryGetProperty(key, out var valueElement))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
var actual = valueElement.GetString();
|
|
if (string.Equals(actual, expectedValue, StringComparison.Ordinal))
|
|
{
|
|
return true;
|
|
}
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
private static string ComputeSha256Base64(string path)
|
|
{
|
|
using var sha = SHA256.Create();
|
|
using var stream = File.OpenRead(path);
|
|
var hash = sha.ComputeHash(stream);
|
|
return Convert.ToBase64String(hash);
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DetectsSitecustomizeStartupHooksAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
// Create site-packages with sitecustomize.py
|
|
var sitePackages = Path.Combine(fixturePath, "lib", "python3.11", "site-packages");
|
|
Directory.CreateDirectory(sitePackages);
|
|
|
|
var sitecustomizePath = Path.Combine(sitePackages, "sitecustomize.py");
|
|
await File.WriteAllTextAsync(sitecustomizePath, "# Site customization\nprint('startup hook')", cancellationToken);
|
|
|
|
// Create a package
|
|
await CreatePythonPackageAsync(fixturePath, "test-pkg", "1.0.0", cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// Verify startup hooks metadata is present
|
|
Assert.True(ComponentHasMetadata(root, "test-pkg", "startupHooks.detected", "true"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DetectsPthFilesWithImportDirectivesAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
var sitePackages = Path.Combine(fixturePath, "lib", "python3.11", "site-packages");
|
|
Directory.CreateDirectory(sitePackages);
|
|
|
|
// Create a .pth file with import directive
|
|
var pthPath = Path.Combine(sitePackages, "test-hooks.pth");
|
|
await File.WriteAllTextAsync(pthPath, "import some_module\n/some/path", cancellationToken);
|
|
|
|
// Create a package
|
|
await CreatePythonPackageAsync(fixturePath, "test-pkg", "1.0.0", cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// Verify .pth import warning metadata is present
|
|
Assert.True(ComponentHasMetadata(root, "test-pkg", "pthFiles.withImports.detected", "true"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DetectsOciLayerSitePackagesAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
// Create OCI layer structure with packages
|
|
var layersDir = Path.Combine(fixturePath, "layers", "layer1", "fs");
|
|
var sitePackages = Path.Combine(layersDir, "usr", "lib", "python3.11", "site-packages");
|
|
Directory.CreateDirectory(sitePackages);
|
|
|
|
// Create a package in the layer
|
|
var packageDir = Path.Combine(sitePackages, "layered-pkg");
|
|
Directory.CreateDirectory(packageDir);
|
|
|
|
var modulePath = Path.Combine(packageDir, "__init__.py");
|
|
await File.WriteAllTextAsync(modulePath, "__version__ = \"1.0.0\"", cancellationToken);
|
|
|
|
var distInfoDir = Path.Combine(sitePackages, "layered-pkg-1.0.0.dist-info");
|
|
Directory.CreateDirectory(distInfoDir);
|
|
|
|
var metadataPath = Path.Combine(distInfoDir, "METADATA");
|
|
await File.WriteAllTextAsync(metadataPath, "Metadata-Version: 2.1\nName: layered-pkg\nVersion: 1.0.0", cancellationToken);
|
|
|
|
var wheelPath = Path.Combine(distInfoDir, "WHEEL");
|
|
await File.WriteAllTextAsync(wheelPath, "Wheel-Version: 1.0", cancellationToken);
|
|
|
|
var recordPath = Path.Combine(distInfoDir, "RECORD");
|
|
await File.WriteAllTextAsync(recordPath, "", cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// Verify the package from OCI layers was discovered
|
|
var found = false;
|
|
foreach (var component in root.EnumerateArray())
|
|
{
|
|
if (component.TryGetProperty("name", out var nameElement) &&
|
|
string.Equals(nameElement.GetString(), "layered-pkg", StringComparison.OrdinalIgnoreCase))
|
|
{
|
|
found = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
Assert.True(found, "Package from OCI layer should be discovered");
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task EggInfoPackagesAreDetectedAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = TestPaths.ResolveFixture("lang", "python", "egg-info");
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
var expectedPath = Path.Combine("lib", "python3.11", "site-packages", "egg_info_pkg-1.2.3.egg-info")
|
|
.Replace('\\', '/');
|
|
|
|
Assert.True(ComponentHasMetadata(root, "egg-info-pkg", "provenance", "egg-info"));
|
|
Assert.True(ComponentHasMetadata(root, "egg-info-pkg", "record.totalEntries", "4"));
|
|
Assert.True(ComponentHasMetadata(root, "egg-info-pkg", "distInfoPath", expectedPath));
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DetectsPythonEnvironmentVariablesAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
// Create environment file with PYTHONPATH
|
|
var envPath = Path.Combine(fixturePath, ".env");
|
|
await File.WriteAllTextAsync(envPath, "PYTHONPATH=/app/lib:/app/vendor", cancellationToken);
|
|
|
|
// Create a package
|
|
await CreatePythonPackageAsync(fixturePath, "test-pkg", "1.0.0", cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// Verify PYTHONPATH warning metadata is present
|
|
Assert.True(ComponentHasMetadata(root, "test-pkg", "env.pythonpath", "/app/lib:/app/vendor"));
|
|
Assert.True(ComponentHasMetadata(root, "test-pkg", "env.pythonpath.warning", "PYTHONPATH is set; may affect module resolution"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task DetectsPyvenvConfigAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
// Create pyvenv.cfg file
|
|
var pyvenvPath = Path.Combine(fixturePath, "pyvenv.cfg");
|
|
await File.WriteAllTextAsync(pyvenvPath, "home = /usr/local/bin\ninclude-system-site-packages = false", cancellationToken);
|
|
|
|
// Create a package
|
|
await CreatePythonPackageAsync(fixturePath, "test-pkg", "1.0.0", cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// Verify PYTHONHOME warning metadata is present (from pyvenv.cfg home)
|
|
Assert.True(ComponentHasMetadata(root, "test-pkg", "env.pythonhome", "/usr/local/bin"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
private static string CreateTemporaryWorkspace()
|
|
{
|
|
var path = Path.Combine(Path.GetTempPath(), $"stellaops-python-{Guid.NewGuid():N}");
|
|
Directory.CreateDirectory(path);
|
|
return path;
|
|
}
|
|
|
|
private static void CreateWheelpkgWheel(string wheelPath)
|
|
{
|
|
Directory.CreateDirectory(Path.GetDirectoryName(wheelPath)!);
|
|
|
|
var initBytes = Encoding.UTF8.GetBytes("__version__ = \"1.0.0\"\n");
|
|
var metadataBytes = Encoding.UTF8.GetBytes(
|
|
$"Metadata-Version: 2.1\nName: wheelpkg\nVersion: 1.0.0\nSummary: Wheel fixture\n{Environment.NewLine}");
|
|
var wheelBytes = Encoding.UTF8.GetBytes(
|
|
"Wheel-Version: 1.0\nGenerator: stellaops-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n");
|
|
|
|
var recordContent = new StringBuilder()
|
|
.AppendLine($"wheelpkg/__init__.py,sha256={ComputeSha256Base64(initBytes)},{initBytes.Length}")
|
|
.AppendLine($"wheelpkg-1.0.0.dist-info/METADATA,sha256={ComputeSha256Base64(metadataBytes)},{metadataBytes.Length}")
|
|
.AppendLine($"wheelpkg-1.0.0.dist-info/WHEEL,sha256={ComputeSha256Base64(wheelBytes)},{wheelBytes.Length}")
|
|
.AppendLine("wheelpkg-1.0.0.dist-info/RECORD,,")
|
|
.ToString();
|
|
var recordBytes = Encoding.UTF8.GetBytes(recordContent);
|
|
|
|
if (File.Exists(wheelPath))
|
|
{
|
|
File.Delete(wheelPath);
|
|
}
|
|
|
|
using (var stream = File.Create(wheelPath))
|
|
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: false))
|
|
{
|
|
WriteEntry(archive, "wheelpkg/__init__.py", initBytes);
|
|
WriteEntry(archive, "wheelpkg-1.0.0.dist-info/METADATA", metadataBytes);
|
|
WriteEntry(archive, "wheelpkg-1.0.0.dist-info/WHEEL", wheelBytes);
|
|
WriteEntry(archive, "wheelpkg-1.0.0.dist-info/RECORD", recordBytes);
|
|
}
|
|
|
|
static void WriteEntry(ZipArchive archive, string entryName, byte[] content)
|
|
{
|
|
var entry = archive.CreateEntry(entryName);
|
|
entry.LastWriteTime = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
|
using var entryStream = entry.Open();
|
|
entryStream.Write(content, 0, content.Length);
|
|
}
|
|
|
|
static string ComputeSha256Base64(byte[] content)
|
|
=> Convert.ToBase64String(SHA256.HashData(content));
|
|
}
|
|
|
|
private static void CreateZipappWithEmbeddedRequirements(string zipappPath)
|
|
{
|
|
if (File.Exists(zipappPath))
|
|
{
|
|
File.Delete(zipappPath);
|
|
}
|
|
|
|
using var fileStream = File.Create(zipappPath);
|
|
|
|
var shebangBytes = Encoding.UTF8.GetBytes("#!/usr/bin/python3.11\n");
|
|
fileStream.Write(shebangBytes);
|
|
|
|
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, leaveOpen: true);
|
|
|
|
WriteTextEntry(archive, "__main__.py", "print('hello')\n");
|
|
WriteTextEntry(archive, "requirements.txt", "requests==2.28.0\nflask==2.1.0\n");
|
|
|
|
static void WriteTextEntry(ZipArchive archive, string name, string content)
|
|
{
|
|
var entry = archive.CreateEntry(name, CompressionLevel.NoCompression);
|
|
entry.LastWriteTime = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
|
using var stream = entry.Open();
|
|
using var writer = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
|
|
writer.Write(content);
|
|
}
|
|
}
|
|
|
|
// ===== SCAN-PY-405-007 Fixtures =====
|
|
|
|
[Fact]
|
|
public async Task RequirementsWithIncludesAreFollowedAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
// Create main requirements.txt that includes another file
|
|
var requirementsPath = Path.Combine(fixturePath, "requirements.txt");
|
|
await File.WriteAllTextAsync(requirementsPath, $"requests==2.28.0{Environment.NewLine}-r requirements-base.txt{Environment.NewLine}", cancellationToken);
|
|
|
|
// Create included requirements file
|
|
var baseRequirementsPath = Path.Combine(fixturePath, "requirements-base.txt");
|
|
await File.WriteAllTextAsync(baseRequirementsPath, $"urllib3==1.26.0{Environment.NewLine}certifi==2022.12.7{Environment.NewLine}", cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// All three packages should be found (from both files)
|
|
Assert.True(ComponentHasMetadata(root, "requests", "declaredOnly", "true"));
|
|
Assert.True(ComponentHasMetadata(root, "urllib3", "declaredOnly", "true"));
|
|
Assert.True(ComponentHasMetadata(root, "certifi", "declaredOnly", "true"));
|
|
|
|
// urllib3 and certifi should come from the included file
|
|
Assert.True(ComponentHasMetadata(root, "urllib3", "lockSource", "requirements-base.txt"));
|
|
Assert.True(ComponentHasMetadata(root, "certifi", "lockSource", "requirements-base.txt"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task PipfileLockDevelopSectionIsParsedAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
// Create Pipfile.lock with default and develop sections
|
|
var pipfileLockPath = Path.Combine(fixturePath, "Pipfile.lock");
|
|
var pipfileLock = """
|
|
{
|
|
"_meta": { "sources": [] },
|
|
"default": {
|
|
"requests": { "version": "==2.28.0" }
|
|
},
|
|
"develop": {
|
|
"pytest": { "version": "==7.0.0" }
|
|
}
|
|
}
|
|
""";
|
|
await File.WriteAllTextAsync(pipfileLockPath, pipfileLock, cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// Both packages should be found
|
|
Assert.True(ComponentHasMetadata(root, "requests", "declaredOnly", "true"));
|
|
Assert.True(ComponentHasMetadata(root, "pytest", "declaredOnly", "true"));
|
|
|
|
// requests should be prod scope, pytest should be dev scope
|
|
Assert.True(ComponentHasMetadata(root, "requests", "scope", "prod"));
|
|
Assert.True(ComponentHasMetadata(root, "pytest", "scope", "dev"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RequirementsDevTxtGetsScopeDevAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
// Create requirements.txt for prod
|
|
var requirementsPath = Path.Combine(fixturePath, "requirements.txt");
|
|
await File.WriteAllTextAsync(requirementsPath, $"flask==2.0.0{Environment.NewLine}", cancellationToken);
|
|
|
|
// Create requirements-dev.txt for dev dependencies
|
|
var requirementsDevPath = Path.Combine(fixturePath, "requirements-dev.txt");
|
|
await File.WriteAllTextAsync(requirementsDevPath, $"pytest==7.0.0{Environment.NewLine}", cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// flask should be prod scope (from requirements.txt)
|
|
Assert.True(ComponentHasMetadata(root, "flask", "scope", "prod"));
|
|
|
|
// pytest should be dev scope (from requirements-dev.txt)
|
|
Assert.True(ComponentHasMetadata(root, "pytest", "scope", "dev"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task Pep508DirectReferenceIsParsedAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
// Create requirements.txt with direct reference
|
|
var requirementsPath = Path.Combine(fixturePath, "requirements.txt");
|
|
await File.WriteAllTextAsync(requirementsPath,
|
|
$"mypackage @ https://example.com/packages/mypackage-1.0.0.whl{Environment.NewLine}",
|
|
cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// Package should be found with URL reference
|
|
Assert.True(ComponentHasMetadata(root, "mypackage", "declaredOnly", "true"));
|
|
Assert.True(ComponentHasMetadata(root, "mypackage", "lockDirectUrl", "https://example.com/packages/mypackage-1.0.0.whl"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
|
|
[Fact]
|
|
public async Task RequirementsCycleIsDetectedAndHandledAsync()
|
|
{
|
|
var cancellationToken = TestContext.Current.CancellationToken;
|
|
var fixturePath = CreateTemporaryWorkspace();
|
|
try
|
|
{
|
|
// Create requirements.txt that includes base
|
|
var requirementsPath = Path.Combine(fixturePath, "requirements.txt");
|
|
await File.WriteAllTextAsync(requirementsPath, $"requests==2.28.0{Environment.NewLine}-r requirements-base.txt{Environment.NewLine}", cancellationToken);
|
|
|
|
// Create requirements-base.txt that includes back to main (cycle)
|
|
var baseRequirementsPath = Path.Combine(fixturePath, "requirements-base.txt");
|
|
await File.WriteAllTextAsync(baseRequirementsPath, $"urllib3==1.26.0{Environment.NewLine}-r requirements.txt{Environment.NewLine}", cancellationToken);
|
|
|
|
var analyzers = new ILanguageAnalyzer[]
|
|
{
|
|
new PythonLanguageAnalyzer()
|
|
};
|
|
|
|
// Should not throw due to infinite loop
|
|
var json = await LanguageAnalyzerTestHarness.RunToJsonAsync(
|
|
fixturePath,
|
|
analyzers,
|
|
cancellationToken);
|
|
|
|
using var document = JsonDocument.Parse(json);
|
|
var root = document.RootElement;
|
|
|
|
// Both packages should still be found (cycle handled gracefully)
|
|
Assert.True(ComponentHasMetadata(root, "requests", "declaredOnly", "true"));
|
|
Assert.True(ComponentHasMetadata(root, "urllib3", "declaredOnly", "true"));
|
|
}
|
|
finally
|
|
{
|
|
Directory.Delete(fixturePath, recursive: true);
|
|
}
|
|
}
|
|
}
|