stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

feat(python-analyzer): Enhance deterministic output tests and add new fixtures

Browse Source 
- Updated TASKS.md to reflect changes in test fixtures for SCAN-PY-405-007.
- Added multiple test cases to ensure deterministic output for various Python package scenarios, including conda environments, requirements files, and vendored directories.
- Created new expected output files for conda packages (numpy, requests) and updated existing test fixtures for container whiteouts, wheel workspaces, and zipapp embedded requirements.
- Introduced helper methods to create wheel and zipapp packages for testing purposes.
- Added metadata files for new test fixtures to validate package detection and dependencies.
This commit is contained in:
StellaOps Bot
2025-12-21 17:51:19 +02:00
parent 22d67f203f
commit 292a6e94e8
29 changed files with 1043 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Packaging/ContainerOverlayHandler.cs
View File

@@ -88,7 +88,12 @@ internal sealed partial class ContainerOverlayHandler
isComplete = false;
}
var layerFiles = enumerateFiles(layer.Path).ToList();
var layerRoot = NormalizePath(layer.Path);
var layerFiles = enumerateFiles(layer.Path)
.Select(file => NormalizeLayerRelativePath(layerRoot, file))
.Where(static file => file is not null)
.Cast<string>()
.ToList();
// First pass: collect whiteouts and opaque markers
var layerWhiteouts = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
@@ -121,6 +126,16 @@ internal sealed partial class ContainerOverlayHandler
: targetName;
layerWhiteouts.Add(targetPath);
// Whiteouts can target files or directories. If a directory is whited out,
// all previously-visible files under it must be removed.
var toRemove = visiblePaths.Where(p => IsUnderDirectory(p, targetPath)).ToList();
foreach (var path in toRemove)
{
visiblePaths.Remove(path);
whiteoutedPaths.Add(path);
}
visiblePaths.Remove(targetPath);
whiteoutedPaths.Add(targetPath);
}
@@ -163,6 +178,32 @@ internal sealed partial class ContainerOverlayHandler
warning);
}
private static string? NormalizeLayerRelativePath(string normalizedLayerRoot, string path)
{
var normalized = NormalizePath(path);
if (normalized.Length == 0)
{
return null;
}
if (!normalized.StartsWith(normalizedLayerRoot, StringComparison.OrdinalIgnoreCase))
{
return null;
}
if (normalized.Length == normalizedLayerRoot.Length)
{
return string.Empty;
}
if (normalized[normalizedLayerRoot.Length] != '/')
{
return null;
}
return normalized[(normalizedLayerRoot.Length + 1)..];
}
/// <summary>
/// Checks if a path would be visible after overlay processing.
/// </summary>

17
src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/Internal/Vendoring/VendoredPackageDetector.cs
View File

@@ -160,29 +160,30 @@ internal static partial class VendoredPackageDetector
// with the same name as the package (normalized to lowercase with underscores).
// E.g., dist-info at "site-packages/pip-23.0.dist-info" means package at "site-packages/pip/"
string? baseDir = null;
var baseDir = string.Empty;
if (!string.IsNullOrEmpty(package.MetadataPath))
{
// Get the directory containing dist-info (usually site-packages)
baseDir = Path.GetDirectoryName(package.MetadataPath);
// In the Python VFS, MetadataPath may be just "<name>-<version>.dist-info" (rooted at VFS root).
// Treat missing directory name as VFS root.
baseDir = Path.GetDirectoryName(package.MetadataPath) ?? string.Empty;
}
else if (!string.IsNullOrEmpty(package.Location))
{
baseDir = package.Location;
}
if (string.IsNullOrEmpty(baseDir))
{
return null;
}
// The package directory is baseDir + package module name
// Use the first top-level module if available, otherwise use the normalized package name
var moduleName = package.TopLevelModules.Length > 0
? package.TopLevelModules[0]
: package.NormalizedName;
if (string.IsNullOrWhiteSpace(moduleName))
{
return null;
}
return Path.Combine(baseDir, moduleName).Replace('\\', '/');
}

294
src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/PythonLanguageAnalyzer.cs
View File

@@ -27,6 +27,8 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
var matchedLocks = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
var hasLockEntries = lockData.Entries.Count > 0;
var containerOverlay = TryBuildContainerOverlay(context.RootPath);
// Detect Python runtime in container layers
var runtimeInfo = PythonContainerAdapter.DetectRuntime(context.RootPath);
@@ -44,6 +46,7 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
var packageDiscovery = new PythonPackageDiscovery();
var discoveryResult = await packageDiscovery.DiscoverAsync(vfs, cancellationToken).ConfigureAwait(false);
var vendoringByPackage = await BuildVendoringMapAsync(vfs, discoveryResult, cancellationToken).ConfigureAwait(false);
foreach (var package in discoveryResult.Packages
.Where(static p => !string.IsNullOrWhiteSpace(p.Version))
@@ -52,6 +55,8 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
{
cancellationToken.ThrowIfCancellationRequested();
vendoringByPackage.TryGetValue(package.NormalizedName, out var vendoringAnalysis);
await EmitDiscoveredPackageAsync(
context,
writer,
@@ -64,6 +69,8 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
environment,
startupHooks,
zipappAnalysis,
containerOverlay,
vendoringAnalysis,
cancellationToken)
.ConfigureAwait(false);
}
@@ -221,6 +228,8 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
PythonEnvironment environment,
PythonStartupHooks startupHooks,
PythonZipappAnalysis zipappAnalysis,
ContainerOverlayHandler.OverlayResult? containerOverlay,
VendoringAnalysis? vendoringAnalysis,
CancellationToken cancellationToken)
{
var version = package.Version!.Trim();
@@ -243,6 +252,21 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
}
var metadataDirectory = TryResolvePhysicalMetadataDirectory(vfs, package, out var metadataFile);
if (vendoringAnalysis is not null)
{
metadata.AddRange(VendoringMetadataBuilder.BuildParentMetadata(vendoringAnalysis));
}
if (metadataFile is not null &&
!metadataFile.IsFromArchive &&
containerOverlay is not null &&
TryGetContainerOverlayPath(containerOverlay, metadataFile.AbsolutePath, out var overlayPath) &&
!ContainerOverlayHandler.IsPathVisible(containerOverlay, overlayPath))
{
return;
}
if (metadataDirectory is not null)
{
PythonDistribution? distribution;
@@ -289,6 +313,7 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
evidence: evidence,
usedByEntrypoint: distribution.UsedByEntrypoint);
EmitVendoredEmbeddedComponents(context, writer, vfs, vendoringAnalysis, version);
return;
}
@@ -313,6 +338,7 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
evidence: archiveDistribution.SortedEvidence,
usedByEntrypoint: archiveDistribution.UsedByEntrypoint);
EmitVendoredEmbeddedComponents(context, writer, vfs, vendoringAnalysis, version);
return;
}
}
@@ -329,6 +355,8 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
metadata: metadata,
evidence: evidenceFallback,
usedByEntrypoint: false);
EmitVendoredEmbeddedComponents(context, writer, vfs, vendoringAnalysis, version);
}
private static string? TryResolvePhysicalMetadataDirectory(
@@ -363,6 +391,13 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
var location = package.Location;
if (string.IsNullOrWhiteSpace(location) && !string.IsNullOrWhiteSpace(package.MetadataPath))
{
if (package.Kind == PythonPackageKind.Conda)
{
var normalizedPath = package.MetadataPath.Replace('\\', '/').Trim('/');
var lastSlash = normalizedPath.LastIndexOf('/');
location = lastSlash > 0 ? normalizedPath[..lastSlash] : normalizedPath;
}
var metadataName = package.Kind == PythonPackageKind.Egg ? "PKG-INFO" : "METADATA";
var file = vfs.GetFile($"{package.MetadataPath}/{metadataName}");
@@ -392,6 +427,19 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
PythonPackageInfo package,
PythonVirtualFile? metadataFile)
{
if (package.Kind == PythonPackageKind.Conda && !string.IsNullOrWhiteSpace(package.MetadataPath))
{
return new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.File,
"conda-meta",
package.MetadataPath.Replace('\\', '/').TrimStart('/'),
Value: null,
Sha256: null)
};
}
if (metadataFile is not null)
{
var locator = metadataFile.IsFromArchive && metadataFile.ArchivePath is not null
@@ -555,6 +603,27 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
metadata.Add(new KeyValuePair<string, string?>("zipapps.detected", "true"));
metadata.Add(new KeyValuePair<string, string?>("zipapps.count", zipappAnalysis.Zipapps.Count.ToString()));
var embeddedDeps = zipappAnalysis.Zipapps
.SelectMany(static z => z.EmbeddedDependencies)
.Select(static d => d.Trim())
.Where(static d => d.Length > 0)
.Distinct(StringComparer.OrdinalIgnoreCase)
.OrderBy(static d => d, StringComparer.Ordinal)
.ToArray();
if (embeddedDeps.Length > 0)
{
metadata.Add(new KeyValuePair<string, string?>("zipapps.embeddedDeps.count", embeddedDeps.Length.ToString()));
var sample = embeddedDeps.Take(12).ToArray();
metadata.Add(new KeyValuePair<string, string?>("zipapps.embeddedDeps.sample", string.Join(';', sample)));
if (embeddedDeps.Length > sample.Length)
{
metadata.Add(new KeyValuePair<string, string?>("zipapps.embeddedDeps.sampleTruncated", "true"));
}
}
// Add version information from zipapp shebangs
var versions = zipappAnalysis.Zipapps
.Where(z => z.PythonVersion != null)
@@ -583,4 +652,229 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
}
}
}
private static async Task<Dictionary<string, VendoringAnalysis>> BuildVendoringMapAsync(
PythonVirtualFileSystem vfs,
PythonPackageDiscoveryResult discoveryResult,
CancellationToken cancellationToken)
{
var results = await VendoredPackageDetector.AnalyzeAllAsync(vfs, discoveryResult, cancellationToken).ConfigureAwait(false);
var map = new Dictionary<string, VendoringAnalysis>(StringComparer.OrdinalIgnoreCase);
foreach (var analysis in results)
{
var key = PythonPackageInfo.NormalizeName(analysis.PackageName);
map[key] = analysis;
}
return map;
}
private static ContainerOverlayHandler.OverlayResult? TryBuildContainerOverlay(string rootPath)
{
IReadOnlyList<ContainerOverlayHandler.LayerInfo> layers;
try
{
layers = ContainerOverlayHandler.DiscoverLayers(rootPath);
}
catch
{
return null;
}
if (layers.Count == 0)
{
return null;
}
try
{
return ContainerOverlayHandler.ProcessLayers(layers, EnumerateOverlayRelevantFiles);
}
catch
{
return null;
}
}
private static IEnumerable<string> EnumerateOverlayRelevantFiles(string layerPath)
{
if (string.IsNullOrWhiteSpace(layerPath) || !Directory.Exists(layerPath))
{
yield break;
}
var options = new EnumerationOptions
{
RecurseSubdirectories = true,
IgnoreInaccessible = true,
AttributesToSkip = FileAttributes.Device | FileAttributes.ReparsePoint
};
var results = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
try
{
foreach (var file in Directory.EnumerateFiles(layerPath, ".wh.*", options))
{
results.Add(file);
}
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
try
{
foreach (var file in Directory.EnumerateFiles(layerPath, "METADATA", options))
{
if (file.Contains(".dist-info", StringComparison.OrdinalIgnoreCase))
{
results.Add(file);
}
}
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
try
{
foreach (var file in Directory.EnumerateFiles(layerPath, "PKG-INFO", options))
{
if (file.Contains(".egg-info", StringComparison.OrdinalIgnoreCase))
{
results.Add(file);
}
}
}
catch (IOException)
{
}
catch (UnauthorizedAccessException)
{
}
foreach (var result in results.OrderBy(static path => path, StringComparer.OrdinalIgnoreCase))
{
yield return result;
}
}
private static bool TryGetContainerOverlayPath(ContainerOverlayHandler.OverlayResult overlay, string absolutePath, out string overlayPath)
{
var normalized = absolutePath.Replace('\\', '/').TrimEnd('/');
foreach (var layer in overlay.ProcessedLayers)
{
var layerPath = layer.Path.Replace('\\', '/').TrimEnd('/');
if (normalized.StartsWith(layerPath + "/", StringComparison.OrdinalIgnoreCase))
{
overlayPath = normalized[(layerPath.Length + 1)..];
return true;
}
}
overlayPath = string.Empty;
return false;
}
private static void EmitVendoredEmbeddedComponents(
LanguageAnalyzerContext context,
LanguageComponentWriter writer,
PythonVirtualFileSystem vfs,
VendoringAnalysis? vendoringAnalysis,
string? parentVersion)
{
if (vendoringAnalysis is null)
{
return;
}
var embeddedToEmit = VendoringMetadataBuilder.GetEmbeddedToEmitSeparately(vendoringAnalysis, parentVersion);
if (embeddedToEmit.Count == 0)
{
return;
}
foreach (var embedded in embeddedToEmit)
{
var embeddedMetadata = VendoringMetadataBuilder.BuildEmbeddedMetadata(embedded, parentVersion, vendoringAnalysis.Confidence);
var evidence = TryBuildVendoredEvidence(context, vfs, embedded.Path);
var componentKey = LanguageExplicitKey.Create(
analyzerId: "python",
ecosystem: "pypi",
name: PythonPathHelper.NormalizePackageName(embedded.Name),
spec: embedded.Version ?? string.Empty,
originLocator: embedded.Path);
writer.AddFromExplicitKey(
analyzerId: "python",
componentKey: componentKey,
purl: embedded.Purl,
name: embedded.Name,
version: embedded.Version,
type: "pypi",
metadata: embeddedMetadata,
evidence: evidence,
usedByEntrypoint: false);
}
}
private static IReadOnlyCollection<LanguageComponentEvidence>? TryBuildVendoredEvidence(
LanguageAnalyzerContext context,
PythonVirtualFileSystem vfs,
string embeddedPath)
{
var candidates = new[]
{
$"{embeddedPath}/__init__.py",
$"{embeddedPath}/_version.py",
$"{embeddedPath}/version.py",
$"{embeddedPath}/__version__.py",
embeddedPath + ".py"
};
foreach (var candidate in candidates)
{
var file = vfs.GetFile(candidate);
if (file is null)
{
continue;
}
if (file.IsFromArchive && file.ArchivePath is not null)
{
return new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.File,
"vendored",
PythonPathHelper.NormalizeRelative(context, file.ArchivePath),
Value: file.AbsolutePath,
Sha256: null)
};
}
return new[]
{
new LanguageComponentEvidence(
LanguageEvidenceKind.File,
"vendored",
PythonPathHelper.NormalizeRelative(context, file.AbsolutePath),
Value: null,
Sha256: null)
};
}
return null;
}
}

2
src/Scanner/__Libraries/StellaOps.Scanner.Analyzers.Lang.Python/TASKS.md
View File

@@ -10,7 +10,7 @@
| SCAN-PY-405-004 | DONE | Container overlay contract implemented: OCI whiteout semantics (`.wh.*`, `.wh..wh..opq`), deterministic layer ordering, `container.overlayIncomplete` metadata marker. | 2025-12-13 |
| SCAN-PY-405-005 | DONE | Vendoring integration: `VendoringMetadataBuilder` for parent metadata + embedded components with High confidence. | 2025-12-13 |
| SCAN-PY-405-006 | DONE | Scope classification added (prod/dev/docs/build) from lock sections and file names per Interlock 4. Usage signals remain default. | 2025-12-13 |
| SCAN-PY-405-007 | DONE | Added test fixtures for includes, Pipfile.lock develop, scope classification, PEP 508 direct refs, cycle detection. | 2025-12-13 |
| SCAN-PY-405-007 | DONE | Added deterministic fixtures + goldens: conda-meta env, requirements includes+editable, Pipfile.lock default+develop, wheel workspace, zipapp embedded requirements, container whiteouts, and vendored directories. | 2025-12-21 |
| SCAN-PY-405-008 | DONE | Docs + deterministic offline bench for Python analyzer contract. | 2025-12-13 |
## Completed Contracts (Action Decisions 2025-12-13)

11
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/conda-env/conda-meta/numpy-1.26.0-0.json Normal file
View File

@@ -0,0 +1,11 @@
{
"name": "numpy",
"version": "1.26.0",
"depends": [
"python >=3.11"
],
"files": [
"numpy/__init__.py"
],
"requested": true
}

12
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/conda-env/conda-meta/requests-2.31.0-0.json Normal file
View File

@@ -0,0 +1,12 @@
{
"name": "requests",
"version": "2.31.0",
"depends": [
"python >=3.11",
"urllib3 >=1.26"
],
"files": [
"requests/__init__.py"
],
"requested": false
}

44
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/conda-env/expected.json Normal file
View File

@@ -0,0 +1,44 @@
[
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/numpy@1.26.0",
"purl": "pkg:pypi/numpy@1.26.0",
"name": "numpy",
"version": "1.26.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"pkg.confidence": "High",
"pkg.kind": "Conda",
"pkg.location": "conda-meta"
},
"evidence": [
{
"kind": "file",
"source": "conda-meta",
"locator": "conda-meta/numpy-1.26.0-0.json"
}
]
},
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/requests@2.31.0",
"purl": "pkg:pypi/requests@2.31.0",
"name": "requests",
"version": "2.31.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"pkg.confidence": "High",
"pkg.kind": "Conda",
"pkg.location": "conda-meta"
},
"evidence": [
{
"kind": "file",
"source": "conda-meta",
"locator": "conda-meta/requests-2.31.0-0.json"
}
]
}
]

36
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/container-whiteouts/expected.json Normal file
View File

@@ -0,0 +1,36 @@
[
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/visible@2.0.0",
"purl": "pkg:pypi/visible@2.0.0",
"name": "visible",
"version": "2.0.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"distInfoPath": "layers/layer1/usr/lib/python3.11/site-packages/visible-2.0.0.dist-info",
"name": "visible",
"normalizedName": "visible",
"pkg.confidence": "High",
"pkg.kind": "Wheel",
"pkg.location": "layers/layer1/usr/lib/python3.11/site-packages/visible-2.0.0.dist-info",
"provenance": "dist-info",
"record.hashMismatches": "0",
"record.hashedEntries": "0",
"record.ioErrors": "0",
"record.missingFiles": "0",
"record.totalEntries": "0",
"runtime.libPaths.count": "2",
"runtime.versions": "3.11",
"summary": "Visible after overlay",
"version": "2.0.0"
},
"evidence": [
{
"kind": "file",
"source": "METADATA",
"locator": "layers/layer1/usr/lib/python3.11/site-packages/visible-2.0.0.dist-info/METADATA"
}
]
}
]

5
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/container-whiteouts/layers/layer0/usr/lib/python3.11/site-packages/whited-1.0.0.dist-info/METADATA Normal file
View File

@@ -0,0 +1,5 @@
Metadata-Version: 2.1
Name: whited
Version: 1.0.0
Summary: Should be removed by whiteout

1
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/container-whiteouts/layers/layer1/usr/lib/python3.11/site-packages/.wh.whited-1.0.0.dist-info Normal file
View File

@@ -0,0 +1 @@

5
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/container-whiteouts/layers/layer1/usr/lib/python3.11/site-packages/visible-2.0.0.dist-info/METADATA Normal file
View File

@@ -0,0 +1,5 @@
Metadata-Version: 2.1
Name: visible
Version: 2.0.0
Summary: Visible after overlay

15
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pipfile-lock-default-develop/Pipfile.lock generated Normal file
View File

@@ -0,0 +1,15 @@
{
"_meta": {
"sources": []
},
"default": {
"requests": {
"version": "==2.28.0"
}
},
"develop": {
"pytest": {
"version": "==7.0.0"
}
}
}

52
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/pipfile-lock-default-develop/expected.json Normal file
View File

@@ -0,0 +1,52 @@
[
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/pytest@7.0.0",
"purl": "pkg:pypi/pytest@7.0.0",
"name": "pytest",
"version": "7.0.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"declaredOnly": "true",
"lockLocator": "Pipfile.lock",
"lockSource": "Pipfile.lock",
"pkg.confidence": "Medium",
"pkg.kind": "DeclaredOnly",
"pkg.location": "Pipfile.lock",
"scope": "dev"
},
"evidence": [
{
"kind": "metadata",
"source": "Pipfile.lock",
"locator": "Pipfile.lock"
}
]
},
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/requests@2.28.0",
"purl": "pkg:pypi/requests@2.28.0",
"name": "requests",
"version": "2.28.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"declaredOnly": "true",
"lockLocator": "Pipfile.lock",
"lockSource": "Pipfile.lock",
"pkg.confidence": "Medium",
"pkg.kind": "DeclaredOnly",
"pkg.location": "Pipfile.lock",
"scope": "prod"
},
"evidence": [
{
"kind": "metadata",
"source": "Pipfile.lock",
"locator": "Pipfile.lock"
}
]
}
]

3
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/requirements-includes-editable/editable-src/pyproject.toml Normal file
View File

@@ -0,0 +1,3 @@
[project]
name = "editable-src"
version = "0.0.0"

106
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/requirements-includes-editable/expected.json Normal file
View File

@@ -0,0 +1,106 @@
[
{
"analyzerId": "python",
"componentKey": "explicit::python::pypi::editable-src::sha256:5bd6cf3f7ac212830a9fcbc83b06ab72e79bafc1c94ea98a3d0560547c96c923",
"name": "editable-src",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"declared.locator": "requirements.txt",
"declared.scope": "prod",
"declared.source": "requirements.txt",
"declared.sourceType": "editable",
"declared.versionSpec": "./editable-src",
"declaredOnly": "true",
"lockEditablePath": "./editable-src",
"lockLocator": "requirements.txt",
"lockSource": "requirements.txt",
"pkg.confidence": "Medium",
"pkg.kind": "DeclaredOnly",
"pkg.location": "requirements.txt",
"scope": "prod"
},
"evidence": [
{
"kind": "metadata",
"source": "requirements.txt",
"locator": "requirements.txt"
}
]
},
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/certifi@2022.12.7",
"purl": "pkg:pypi/certifi@2022.12.7",
"name": "certifi",
"version": "2022.12.7",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"declaredOnly": "true",
"lockLocator": "requirements-base.txt",
"lockSource": "requirements-base.txt",
"pkg.confidence": "Medium",
"pkg.kind": "DeclaredOnly",
"pkg.location": "requirements-base.txt",
"scope": "prod"
},
"evidence": [
{
"kind": "metadata",
"source": "requirements-base.txt",
"locator": "requirements-base.txt"
}
]
},
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/requests@2.28.0",
"purl": "pkg:pypi/requests@2.28.0",
"name": "requests",
"version": "2.28.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"declaredOnly": "true",
"lockLocator": "requirements.txt",
"lockSource": "requirements.txt",
"pkg.confidence": "Medium",
"pkg.kind": "DeclaredOnly",
"pkg.location": "requirements.txt",
"scope": "prod"
},
"evidence": [
{
"kind": "metadata",
"source": "requirements.txt",
"locator": "requirements.txt"
}
]
},
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/urllib3@1.26.0",
"purl": "pkg:pypi/urllib3@1.26.0",
"name": "urllib3",
"version": "1.26.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"declaredOnly": "true",
"lockLocator": "requirements-base.txt",
"lockSource": "requirements-base.txt",
"pkg.confidence": "Medium",
"pkg.kind": "DeclaredOnly",
"pkg.location": "requirements-base.txt",
"scope": "prod"
},
"evidence": [
{
"kind": "metadata",
"source": "requirements-base.txt",
"locator": "requirements-base.txt"
}
]
}
]

2
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/requirements-includes-editable/requirements-base.txt Normal file
View File

@@ -0,0 +1,2 @@
urllib3==1.26.0
certifi==2022.12.7

3
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/requirements-includes-editable/requirements.txt Normal file
View File

@@ -0,0 +1,3 @@
requests==2.28.0
-r requirements-base.txt
-e ./editable-src

65
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/vendored-directory/expected.json Normal file
View File

@@ -0,0 +1,65 @@
[
{
"analyzerId": "python",
"componentKey": "explicit::python::pypi::urllib3::sha256:aa29f86a6e70276f0f1bc8b4fc71abb66af19af7dbcf4bfe40b40c3c4aa08467",
"purl": "pkg:pypi/urllib3@1.26.0",
"name": "urllib3",
"version": "1.26.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"embedded": "true",
"embedded.confidence": "High",
"embedded.parentPackage": "requests",
"embedded.parentVersion": "2.0.0",
"embedded.path": "requests/_vendor/urllib3",
"embedded.versionSource": "heuristic"
},
"evidence": [
{
"kind": "file",
"source": "vendored",
"locator": "lib/python3.11/site-packages/requests/_vendor/urllib3/__init__.py"
}
]
},
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/requests@2.0.0",
"purl": "pkg:pypi/requests@2.0.0",
"name": "requests",
"version": "2.0.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"distInfoPath": "lib/python3.11/site-packages/requests-2.0.0.dist-info",
"name": "requests",
"normalizedName": "requests",
"pkg.confidence": "High",
"pkg.kind": "Wheel",
"pkg.location": "lib/python3.11/site-packages/requests-2.0.0.dist-info",
"provenance": "dist-info",
"record.hashMismatches": "0",
"record.hashedEntries": "0",
"record.ioErrors": "0",
"record.missingFiles": "0",
"record.totalEntries": "0",
"runtime.libPaths.count": "1",
"runtime.versions": "3.11",
"summary": "Parent package with vendored deps",
"vendored.confidence": "High",
"vendored.detected": "true",
"vendored.packageCount": "1",
"vendored.packages": "urllib3@1.26.0",
"vendored.paths": "requests/_vendor",
"version": "2.0.0"
},
"evidence": [
{
"kind": "file",
"source": "METADATA",
"locator": "lib/python3.11/site-packages/requests-2.0.0.dist-info/METADATA"
}
]
}
]

5
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/vendored-directory/lib/python3.11/site-packages/requests-2.0.0.dist-info/METADATA Normal file
View File

@@ -0,0 +1,5 @@
Metadata-Version: 2.1
Name: requests
Version: 2.0.0
Summary: Parent package with vendored deps

1
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/vendored-directory/lib/python3.11/site-packages/requests/__init__.py Normal file
View File

@@ -0,0 +1 @@
__version__ = "2.0.0"

1
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/vendored-directory/lib/python3.11/site-packages/requests/_vendor/urllib3/__init__.py Normal file
View File

@@ -0,0 +1 @@
__version__ = "1.26.0"

51
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/wheel-workspace/expected.json Normal file
View File

@@ -0,0 +1,51 @@
[
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/wheelpkg@1.0.0",
"purl": "pkg:pypi/wheelpkg@1.0.0",
"name": "wheelpkg",
"version": "1.0.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"distInfoPath": "archives/wheel/wheelpkg-1.0.0-py3-none-any.whl/wheelpkg-1.0.0.dist-info",
"name": "wheelpkg",
"normalizedName": "wheelpkg",
"pkg.confidence": "Definitive",
"pkg.kind": "Wheel",
"pkg.location": "archives/wheel/wheelpkg-1.0.0-py3-none-any.whl",
"provenance": "dist-info",
"record.hashMismatches": "0",
"record.hashedEntries": "3",
"record.ioErrors": "0",
"record.missingFiles": "0",
"record.totalEntries": "4",
"summary": "Wheel fixture",
"version": "1.0.0",
"wheel.generator": "stellaops-test",
"wheel.rootIsPurelib": "true",
"wheel.tags": "py3-none-any",
"wheel.version": "1.0"
},
"evidence": [
{
"kind": "file",
"source": "METADATA",
"locator": "dist/wheelpkg-1.0.0-py3-none-any.whl",
"value": "wheelpkg-1.0.0.dist-info/METADATA"
},
{
"kind": "file",
"source": "RECORD",
"locator": "dist/wheelpkg-1.0.0-py3-none-any.whl",
"value": "wheelpkg-1.0.0.dist-info/RECORD"
},
{
"kind": "file",
"source": "WHEEL",
"locator": "dist/wheelpkg-1.0.0-py3-none-any.whl",
"value": "wheelpkg-1.0.0.dist-info/WHEEL"
}
]
}
]

41
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/zipapp-embedded-requirements/expected.json Normal file
View File

@@ -0,0 +1,41 @@
[
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/hostpkg@0.1.0",
"purl": "pkg:pypi/hostpkg@0.1.0",
"name": "hostpkg",
"version": "0.1.0",
"type": "pypi",
"usedByEntrypoint": false,
"metadata": {
"distInfoPath": "lib/python3.11/site-packages/hostpkg-0.1.0.dist-info",
"name": "hostpkg",
"normalizedName": "hostpkg",
"pkg.confidence": "High",
"pkg.kind": "Wheel",
"pkg.location": "lib/python3.11/site-packages/hostpkg-0.1.0.dist-info",
"provenance": "dist-info",
"record.hashMismatches": "0",
"record.hashedEntries": "0",
"record.ioErrors": "0",
"record.missingFiles": "0",
"record.totalEntries": "0",
"runtime.libPaths.count": "1",
"runtime.versions": "3.11",
"summary": "Host package for zipapp fixture",
"version": "0.1.0",
"zipapps.count": "1",
"zipapps.detected": "true",
"zipapps.embeddedDeps.count": "2",
"zipapps.embeddedDeps.sample": "flask;requests",
"zipapps.pythonVersions": "3.11"
},
"evidence": [
{
"kind": "file",
"source": "METADATA",
"locator": "lib/python3.11/site-packages/hostpkg-0.1.0.dist-info/METADATA"
}
]
}
]

5
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python/zipapp-embedded-requirements/lib/python3.11/site-packages/hostpkg-0.1.0.dist-info/METADATA Normal file
View File

@@ -0,0 +1,5 @@
Metadata-Version: 2.1
Name: hostpkg
Version: 0.1.0
Summary: Host package for zipapp fixture

213
src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Python/PythonLanguageAnalyzerTests.cs
View File

@@ -85,6 +85,148 @@ public sealed class PythonLanguageAnalyzerTests
usageHints);
}
[Fact]
public async Task CondaEnvFixtureProducesDeterministicOutputAsync()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = TestPaths.ResolveFixture("lang", "python", "conda-env");
var goldenPath = Path.Combine(fixturePath, "expected.json");
var analyzers = new ILanguageAnalyzer[]
{
new PythonLanguageAnalyzer()
};
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
fixturePath,
goldenPath,
analyzers,
cancellationToken);
}
[Fact]
public async Task RequirementsIncludesEditableFixtureProducesDeterministicOutputAsync()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = TestPaths.ResolveFixture("lang", "python", "requirements-includes-editable");
var goldenPath = Path.Combine(fixturePath, "expected.json");
var analyzers = new ILanguageAnalyzer[]
{
new PythonLanguageAnalyzer()
};
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
fixturePath,
goldenPath,
analyzers,
cancellationToken);
}
[Fact]
public async Task PipfileLockDefaultDevelopFixtureProducesDeterministicOutputAsync()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = TestPaths.ResolveFixture("lang", "python", "pipfile-lock-default-develop");
var goldenPath = Path.Combine(fixturePath, "expected.json");
var analyzers = new ILanguageAnalyzer[]
{
new PythonLanguageAnalyzer()
};
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
fixturePath,
goldenPath,
analyzers,
cancellationToken);
}
[Fact]
public async Task WheelWorkspaceFixtureProducesDeterministicOutputAsync()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = TestPaths.ResolveFixture("lang", "python", "wheel-workspace");
var goldenPath = Path.Combine(fixturePath, "expected.json");
var distDir = Path.Combine(fixturePath, "dist");
Directory.CreateDirectory(distDir);
var wheelPath = Path.Combine(distDir, "wheelpkg-1.0.0-py3-none-any.whl");
CreateWheelpkgWheel(wheelPath);
var analyzers = new ILanguageAnalyzer[]
{
new PythonLanguageAnalyzer()
};
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
fixturePath,
goldenPath,
analyzers,
cancellationToken);
}
[Fact]
public async Task ZipappEmbeddedRequirementsFixtureProducesDeterministicOutputAsync()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = TestPaths.ResolveFixture("lang", "python", "zipapp-embedded-requirements");
var goldenPath = Path.Combine(fixturePath, "expected.json");
var zipappPath = Path.Combine(fixturePath, "myapp.pyz");
CreateZipappWithEmbeddedRequirements(zipappPath);
var analyzers = new ILanguageAnalyzer[]
{
new PythonLanguageAnalyzer()
};
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
fixturePath,
goldenPath,
analyzers,
cancellationToken);
}
[Fact]
public async Task ContainerWhiteoutsFixtureProducesDeterministicOutputAsync()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = TestPaths.ResolveFixture("lang", "python", "container-whiteouts");
var goldenPath = Path.Combine(fixturePath, "expected.json");
var analyzers = new ILanguageAnalyzer[]
{
new PythonLanguageAnalyzer()
};
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
fixturePath,
goldenPath,
analyzers,
cancellationToken);
}
[Fact]
public async Task VendoredDirectoryFixtureProducesDeterministicOutputAsync()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = TestPaths.ResolveFixture("lang", "python", "vendored-directory");
var goldenPath = Path.Combine(fixturePath, "expected.json");
var analyzers = new ILanguageAnalyzer[]
{
new PythonLanguageAnalyzer()
};
await LanguageAnalyzerTestHarness.AssertDeterministicAsync(
fixturePath,
goldenPath,
analyzers,
cancellationToken);
}
[Fact]
public async Task LockfileCollectorEmitsDeclaredOnlyComponentsAsync()
{
@@ -580,6 +722,77 @@ public sealed class PythonLanguageAnalyzerTests
return path;
}
private static void CreateWheelpkgWheel(string wheelPath)
{
Directory.CreateDirectory(Path.GetDirectoryName(wheelPath)!);
var initBytes = Encoding.UTF8.GetBytes("__version__ = \"1.0.0\"\n");
var metadataBytes = Encoding.UTF8.GetBytes(
$"Metadata-Version: 2.1\nName: wheelpkg\nVersion: 1.0.0\nSummary: Wheel fixture\n{Environment.NewLine}");
var wheelBytes = Encoding.UTF8.GetBytes(
"Wheel-Version: 1.0\nGenerator: stellaops-test\nRoot-Is-Purelib: true\nTag: py3-none-any\n");
var recordContent = new StringBuilder()
.AppendLine($"wheelpkg/__init__.py,sha256={ComputeSha256Base64(initBytes)},{initBytes.Length}")
.AppendLine($"wheelpkg-1.0.0.dist-info/METADATA,sha256={ComputeSha256Base64(metadataBytes)},{metadataBytes.Length}")
.AppendLine($"wheelpkg-1.0.0.dist-info/WHEEL,sha256={ComputeSha256Base64(wheelBytes)},{wheelBytes.Length}")
.AppendLine("wheelpkg-1.0.0.dist-info/RECORD,,")
.ToString();
var recordBytes = Encoding.UTF8.GetBytes(recordContent);
if (File.Exists(wheelPath))
{
File.Delete(wheelPath);
}
using (var stream = File.Create(wheelPath))
using (var archive = new ZipArchive(stream, ZipArchiveMode.Create, leaveOpen: false))
{
WriteEntry(archive, "wheelpkg/__init__.py", initBytes);
WriteEntry(archive, "wheelpkg-1.0.0.dist-info/METADATA", metadataBytes);
WriteEntry(archive, "wheelpkg-1.0.0.dist-info/WHEEL", wheelBytes);
WriteEntry(archive, "wheelpkg-1.0.0.dist-info/RECORD", recordBytes);
}
static void WriteEntry(ZipArchive archive, string entryName, byte[] content)
{
var entry = archive.CreateEntry(entryName);
entry.LastWriteTime = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero);
using var entryStream = entry.Open();
entryStream.Write(content, 0, content.Length);
}
static string ComputeSha256Base64(byte[] content)
=> Convert.ToBase64String(SHA256.HashData(content));
}
private static void CreateZipappWithEmbeddedRequirements(string zipappPath)
{
if (File.Exists(zipappPath))
{
File.Delete(zipappPath);
}
using var fileStream = File.Create(zipappPath);
var shebangBytes = Encoding.UTF8.GetBytes("#!/usr/bin/python3.11\n");
fileStream.Write(shebangBytes);
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, leaveOpen: true);
WriteTextEntry(archive, "__main__.py", "print('hello')\n");
WriteTextEntry(archive, "requirements.txt", "requests==2.28.0\nflask==2.1.0\n");
static void WriteTextEntry(ZipArchive archive, string name, string content)
{
var entry = archive.CreateEntry(name, CompressionLevel.NoCompression);
entry.LastWriteTime = new DateTimeOffset(2020, 1, 1, 0, 0, 0, TimeSpan.Zero);
using var stream = entry.Open();
using var writer = new StreamWriter(stream, new UTF8Encoding(encoderShouldEmitUTF8Identifier: false));
writer.Write(content);
}
}
// ===== SCAN-PY-405-007 Fixtures =====
[Fact]