up
Some checks failed
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled

This commit is contained in:
StellaOps Bot
2025-11-28 09:40:40 +02:00
parent 1c6730a1d2
commit 05da719048
206 changed files with 34741 additions and 1751 deletions

View File

@@ -0,0 +1,423 @@
using System.Collections.Frozen;
using System.Collections.Immutable;
using System.Text.RegularExpressions;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.Framework;
/// <summary>
/// Detects framework usage hints from Python source code.
/// </summary>
internal sealed partial class PythonFrameworkDetector
{
// File patterns that strongly indicate frameworks
private static readonly FrozenDictionary<string, (PythonFrameworkKind Kind, PythonFrameworkConfidence Confidence)> FilePatterns =
new Dictionary<string, (PythonFrameworkKind, PythonFrameworkConfidence)>(StringComparer.OrdinalIgnoreCase)
{
// Django
["manage.py"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.High),
["settings.py"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.Medium),
["urls.py"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.Medium),
["wsgi.py"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.Medium),
["asgi.py"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.Medium),
// Celery
["celery.py"] = (PythonFrameworkKind.Celery, PythonFrameworkConfidence.High),
["tasks.py"] = (PythonFrameworkKind.Celery, PythonFrameworkConfidence.Low),
// Gunicorn
["gunicorn.conf.py"] = (PythonFrameworkKind.Gunicorn, PythonFrameworkConfidence.Definitive),
["gunicorn_config.py"] = (PythonFrameworkKind.Gunicorn, PythonFrameworkConfidence.Definitive),
// uWSGI
["uwsgi.ini"] = (PythonFrameworkKind.Uwsgi, PythonFrameworkConfidence.Definitive),
// Pytest
["conftest.py"] = (PythonFrameworkKind.Pytest, PythonFrameworkConfidence.High),
["pytest.ini"] = (PythonFrameworkKind.Pytest, PythonFrameworkConfidence.Definitive),
// Jupyter
["*.ipynb"] = (PythonFrameworkKind.Jupyter, PythonFrameworkConfidence.Definitive),
}.ToFrozenDictionary();
// Import patterns that indicate frameworks
private static readonly FrozenDictionary<string, (PythonFrameworkKind Kind, PythonFrameworkConfidence Confidence)> ImportPatterns =
new Dictionary<string, (PythonFrameworkKind, PythonFrameworkConfidence)>(StringComparer.Ordinal)
{
// Django
["django"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.High),
["django.conf"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.High),
["django.urls"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.High),
["django.views"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.High),
["django.db"] = (PythonFrameworkKind.Django, PythonFrameworkConfidence.High),
// Flask
["flask"] = (PythonFrameworkKind.Flask, PythonFrameworkConfidence.High),
["flask_restful"] = (PythonFrameworkKind.Flask, PythonFrameworkConfidence.High),
["flask_sqlalchemy"] = (PythonFrameworkKind.Flask, PythonFrameworkConfidence.High),
// FastAPI
["fastapi"] = (PythonFrameworkKind.FastAPI, PythonFrameworkConfidence.High),
["starlette"] = (PythonFrameworkKind.Starlette, PythonFrameworkConfidence.High),
// Celery
["celery"] = (PythonFrameworkKind.Celery, PythonFrameworkConfidence.High),
// RQ
["rq"] = (PythonFrameworkKind.RQ, PythonFrameworkConfidence.High),
// Click
["click"] = (PythonFrameworkKind.Click, PythonFrameworkConfidence.High),
// Typer
["typer"] = (PythonFrameworkKind.Typer, PythonFrameworkConfidence.High),
// Pytest
["pytest"] = (PythonFrameworkKind.Pytest, PythonFrameworkConfidence.High),
// Streamlit
["streamlit"] = (PythonFrameworkKind.Streamlit, PythonFrameworkConfidence.Definitive),
// Gradio
["gradio"] = (PythonFrameworkKind.Gradio, PythonFrameworkConfidence.Definitive),
// Pydantic Settings
["pydantic_settings"] = (PythonFrameworkKind.PydanticSettings, PythonFrameworkConfidence.Definitive),
}.ToFrozenDictionary();
// Django patterns
[GeneratedRegex(@"INSTALLED_APPS\s*=\s*\[", RegexOptions.Compiled)]
private static partial Regex DjangoInstalledAppsPattern();
[GeneratedRegex(@"MIDDLEWARE\s*=\s*\[", RegexOptions.Compiled)]
private static partial Regex DjangoMiddlewarePattern();
[GeneratedRegex(@"ROOT_URLCONF\s*=", RegexOptions.Compiled)]
private static partial Regex DjangoRootUrlConfPattern();
[GeneratedRegex(@"os\.environ\.setdefault\s*\(\s*[""']DJANGO_SETTINGS_MODULE[""']", RegexOptions.Compiled)]
private static partial Regex DjangoSettingsModulePattern();
// Flask patterns
[GeneratedRegex(@"Flask\s*\(\s*__name__", RegexOptions.Compiled)]
private static partial Regex FlaskAppPattern();
[GeneratedRegex(@"Blueprint\s*\(", RegexOptions.Compiled)]
private static partial Regex FlaskBlueprintPattern();
// FastAPI patterns
[GeneratedRegex(@"FastAPI\s*\(", RegexOptions.Compiled)]
private static partial Regex FastAPIAppPattern();
[GeneratedRegex(@"APIRouter\s*\(", RegexOptions.Compiled)]
private static partial Regex FastAPIRouterPattern();
// Celery patterns
[GeneratedRegex(@"Celery\s*\(", RegexOptions.Compiled)]
private static partial Regex CeleryAppPattern();
[GeneratedRegex(@"@\s*(?:app\.task|celery\.task|shared_task)", RegexOptions.Compiled)]
private static partial Regex CeleryTaskPattern();
// AWS Lambda patterns
[GeneratedRegex(@"def\s+(lambda_handler|handler)\s*\(\s*event\s*,\s*context\s*\)", RegexOptions.Compiled)]
private static partial Regex LambdaHandlerPattern();
[GeneratedRegex(@"def\s+\w+\s*\(\s*event\s*:\s*dict\s*,\s*context\s*:\s*LambdaContext", RegexOptions.Compiled)]
private static partial Regex LambdaTypedHandlerPattern();
// Click patterns
[GeneratedRegex(@"@\s*click\.command", RegexOptions.Compiled)]
private static partial Regex ClickCommandPattern();
[GeneratedRegex(@"@\s*click\.group", RegexOptions.Compiled)]
private static partial Regex ClickGroupPattern();
// Typer patterns
[GeneratedRegex(@"typer\.Typer\s*\(", RegexOptions.Compiled)]
private static partial Regex TyperAppPattern();
[GeneratedRegex(@"@\s*app\.command", RegexOptions.Compiled)]
private static partial Regex TyperCommandPattern();
// Logging patterns
[GeneratedRegex(@"logging\.config\.dictConfig", RegexOptions.Compiled)]
private static partial Regex LoggingDictConfigPattern();
[GeneratedRegex(@"logging\.config\.fileConfig", RegexOptions.Compiled)]
private static partial Regex LoggingFileConfigPattern();
[GeneratedRegex(@"LOGGING\s*=\s*\{", RegexOptions.Compiled)]
private static partial Regex DjangoLoggingPattern();
// Gunicorn patterns
[GeneratedRegex(@"bind\s*=\s*[""']", RegexOptions.Compiled)]
private static partial Regex GunicornBindPattern();
[GeneratedRegex(@"workers\s*=", RegexOptions.Compiled)]
private static partial Regex GunicornWorkersPattern();
/// <summary>
/// Detects framework hints from Python source code.
/// </summary>
public async Task<ImmutableArray<PythonFrameworkHint>> DetectAsync(
PythonVirtualFileSystem vfs,
CancellationToken cancellationToken = default)
{
var hints = new List<PythonFrameworkHint>();
// First pass: check file patterns
foreach (var file in vfs.Files)
{
cancellationToken.ThrowIfCancellationRequested();
var fileName = Path.GetFileName(file.VirtualPath);
if (FilePatterns.TryGetValue(fileName, out var fileHint))
{
hints.Add(new PythonFrameworkHint(
Kind: fileHint.Kind,
SourceFile: file.VirtualPath,
LineNumber: null,
Evidence: $"file pattern: {fileName}",
Confidence: fileHint.Confidence));
}
// Special case for Jupyter notebooks
if (file.VirtualPath.EndsWith(".ipynb", StringComparison.OrdinalIgnoreCase))
{
hints.Add(new PythonFrameworkHint(
Kind: PythonFrameworkKind.Jupyter,
SourceFile: file.VirtualPath,
LineNumber: null,
Evidence: "Jupyter notebook file",
Confidence: PythonFrameworkConfidence.Definitive));
}
}
// Second pass: scan Python files for patterns
var pythonFiles = vfs.Files
.Where(f => f.VirtualPath.EndsWith(".py", StringComparison.OrdinalIgnoreCase))
.ToList();
foreach (var file in pythonFiles)
{
cancellationToken.ThrowIfCancellationRequested();
var fileHints = await DetectInFileAsync(vfs, file, cancellationToken).ConfigureAwait(false);
hints.AddRange(fileHints);
}
// Deduplicate and prioritize by confidence
return hints
.GroupBy(h => (h.Kind, h.SourceFile))
.Select(g => g.OrderByDescending(h => h.Confidence).First())
.ToImmutableArray();
}
private async Task<IEnumerable<PythonFrameworkHint>> DetectInFileAsync(
PythonVirtualFileSystem vfs,
PythonVirtualFile file,
CancellationToken cancellationToken)
{
var hints = new List<PythonFrameworkHint>();
try
{
using var stream = await vfs.OpenReadAsync(file.VirtualPath, cancellationToken).ConfigureAwait(false);
if (stream is null)
{
return hints;
}
using var reader = new StreamReader(stream);
var content = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false);
var lines = content.Split('\n');
for (var lineNum = 0; lineNum < lines.Length; lineNum++)
{
cancellationToken.ThrowIfCancellationRequested();
var line = lines[lineNum];
var trimmed = line.TrimStart();
// Skip comments
if (trimmed.StartsWith('#'))
{
continue;
}
// Check for imports
if (trimmed.StartsWith("import ", StringComparison.Ordinal) ||
trimmed.StartsWith("from ", StringComparison.Ordinal))
{
var importHints = DetectImportPatterns(trimmed, file.VirtualPath, lineNum + 1);
hints.AddRange(importHints);
}
// Django patterns
if (DjangoInstalledAppsPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Django, file.VirtualPath, lineNum + 1,
"INSTALLED_APPS configuration", PythonFrameworkConfidence.Definitive));
}
if (DjangoSettingsModulePattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Django, file.VirtualPath, lineNum + 1,
"DJANGO_SETTINGS_MODULE", PythonFrameworkConfidence.Definitive));
}
// Flask patterns
if (FlaskAppPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Flask, file.VirtualPath, lineNum + 1,
"Flask(__name__)", PythonFrameworkConfidence.Definitive));
}
if (FlaskBlueprintPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Flask, file.VirtualPath, lineNum + 1,
"Blueprint()", PythonFrameworkConfidence.High));
}
// FastAPI patterns
if (FastAPIAppPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.FastAPI, file.VirtualPath, lineNum + 1,
"FastAPI()", PythonFrameworkConfidence.Definitive));
}
if (FastAPIRouterPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.FastAPI, file.VirtualPath, lineNum + 1,
"APIRouter()", PythonFrameworkConfidence.High));
}
// Celery patterns
if (CeleryAppPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Celery, file.VirtualPath, lineNum + 1,
"Celery()", PythonFrameworkConfidence.Definitive));
}
if (CeleryTaskPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Celery, file.VirtualPath, lineNum + 1,
"@app.task decorator", PythonFrameworkConfidence.High));
}
// AWS Lambda patterns
if (LambdaHandlerPattern().IsMatch(line) || LambdaTypedHandlerPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.AwsLambda, file.VirtualPath, lineNum + 1,
"Lambda handler function", PythonFrameworkConfidence.High));
}
// Click patterns
if (ClickCommandPattern().IsMatch(line) || ClickGroupPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Click, file.VirtualPath, lineNum + 1,
"@click.command/group", PythonFrameworkConfidence.High));
}
// Typer patterns
if (TyperAppPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Typer, file.VirtualPath, lineNum + 1,
"typer.Typer()", PythonFrameworkConfidence.Definitive));
}
if (TyperCommandPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Typer, file.VirtualPath, lineNum + 1,
"@app.command", PythonFrameworkConfidence.High));
}
// Logging patterns
if (LoggingDictConfigPattern().IsMatch(line) || LoggingFileConfigPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.LoggingConfig, file.VirtualPath, lineNum + 1,
"logging.config", PythonFrameworkConfidence.High));
}
if (DjangoLoggingPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.LoggingConfig, file.VirtualPath, lineNum + 1,
"Django LOGGING dict", PythonFrameworkConfidence.High));
}
// Gunicorn patterns (in config files)
if (file.VirtualPath.Contains("gunicorn", StringComparison.OrdinalIgnoreCase))
{
if (GunicornBindPattern().IsMatch(line) || GunicornWorkersPattern().IsMatch(line))
{
hints.Add(CreateHint(PythonFrameworkKind.Gunicorn, file.VirtualPath, lineNum + 1,
"Gunicorn configuration", PythonFrameworkConfidence.Definitive));
}
}
}
}
catch (IOException)
{
// Skip unreadable files
}
return hints;
}
private static IEnumerable<PythonFrameworkHint> DetectImportPatterns(string line, string sourceFile, int lineNumber)
{
var trimmed = line.Trim();
string? moduleName = null;
if (trimmed.StartsWith("import ", StringComparison.Ordinal))
{
var parts = trimmed[7..].Split(',');
foreach (var part in parts)
{
moduleName = part.Trim().Split(new[] { " as ", " " }, StringSplitOptions.RemoveEmptyEntries)[0];
if (ImportPatterns.TryGetValue(moduleName, out var hint))
{
yield return CreateHint(hint.Kind, sourceFile, lineNumber,
$"import {moduleName}", hint.Confidence);
}
}
}
else if (trimmed.StartsWith("from ", StringComparison.Ordinal))
{
var parts = trimmed[5..].Split(new[] { " import " }, StringSplitOptions.RemoveEmptyEntries);
if (parts.Length > 0)
{
moduleName = parts[0].Trim();
// Check base module
var baseModule = moduleName.Split('.')[0];
if (ImportPatterns.TryGetValue(baseModule, out var hint))
{
yield return CreateHint(hint.Kind, sourceFile, lineNumber,
$"from {moduleName}", hint.Confidence);
}
// Check full module path
if (ImportPatterns.TryGetValue(moduleName, out hint))
{
yield return CreateHint(hint.Kind, sourceFile, lineNumber,
$"from {moduleName}", hint.Confidence);
}
}
}
}
private static PythonFrameworkHint CreateHint(
PythonFrameworkKind kind,
string sourceFile,
int lineNumber,
string evidence,
PythonFrameworkConfidence confidence)
{
return new PythonFrameworkHint(
Kind: kind,
SourceFile: sourceFile,
LineNumber: lineNumber,
Evidence: evidence,
Confidence: confidence);
}
}

View File

@@ -0,0 +1,151 @@
using System.Collections.Immutable;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.Framework;
/// <summary>
/// Represents a detected framework or configuration hint in a Python project.
/// These are hints/suggestions, not definitive detections.
/// </summary>
/// <param name="Kind">The type of framework or configuration.</param>
/// <param name="SourceFile">The file where this hint was detected.</param>
/// <param name="LineNumber">The line number (if available).</param>
/// <param name="Evidence">The code pattern that indicated this hint.</param>
/// <param name="Confidence">Confidence level for this detection.</param>
/// <param name="Metadata">Additional metadata about the detection.</param>
internal sealed record PythonFrameworkHint(
PythonFrameworkKind Kind,
string SourceFile,
int? LineNumber,
string Evidence,
PythonFrameworkConfidence Confidence,
ImmutableDictionary<string, string>? Metadata = null)
{
/// <summary>
/// Gets whether this is a web framework.
/// </summary>
public bool IsWebFramework => Kind is
PythonFrameworkKind.Django or
PythonFrameworkKind.Flask or
PythonFrameworkKind.FastAPI or
PythonFrameworkKind.Starlette or
PythonFrameworkKind.Tornado or
PythonFrameworkKind.Bottle or
PythonFrameworkKind.Pyramid;
/// <summary>
/// Gets whether this is a task queue.
/// </summary>
public bool IsTaskQueue => Kind is
PythonFrameworkKind.Celery or
PythonFrameworkKind.RQ or
PythonFrameworkKind.Huey or
PythonFrameworkKind.Dramatiq;
/// <summary>
/// Gets whether this is a serverless runtime.
/// </summary>
public bool IsServerless => Kind is
PythonFrameworkKind.AwsLambda or
PythonFrameworkKind.AzureFunctions or
PythonFrameworkKind.GoogleCloudFunctions;
/// <summary>
/// Gets whether this is a CLI framework.
/// </summary>
public bool IsCliFramework => Kind is
PythonFrameworkKind.Click or
PythonFrameworkKind.Typer or
PythonFrameworkKind.Argparse;
/// <summary>
/// Generates metadata entries for this hint.
/// </summary>
public IEnumerable<KeyValuePair<string, string?>> ToMetadata(string prefix)
{
yield return new($"{prefix}.kind", Kind.ToString());
yield return new($"{prefix}.file", SourceFile);
if (LineNumber.HasValue)
{
yield return new($"{prefix}.line", LineNumber.Value.ToString());
}
yield return new($"{prefix}.evidence", Evidence);
yield return new($"{prefix}.confidence", Confidence.ToString());
if (IsWebFramework)
{
yield return new($"{prefix}.category", "WebFramework");
}
else if (IsTaskQueue)
{
yield return new($"{prefix}.category", "TaskQueue");
}
else if (IsServerless)
{
yield return new($"{prefix}.category", "Serverless");
}
else if (IsCliFramework)
{
yield return new($"{prefix}.category", "CLI");
}
if (Metadata is not null)
{
foreach (var (key, value) in Metadata)
{
yield return new($"{prefix}.{key}", value);
}
}
}
}
/// <summary>
/// Represents an AWS Lambda handler configuration.
/// </summary>
/// <param name="HandlerPath">The handler path (module.function).</param>
/// <param name="ModulePath">The module file path.</param>
/// <param name="FunctionName">The handler function name.</param>
/// <param name="Runtime">The detected Python runtime (if available).</param>
internal sealed record PythonLambdaHandler(
string HandlerPath,
string ModulePath,
string FunctionName,
string? Runtime = null);
/// <summary>
/// Represents a Django project configuration.
/// </summary>
/// <param name="SettingsModule">The settings module path.</param>
/// <param name="InstalledApps">List of installed apps.</param>
/// <param name="Middlewares">List of middleware classes.</param>
/// <param name="RootUrlConf">The root URL configuration module.</param>
internal sealed record PythonDjangoConfig(
string SettingsModule,
ImmutableArray<string> InstalledApps,
ImmutableArray<string> Middlewares,
string? RootUrlConf = null);
/// <summary>
/// Represents a Flask application configuration.
/// </summary>
/// <param name="AppVariable">The Flask app variable name.</param>
/// <param name="ModulePath">The module containing the app.</param>
/// <param name="Blueprints">Registered blueprints.</param>
internal sealed record PythonFlaskConfig(
string AppVariable,
string ModulePath,
ImmutableArray<string> Blueprints);
/// <summary>
/// Represents a Celery configuration.
/// </summary>
/// <param name="AppVariable">The Celery app variable name.</param>
/// <param name="ModulePath">The module containing the app.</param>
/// <param name="BrokerUrl">The broker URL pattern (if detected).</param>
/// <param name="Tasks">Discovered task modules.</param>
internal sealed record PythonCeleryConfig(
string AppVariable,
string ModulePath,
string? BrokerUrl,
ImmutableArray<string> Tasks);

View File

@@ -0,0 +1,186 @@
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.Framework;
/// <summary>
/// Types of Python frameworks and configurations detected.
/// </summary>
internal enum PythonFrameworkKind
{
/// <summary>
/// Unknown framework.
/// </summary>
Unknown,
// Web Frameworks
/// <summary>
/// Django web framework.
/// </summary>
Django,
/// <summary>
/// Flask web framework.
/// </summary>
Flask,
/// <summary>
/// FastAPI async web framework.
/// </summary>
FastAPI,
/// <summary>
/// Starlette ASGI framework.
/// </summary>
Starlette,
/// <summary>
/// Tornado async web framework.
/// </summary>
Tornado,
/// <summary>
/// Bottle micro framework.
/// </summary>
Bottle,
/// <summary>
/// Pyramid web framework.
/// </summary>
Pyramid,
// Task Queues
/// <summary>
/// Celery distributed task queue.
/// </summary>
Celery,
/// <summary>
/// RQ (Redis Queue) task queue.
/// </summary>
RQ,
/// <summary>
/// Huey task queue.
/// </summary>
Huey,
/// <summary>
/// Dramatiq task queue.
/// </summary>
Dramatiq,
// Serverless
/// <summary>
/// AWS Lambda handler.
/// </summary>
AwsLambda,
/// <summary>
/// Azure Functions.
/// </summary>
AzureFunctions,
/// <summary>
/// Google Cloud Functions.
/// </summary>
GoogleCloudFunctions,
// Application Servers
/// <summary>
/// Gunicorn WSGI server.
/// </summary>
Gunicorn,
/// <summary>
/// uWSGI server.
/// </summary>
Uwsgi,
/// <summary>
/// Uvicorn ASGI server.
/// </summary>
Uvicorn,
/// <summary>
/// Hypercorn ASGI server.
/// </summary>
Hypercorn,
// CLI Frameworks
/// <summary>
/// Click CLI framework.
/// </summary>
Click,
/// <summary>
/// Typer CLI framework (Click-based).
/// </summary>
Typer,
/// <summary>
/// Argparse standard library CLI.
/// </summary>
Argparse,
// Testing Frameworks
/// <summary>
/// Pytest testing framework.
/// </summary>
Pytest,
/// <summary>
/// Unittest standard library.
/// </summary>
Unittest,
// Data/ML Frameworks
/// <summary>
/// Jupyter notebook.
/// </summary>
Jupyter,
/// <summary>
/// Streamlit data app.
/// </summary>
Streamlit,
/// <summary>
/// Gradio ML demo.
/// </summary>
Gradio,
// Configuration
/// <summary>
/// Python logging configuration.
/// </summary>
LoggingConfig,
/// <summary>
/// Pydantic settings configuration.
/// </summary>
PydanticSettings
}
/// <summary>
/// Confidence level for framework detection.
/// </summary>
internal enum PythonFrameworkConfidence
{
/// <summary>
/// Low confidence - heuristic match based on file patterns.
/// </summary>
Low = 0,
/// <summary>
/// Medium confidence - import detected but usage unclear.
/// </summary>
Medium = 1,
/// <summary>
/// High confidence - clear usage pattern detected.
/// </summary>
High = 2,
/// <summary>
/// Definitive - explicit configuration or initialization found.
/// </summary>
Definitive = 3
}

View File

@@ -0,0 +1,327 @@
using System.Collections.Immutable;
using System.Text.RegularExpressions;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.Framework;
/// <summary>
/// Parses Python project configuration files (pyproject.toml, setup.cfg, setup.py).
/// </summary>
internal sealed partial class PythonProjectConfigParser
{
// pyproject.toml patterns
[GeneratedRegex(@"^\[project\]", RegexOptions.Compiled | RegexOptions.Multiline)]
private static partial Regex PyprojectProjectSection();
[GeneratedRegex(@"^\[project\.optional-dependencies\]", RegexOptions.Compiled | RegexOptions.Multiline)]
private static partial Regex PyprojectOptionalDepsSection();
[GeneratedRegex(@"^\[tool\.poetry\.extras\]", RegexOptions.Compiled | RegexOptions.Multiline)]
private static partial Regex PoetryExtrasSection();
[GeneratedRegex(@"^\[tool\.poetry\.group\.(\w+)\.dependencies\]", RegexOptions.Compiled | RegexOptions.Multiline)]
private static partial Regex PoetryGroupSection();
// Pattern to extract key = value or key = [...] lines
[GeneratedRegex(@"^(\w+)\s*=\s*\[(.*?)\]", RegexOptions.Compiled | RegexOptions.Singleline)]
private static partial Regex ArrayValuePattern();
[GeneratedRegex(@"^name\s*=\s*[""']([^""']+)[""']", RegexOptions.Compiled | RegexOptions.Multiline)]
private static partial Regex ProjectNamePattern();
[GeneratedRegex(@"^version\s*=\s*[""']([^""']+)[""']", RegexOptions.Compiled | RegexOptions.Multiline)]
private static partial Regex ProjectVersionPattern();
/// <summary>
/// Parses pyproject.toml and extracts optional dependencies.
/// </summary>
public async Task<PythonProjectConfig?> ParsePyprojectAsync(
PythonVirtualFileSystem vfs,
string pyprojectPath,
CancellationToken cancellationToken = default)
{
using var stream = await vfs.OpenReadAsync(pyprojectPath, cancellationToken).ConfigureAwait(false);
if (stream is null)
{
return null;
}
using var reader = new StreamReader(stream);
var content = await reader.ReadToEndAsync(cancellationToken).ConfigureAwait(false);
return ParsePyprojectContent(content, pyprojectPath);
}
private static PythonProjectConfig ParsePyprojectContent(string content, string filePath)
{
string? projectName = null;
string? projectVersion = null;
var optionalDependencies = new Dictionary<string, ImmutableArray<string>>();
var scripts = new Dictionary<string, string>();
var extras = new List<string>();
// Extract project name and version
var nameMatch = ProjectNamePattern().Match(content);
if (nameMatch.Success)
{
projectName = nameMatch.Groups[1].Value;
}
var versionMatch = ProjectVersionPattern().Match(content);
if (versionMatch.Success)
{
projectVersion = versionMatch.Groups[1].Value;
}
// Parse optional dependencies section
var optDepsMatch = PyprojectOptionalDepsSection().Match(content);
if (optDepsMatch.Success)
{
var sectionStart = optDepsMatch.Index + optDepsMatch.Length;
var sectionContent = ExtractSectionContent(content, sectionStart);
optionalDependencies = ParseOptionalDependencies(sectionContent);
extras.AddRange(optionalDependencies.Keys);
}
// Parse Poetry extras section
var poetryExtrasMatch = PoetryExtrasSection().Match(content);
if (poetryExtrasMatch.Success)
{
var sectionStart = poetryExtrasMatch.Index + poetryExtrasMatch.Length;
var sectionContent = ExtractSectionContent(content, sectionStart);
var poetryExtras = ParseOptionalDependencies(sectionContent);
foreach (var (key, value) in poetryExtras)
{
if (!optionalDependencies.ContainsKey(key))
{
optionalDependencies[key] = value;
extras.Add(key);
}
}
}
// Parse Poetry group dependencies
foreach (Match groupMatch in PoetryGroupSection().Matches(content))
{
var groupName = groupMatch.Groups[1].Value;
if (!extras.Contains(groupName))
{
extras.Add(groupName);
}
}
// Parse scripts section
scripts = ParseScriptsSection(content);
return new PythonProjectConfig(
FilePath: filePath,
ProjectName: projectName,
ProjectVersion: projectVersion,
OptionalDependencies: optionalDependencies.ToImmutableDictionary(),
Extras: extras.Distinct().ToImmutableArray(),
Scripts: scripts.ToImmutableDictionary());
}
private static string ExtractSectionContent(string content, int startIndex)
{
// Find the next section header or end of file
var nextSection = content.IndexOf("\n[", startIndex, StringComparison.Ordinal);
if (nextSection < 0)
{
return content[startIndex..];
}
return content[startIndex..nextSection];
}
private static Dictionary<string, ImmutableArray<string>> ParseOptionalDependencies(string sectionContent)
{
var result = new Dictionary<string, ImmutableArray<string>>();
var lines = sectionContent.Split('\n');
string? currentKey = null;
var currentValues = new List<string>();
var inArray = false;
foreach (var line in lines)
{
var trimmed = line.Trim();
if (string.IsNullOrEmpty(trimmed) || trimmed.StartsWith('#'))
{
continue;
}
// Check for new key
if (!inArray && trimmed.Contains('='))
{
// Save previous key
if (currentKey is not null)
{
result[currentKey] = currentValues.ToImmutableArray();
currentValues = [];
}
var parts = trimmed.Split('=', 2);
currentKey = parts[0].Trim();
var value = parts.Length > 1 ? parts[1].Trim() : "";
if (value.StartsWith('['))
{
if (value.EndsWith(']'))
{
// Single-line array
currentValues = ParseArrayValues(value);
}
else
{
// Multi-line array
inArray = true;
currentValues = ParseArrayValues(value);
}
}
}
else if (inArray)
{
if (trimmed.EndsWith(']'))
{
currentValues.AddRange(ParseArrayValues(trimmed));
inArray = false;
}
else
{
currentValues.AddRange(ParseArrayValues(trimmed));
}
}
}
// Save last key
if (currentKey is not null)
{
result[currentKey] = currentValues.ToImmutableArray();
}
return result;
}
private static List<string> ParseArrayValues(string value)
{
var result = new List<string>();
var cleaned = value.Trim('[', ']', ' ', '\t');
if (string.IsNullOrEmpty(cleaned))
{
return result;
}
// Split by comma, handling quoted strings
var parts = cleaned.Split(',');
foreach (var part in parts)
{
var trimmed = part.Trim().Trim('"', '\'', ' ');
if (!string.IsNullOrEmpty(trimmed))
{
result.Add(trimmed);
}
}
return result;
}
private static Dictionary<string, string> ParseScriptsSection(string content)
{
var result = new Dictionary<string, string>();
// Look for [project.scripts] or [tool.poetry.scripts]
var scriptsPatterns = new[]
{
@"\[project\.scripts\]",
@"\[tool\.poetry\.scripts\]"
};
foreach (var pattern in scriptsPatterns)
{
var match = Regex.Match(content, pattern);
if (match.Success)
{
var sectionStart = match.Index + match.Length;
var sectionContent = ExtractSectionContent(content, sectionStart);
var scripts = ParseKeyValueSection(sectionContent);
foreach (var (key, value) in scripts)
{
result[key] = value;
}
}
}
return result;
}
private static Dictionary<string, string> ParseKeyValueSection(string sectionContent)
{
var result = new Dictionary<string, string>();
var lines = sectionContent.Split('\n');
foreach (var line in lines)
{
var trimmed = line.Trim();
if (string.IsNullOrEmpty(trimmed) || trimmed.StartsWith('#'))
{
continue;
}
if (trimmed.Contains('='))
{
var parts = trimmed.Split('=', 2);
var key = parts[0].Trim();
var value = parts.Length > 1 ? parts[1].Trim().Trim('"', '\'') : "";
result[key] = value;
}
}
return result;
}
}
/// <summary>
/// Represents parsed Python project configuration.
/// </summary>
/// <param name="FilePath">Path to the configuration file.</param>
/// <param name="ProjectName">The project name.</param>
/// <param name="ProjectVersion">The project version.</param>
/// <param name="OptionalDependencies">Optional dependencies by group name.</param>
/// <param name="Extras">List of available extras.</param>
/// <param name="Scripts">Entry point scripts.</param>
internal sealed record PythonProjectConfig(
string FilePath,
string? ProjectName,
string? ProjectVersion,
ImmutableDictionary<string, ImmutableArray<string>> OptionalDependencies,
ImmutableArray<string> Extras,
ImmutableDictionary<string, string> Scripts)
{
/// <summary>
/// Generates metadata entries for this configuration.
/// </summary>
public IEnumerable<KeyValuePair<string, string?>> ToMetadata(string prefix)
{
yield return new($"{prefix}.path", FilePath);
if (ProjectName is not null)
{
yield return new($"{prefix}.name", ProjectName);
}
if (ProjectVersion is not null)
{
yield return new($"{prefix}.version", ProjectVersion);
}
if (Extras.Length > 0)
{
yield return new($"{prefix}.extras", string.Join(",", Extras));
}
if (Scripts.Count > 0)
{
yield return new($"{prefix}.scripts", string.Join(",", Scripts.Keys));
}
}
}

View File

@@ -0,0 +1,395 @@
using System.Collections.Immutable;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Capabilities;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Entrypoints;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Framework;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Imports;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Packaging;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.Observations;
/// <summary>
/// Builds AOC-compliant observation documents from analysis results.
/// </summary>
internal sealed class PythonObservationBuilder
{
private const string SchemaVersion = "python-aoc-v1";
private readonly List<PythonObservationPackage> _packages = [];
private readonly List<PythonObservationModule> _modules = [];
private readonly List<PythonObservationEntrypoint> _entrypoints = [];
private readonly List<PythonObservationDependencyEdge> _dependencyEdges = [];
private readonly List<PythonObservationImportEdge> _importEdges = [];
private readonly List<PythonObservationNativeExtension> _nativeExtensions = [];
private readonly List<PythonObservationFrameworkHint> _frameworks = [];
private readonly List<PythonObservationWarning> _warnings = [];
private readonly List<string> _securitySensitiveCapabilities = [];
private PythonObservationEnvironment? _environment;
private PythonObservationRuntimeEvidence? _runtimeEvidence;
private bool _usesProcessExecution;
private bool _usesNetworkAccess;
private bool _usesFileSystem;
private bool _usesCodeExecution;
private bool _usesDeserialization;
private bool _usesNativeCode;
private bool _usesAsyncAwait;
private bool _usesMultiprocessing;
/// <summary>
/// Adds packages from package discovery results.
/// </summary>
public PythonObservationBuilder AddPackages(IEnumerable<PythonPackageInfo> packages)
{
foreach (var pkg in packages)
{
_packages.Add(new PythonObservationPackage(
Name: pkg.Name,
Version: pkg.Version ?? "unknown",
Source: pkg.Kind.ToString(),
Platform: null,
IsDirect: pkg.IsDirectDependency,
InstallerKind: pkg.InstallerTool,
DistInfoPath: pkg.MetadataPath,
Groups: pkg.Extras,
Extras: pkg.Extras));
// Add dependency edges
foreach (var dep in pkg.Dependencies)
{
_dependencyEdges.Add(new PythonObservationDependencyEdge(
FromPackage: pkg.Name,
ToPackage: ExtractPackageName(dep),
VersionConstraint: ExtractVersionConstraint(dep),
Extra: null,
IsOptional: false));
}
}
return this;
}
/// <summary>
/// Adds modules from import graph analysis.
/// </summary>
public PythonObservationBuilder AddModules(
IEnumerable<PythonModuleNode> moduleNodes,
PythonImportGraph? importGraph = null)
{
foreach (var node in moduleNodes)
{
var imports = importGraph?.GetImportsForFile(node.VirtualPath ?? "")
.SelectMany(i => i.ImportedNames)
.ToImmutableArray() ?? ImmutableArray<string>.Empty;
_modules.Add(new PythonObservationModule(
Name: node.ModulePath,
Type: node.IsPackage ? "package" : "module",
FilePath: node.VirtualPath ?? "",
Line: null,
IsNamespacePackage: node.IsNamespacePackage,
ParentPackage: ExtractParentPackage(node.ModulePath),
Imports: imports));
}
return this;
}
/// <summary>
/// Adds import edges from the import graph.
/// </summary>
public PythonObservationBuilder AddImportEdges(IEnumerable<PythonImportEdge> edges)
{
foreach (var edge in edges)
{
_importEdges.Add(new PythonObservationImportEdge(
FromModule: edge.From,
ToModule: edge.To,
Kind: MapImportKind(edge.Import.Kind),
Confidence: MapImportConfidence(edge.Import.Confidence),
ResolvedPath: null,
SourceFile: edge.Import.SourceFile,
Line: edge.Import.LineNumber ?? 0,
ResolverTrace: ImmutableArray<string>.Empty));
}
return this;
}
/// <summary>
/// Adds entrypoints from entrypoint discovery.
/// </summary>
public PythonObservationBuilder AddEntrypoints(IEnumerable<PythonEntrypoint> entrypoints)
{
foreach (var ep in entrypoints)
{
_entrypoints.Add(new PythonObservationEntrypoint(
Path: ep.VirtualPath ?? ep.Target,
Type: ep.Kind.ToString(),
Handler: ep.Callable,
RequiredPackages: ImmutableArray<string>.Empty,
InvocationContext: ep.InvocationContext.InvocationType.ToString()));
}
return this;
}
/// <summary>
/// Adds capabilities from capability detection.
/// </summary>
public PythonObservationBuilder AddCapabilities(IEnumerable<PythonCapability> capabilities)
{
foreach (var cap in capabilities)
{
switch (cap.Kind)
{
case PythonCapabilityKind.ProcessExecution:
_usesProcessExecution = true;
break;
case PythonCapabilityKind.NetworkAccess:
_usesNetworkAccess = true;
break;
case PythonCapabilityKind.FileSystemAccess:
_usesFileSystem = true;
break;
case PythonCapabilityKind.CodeExecution:
_usesCodeExecution = true;
break;
case PythonCapabilityKind.Deserialization:
_usesDeserialization = true;
break;
case PythonCapabilityKind.Ctypes or PythonCapabilityKind.Cffi or PythonCapabilityKind.NativeCodeExecution:
_usesNativeCode = true;
break;
case PythonCapabilityKind.AsyncAwait:
_usesAsyncAwait = true;
break;
case PythonCapabilityKind.Multiprocessing:
_usesMultiprocessing = true;
break;
}
if (cap.IsSecuritySensitive && !_securitySensitiveCapabilities.Contains(cap.Kind.ToString()))
{
_securitySensitiveCapabilities.Add(cap.Kind.ToString());
}
}
return this;
}
/// <summary>
/// Adds native extensions from extension scanning.
/// </summary>
public PythonObservationBuilder AddNativeExtensions(IEnumerable<PythonNativeExtension> extensions)
{
foreach (var ext in extensions)
{
_nativeExtensions.Add(new PythonObservationNativeExtension(
ModuleName: ext.ModuleName,
Path: ext.Path,
Kind: ext.Kind.ToString(),
Platform: ext.Platform,
Architecture: ext.Architecture,
PackageName: ext.PackageName));
_usesNativeCode = true;
}
return this;
}
/// <summary>
/// Adds framework hints from framework detection.
/// </summary>
public PythonObservationBuilder AddFrameworkHints(IEnumerable<PythonFrameworkHint> hints)
{
foreach (var hint in hints)
{
string? category = null;
if (hint.IsWebFramework) category = "WebFramework";
else if (hint.IsTaskQueue) category = "TaskQueue";
else if (hint.IsServerless) category = "Serverless";
else if (hint.IsCliFramework) category = "CLI";
_frameworks.Add(new PythonObservationFrameworkHint(
Kind: hint.Kind.ToString(),
SourceFile: hint.SourceFile,
Line: hint.LineNumber,
Evidence: hint.Evidence,
Confidence: MapConfidence(hint.Confidence),
Category: category));
}
return this;
}
/// <summary>
/// Sets environment information.
/// </summary>
public PythonObservationBuilder SetEnvironment(
string? pythonVersion,
IEnumerable<string>? sitePackagesPaths = null,
IEnumerable<string>? requirementsFiles = null,
IEnumerable<string>? pyprojectFiles = null,
string? virtualenvPath = null,
string? condaPrefix = null,
bool isContainer = false)
{
_environment = new PythonObservationEnvironment(
PythonVersion: pythonVersion,
SitePackagesPaths: sitePackagesPaths?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
VersionSources: ImmutableArray<PythonObservationVersionSource>.Empty,
RequirementsFiles: requirementsFiles?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
PyprojectFiles: pyprojectFiles?.ToImmutableArray() ?? ImmutableArray<string>.Empty,
VirtualenvPath: virtualenvPath,
CondaPrefix: condaPrefix,
IsContainer: isContainer);
return this;
}
/// <summary>
/// Adds a warning.
/// </summary>
public PythonObservationBuilder AddWarning(
string code,
string message,
string? filePath = null,
int? line = null,
string severity = "warning")
{
_warnings.Add(new PythonObservationWarning(
Code: code,
Message: message,
FilePath: filePath,
Line: line,
Severity: severity));
return this;
}
/// <summary>
/// Sets runtime evidence from optional runtime analysis.
/// </summary>
public PythonObservationBuilder SetRuntimeEvidence(PythonObservationRuntimeEvidence evidence)
{
_runtimeEvidence = evidence;
return this;
}
/// <summary>
/// Builds the final observation document.
/// </summary>
public PythonObservationDocument Build()
{
var detectedFrameworks = _frameworks
.Select(f => f.Kind)
.Distinct()
.ToImmutableArray();
return new PythonObservationDocument(
Schema: SchemaVersion,
Packages: _packages.ToImmutableArray(),
Modules: _modules.ToImmutableArray(),
Entrypoints: _entrypoints.ToImmutableArray(),
DependencyEdges: _dependencyEdges.ToImmutableArray(),
ImportEdges: _importEdges.ToImmutableArray(),
NativeExtensions: _nativeExtensions.ToImmutableArray(),
Frameworks: _frameworks.ToImmutableArray(),
Warnings: _warnings.ToImmutableArray(),
Environment: _environment ?? new PythonObservationEnvironment(
PythonVersion: null,
SitePackagesPaths: ImmutableArray<string>.Empty,
VersionSources: ImmutableArray<PythonObservationVersionSource>.Empty,
RequirementsFiles: ImmutableArray<string>.Empty,
PyprojectFiles: ImmutableArray<string>.Empty,
VirtualenvPath: null,
CondaPrefix: null,
IsContainer: false),
Capabilities: new PythonObservationCapabilitySummary(
UsesProcessExecution: _usesProcessExecution,
UsesNetworkAccess: _usesNetworkAccess,
UsesFileSystem: _usesFileSystem,
UsesCodeExecution: _usesCodeExecution,
UsesDeserialization: _usesDeserialization,
UsesNativeCode: _usesNativeCode,
UsesAsyncAwait: _usesAsyncAwait,
UsesMultiprocessing: _usesMultiprocessing,
DetectedFrameworks: detectedFrameworks,
SecuritySensitiveCapabilities: _securitySensitiveCapabilities.ToImmutableArray()),
RuntimeEvidence: _runtimeEvidence);
}
private static PythonObservationImportKind MapImportKind(PythonImportKind kind)
{
return kind switch
{
PythonImportKind.Import => PythonObservationImportKind.Import,
PythonImportKind.FromImport => PythonObservationImportKind.FromImport,
PythonImportKind.RelativeImport => PythonObservationImportKind.RelativeImport,
PythonImportKind.ImportlibImportModule => PythonObservationImportKind.DynamicImport,
PythonImportKind.BuiltinImport => PythonObservationImportKind.DynamicImport,
_ => PythonObservationImportKind.Import
};
}
private static PythonObservationConfidence MapImportConfidence(PythonImportConfidence confidence)
{
return confidence switch
{
PythonImportConfidence.Low => PythonObservationConfidence.Low,
PythonImportConfidence.Medium => PythonObservationConfidence.Medium,
PythonImportConfidence.High => PythonObservationConfidence.High,
PythonImportConfidence.Definitive => PythonObservationConfidence.Definitive,
_ => PythonObservationConfidence.Medium
};
}
private static PythonObservationConfidence MapConfidence(PythonFrameworkConfidence confidence)
{
return confidence switch
{
PythonFrameworkConfidence.Low => PythonObservationConfidence.Low,
PythonFrameworkConfidence.Medium => PythonObservationConfidence.Medium,
PythonFrameworkConfidence.High => PythonObservationConfidence.High,
PythonFrameworkConfidence.Definitive => PythonObservationConfidence.Definitive,
_ => PythonObservationConfidence.Medium
};
}
private static string ExtractPackageName(string dependency)
{
// Extract package name from dependency spec like "requests>=2.0" or "numpy[extra]"
var name = dependency;
var bracketIdx = name.IndexOf('[');
if (bracketIdx > 0) name = name[..bracketIdx];
foreach (var op in new[] { ">=", "<=", "==", "!=", ">", "<", "~=", "^" })
{
var opIdx = name.IndexOf(op, StringComparison.Ordinal);
if (opIdx > 0) name = name[..opIdx];
}
return name.Trim();
}
private static string? ExtractVersionConstraint(string dependency)
{
foreach (var op in new[] { ">=", "<=", "==", "!=", ">", "<", "~=", "^" })
{
var opIdx = dependency.IndexOf(op, StringComparison.Ordinal);
if (opIdx > 0) return dependency[opIdx..].Trim();
}
return null;
}
private static string? ExtractParentPackage(string moduleName)
{
var lastDot = moduleName.LastIndexOf('.');
return lastDot > 0 ? moduleName[..lastDot] : null;
}
}

View File

@@ -0,0 +1,231 @@
using System.Collections.Immutable;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.Observations;
/// <summary>
/// AOC-compliant observation document for Python project analysis.
/// Contains packages, modules, entrypoints, dependency edges, capabilities, and warnings.
/// </summary>
internal sealed record PythonObservationDocument(
string Schema,
ImmutableArray<PythonObservationPackage> Packages,
ImmutableArray<PythonObservationModule> Modules,
ImmutableArray<PythonObservationEntrypoint> Entrypoints,
ImmutableArray<PythonObservationDependencyEdge> DependencyEdges,
ImmutableArray<PythonObservationImportEdge> ImportEdges,
ImmutableArray<PythonObservationNativeExtension> NativeExtensions,
ImmutableArray<PythonObservationFrameworkHint> Frameworks,
ImmutableArray<PythonObservationWarning> Warnings,
PythonObservationEnvironment Environment,
PythonObservationCapabilitySummary Capabilities,
PythonObservationRuntimeEvidence? RuntimeEvidence = null);
/// <summary>
/// Python package detected in the project (from pip, conda, or other package managers).
/// </summary>
internal sealed record PythonObservationPackage(
string Name,
string Version,
string Source,
string? Platform,
bool IsDirect,
string? InstallerKind,
string? DistInfoPath,
ImmutableArray<string> Groups,
ImmutableArray<string> Extras);
/// <summary>
/// Python module or package detected in the project.
/// </summary>
internal sealed record PythonObservationModule(
string Name,
string Type,
string FilePath,
int? Line,
bool IsNamespacePackage,
string? ParentPackage,
ImmutableArray<string> Imports);
/// <summary>
/// Entrypoint detected in the Python project.
/// </summary>
internal sealed record PythonObservationEntrypoint(
string Path,
string Type,
string? Handler,
ImmutableArray<string> RequiredPackages,
string? InvocationContext);
/// <summary>
/// Package dependency edge (declared in requirements or pyproject).
/// </summary>
internal sealed record PythonObservationDependencyEdge(
string FromPackage,
string ToPackage,
string? VersionConstraint,
string? Extra,
bool IsOptional);
/// <summary>
/// Import edge between modules with reason codes and confidence.
/// </summary>
internal sealed record PythonObservationImportEdge(
string FromModule,
string ToModule,
PythonObservationImportKind Kind,
PythonObservationConfidence Confidence,
string? ResolvedPath,
string SourceFile,
int Line,
ImmutableArray<string> ResolverTrace);
/// <summary>
/// Import edge types.
/// </summary>
internal enum PythonObservationImportKind
{
/// <summary>Standard import statement.</summary>
Import,
/// <summary>From X import Y statement.</summary>
FromImport,
/// <summary>Relative import within package.</summary>
RelativeImport,
/// <summary>Dynamic import via importlib.</summary>
DynamicImport,
/// <summary>Namespace package implicit import.</summary>
NamespacePackage,
/// <summary>Native extension load.</summary>
NativeExtension,
/// <summary>Heuristic/hint-based import (not definitively resolved).</summary>
Hint
}
/// <summary>
/// Confidence level for observations.
/// </summary>
internal enum PythonObservationConfidence
{
/// <summary>Low confidence - heuristic match.</summary>
Low = 0,
/// <summary>Medium confidence - likely correct.</summary>
Medium = 1,
/// <summary>High confidence - clear evidence.</summary>
High = 2,
/// <summary>Definitive - direct evidence found.</summary>
Definitive = 3
}
/// <summary>
/// Native extension detected in the project.
/// </summary>
internal sealed record PythonObservationNativeExtension(
string ModuleName,
string Path,
string Kind,
string? Platform,
string? Architecture,
string? PackageName);
/// <summary>
/// Framework hint detected in the project.
/// </summary>
internal sealed record PythonObservationFrameworkHint(
string Kind,
string SourceFile,
int? Line,
string Evidence,
PythonObservationConfidence Confidence,
string? Category);
/// <summary>
/// Analysis warning generated during scanning.
/// </summary>
internal sealed record PythonObservationWarning(
string Code,
string Message,
string? FilePath,
int? Line,
string Severity);
/// <summary>
/// Environment profile with Python version, package manager settings, and paths.
/// </summary>
internal sealed record PythonObservationEnvironment(
string? PythonVersion,
ImmutableArray<string> SitePackagesPaths,
ImmutableArray<PythonObservationVersionSource> VersionSources,
ImmutableArray<string> RequirementsFiles,
ImmutableArray<string> PyprojectFiles,
string? VirtualenvPath,
string? CondaPrefix,
bool IsContainer);
/// <summary>
/// Python version source with provenance.
/// </summary>
internal sealed record PythonObservationVersionSource(
string? Version,
string Source,
string SourceType);
/// <summary>
/// Capability summary for the Python project.
/// </summary>
internal sealed record PythonObservationCapabilitySummary(
bool UsesProcessExecution,
bool UsesNetworkAccess,
bool UsesFileSystem,
bool UsesCodeExecution,
bool UsesDeserialization,
bool UsesNativeCode,
bool UsesAsyncAwait,
bool UsesMultiprocessing,
ImmutableArray<string> DetectedFrameworks,
ImmutableArray<string> SecuritySensitiveCapabilities);
/// <summary>
/// Optional runtime evidence section for Python.
/// </summary>
internal sealed record PythonObservationRuntimeEvidence(
bool HasEvidence,
string? RuntimePythonVersion,
string? RuntimePlatform,
int LoadedModulesCount,
ImmutableArray<string> LoadedPackages,
ImmutableArray<string> LoadedModules,
ImmutableDictionary<string, string> PathHashes,
ImmutableArray<string> RuntimeCapabilities,
ImmutableArray<PythonObservationRuntimeError> Errors)
{
/// <summary>
/// Empty runtime evidence instance.
/// </summary>
public static PythonObservationRuntimeEvidence Empty { get; } = new(
HasEvidence: false,
RuntimePythonVersion: null,
RuntimePlatform: null,
LoadedModulesCount: 0,
LoadedPackages: ImmutableArray<string>.Empty,
LoadedModules: ImmutableArray<string>.Empty,
PathHashes: ImmutableDictionary<string, string>.Empty,
RuntimeCapabilities: ImmutableArray<string>.Empty,
Errors: ImmutableArray<PythonObservationRuntimeError>.Empty);
}
/// <summary>
/// Runtime error captured during execution.
/// </summary>
internal sealed record PythonObservationRuntimeError(
string Timestamp,
string Message,
string? Path,
string? PathSha256);

View File

@@ -0,0 +1,73 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.Observations;
/// <summary>
/// Serializes Python observation documents to JSON.
/// </summary>
internal static class PythonObservationSerializer
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = true,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
Converters =
{
new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)
}
};
private static readonly JsonSerializerOptions CompactSerializerOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
WriteIndented = false,
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
Converters =
{
new JsonStringEnumConverter(JsonNamingPolicy.CamelCase)
}
};
/// <summary>
/// Serializes the observation document to JSON.
/// </summary>
public static string Serialize(PythonObservationDocument document, bool compact = false)
{
var options = compact ? CompactSerializerOptions : SerializerOptions;
return JsonSerializer.Serialize(document, options);
}
/// <summary>
/// Serializes the observation document to a stream.
/// </summary>
public static async Task SerializeAsync(
PythonObservationDocument document,
Stream stream,
bool compact = false,
CancellationToken cancellationToken = default)
{
var options = compact ? CompactSerializerOptions : SerializerOptions;
await JsonSerializer.SerializeAsync(stream, document, options, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Deserializes a JSON string to an observation document.
/// </summary>
public static PythonObservationDocument? Deserialize(string json)
{
return JsonSerializer.Deserialize<PythonObservationDocument>(json, SerializerOptions);
}
/// <summary>
/// Deserializes a stream to an observation document.
/// </summary>
public static async Task<PythonObservationDocument?> DeserializeAsync(
Stream stream,
CancellationToken cancellationToken = default)
{
return await JsonSerializer.DeserializeAsync<PythonObservationDocument>(
stream, SerializerOptions, cancellationToken).ConfigureAwait(false);
}
}

View File

@@ -0,0 +1,528 @@
using System.IO.Compression;
using System.Text;
using System.Text.RegularExpressions;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal;
/// <summary>
/// Analyzes Python zipapp archives (.pyz, .pyzw) for runtime information,
/// entry points, and startup behavior.
/// </summary>
internal static partial class PythonZipappAdapter
{
private static readonly string[] LayerRootCandidates = { "layers", ".layers", "layer" };
/// <summary>
/// Discovers zipapp files in the workspace and container layers.
/// </summary>
public static IReadOnlyCollection<string> DiscoverZipapps(string rootPath)
{
var discovered = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
// Search in root path
DiscoverInDirectory(rootPath, discovered);
// Search in container layers
foreach (var layerRoot in EnumerateLayerRoots(rootPath))
{
DiscoverInDirectory(layerRoot, discovered);
// Check common locations within layers
var appDir = Path.Combine(layerRoot, "app");
if (Directory.Exists(appDir))
{
DiscoverInDirectory(appDir, discovered);
}
var optDir = Path.Combine(layerRoot, "opt");
if (Directory.Exists(optDir))
{
DiscoverInDirectory(optDir, discovered);
}
var usrLocalBin = Path.Combine(layerRoot, "usr", "local", "bin");
if (Directory.Exists(usrLocalBin))
{
DiscoverInDirectory(usrLocalBin, discovered);
}
}
return discovered
.OrderBy(static path => path, StringComparer.Ordinal)
.ToArray();
}
/// <summary>
/// Analyzes a zipapp archive for runtime information.
/// </summary>
public static PythonZipappInfo? AnalyzeZipapp(string zipappPath)
{
if (!File.Exists(zipappPath))
{
return null;
}
try
{
var shebang = ExtractShebang(zipappPath);
var pythonVersion = shebang != null ? ParsePythonVersionFromShebang(shebang) : null;
var hasMain = false;
var hasInit = false;
var entryModule = (string?)null;
var warnings = new List<string>();
var dependencies = new List<string>();
// Open as zip archive to inspect contents
using var stream = File.OpenRead(zipappPath);
// Skip shebang if present
var firstByte = stream.ReadByte();
if (firstByte == '#')
{
// Skip to end of shebang line
while (stream.ReadByte() is int b && b != '\n' && b != -1)
{
}
}
else
{
stream.Position = 0;
}
using var archive = new ZipArchive(stream, ZipArchiveMode.Read);
foreach (var entry in archive.Entries)
{
var name = entry.FullName.Replace('\\', '/');
if (string.Equals(name, "__main__.py", StringComparison.OrdinalIgnoreCase))
{
hasMain = true;
entryModule = TryExtractEntryModule(entry);
}
else if (string.Equals(name, "__init__.py", StringComparison.OrdinalIgnoreCase))
{
hasInit = true;
}
else if (name.EndsWith("/__main__.py", StringComparison.OrdinalIgnoreCase))
{
// Package with __main__.py
var package = Path.GetDirectoryName(name)?.Replace('/', '.');
if (!string.IsNullOrEmpty(package))
{
entryModule ??= package;
}
}
else if (name.EndsWith("/requirements.txt", StringComparison.OrdinalIgnoreCase) ||
string.Equals(name, "requirements.txt", StringComparison.OrdinalIgnoreCase))
{
var reqs = ExtractRequirements(entry);
dependencies.AddRange(reqs);
}
}
// Generate warnings
if (!hasMain && !hasInit)
{
warnings.Add("Zipapp missing __main__.py; may not be directly executable");
}
if (shebang != null && shebang.Contains("/env ", StringComparison.OrdinalIgnoreCase))
{
warnings.Add("Zipapp uses /usr/bin/env shebang; Python version may vary by environment");
}
var isWindows = zipappPath.EndsWith(".pyzw", StringComparison.OrdinalIgnoreCase);
if (isWindows)
{
warnings.Add("Zipapp is Windows-specific (.pyzw); uses pythonw.exe without console");
}
return new PythonZipappInfo(
Path: zipappPath,
FileName: Path.GetFileName(zipappPath),
Shebang: shebang,
PythonVersion: pythonVersion,
HasMainPy: hasMain,
EntryModule: entryModule,
IsWindowsApp: isWindows,
EmbeddedDependencies: dependencies,
Warnings: warnings);
}
catch (IOException)
{
return null;
}
catch (InvalidDataException)
{
return null;
}
}
/// <summary>
/// Analyzes all zipapps in the workspace.
/// </summary>
public static PythonZipappAnalysis AnalyzeAll(string rootPath)
{
var zipapps = new List<PythonZipappInfo>();
var allWarnings = new List<string>();
foreach (var zipappPath in DiscoverZipapps(rootPath))
{
var info = AnalyzeZipapp(zipappPath);
if (info != null)
{
zipapps.Add(info);
allWarnings.AddRange(info.Warnings);
}
}
if (zipapps.Count > 1)
{
allWarnings.Add($"Multiple zipapps detected ({zipapps.Count}); entry point resolution may be ambiguous");
}
return new PythonZipappAnalysis(zipapps, allWarnings);
}
private static string? ExtractShebang(string path)
{
try
{
using var stream = File.OpenRead(path);
using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: false, leaveOpen: true);
var firstLine = reader.ReadLine();
if (firstLine != null && firstLine.StartsWith("#!"))
{
return firstLine[2..].Trim();
}
return null;
}
catch (IOException)
{
return null;
}
}
private static string? ParsePythonVersionFromShebang(string shebang)
{
// Match patterns like:
// /usr/bin/python3.11
// /usr/bin/env python3.10
// python3.9
var match = PythonVersionPattern().Match(shebang);
if (match.Success)
{
return match.Groups["version"].Value;
}
// Check for generic python3 or python
if (shebang.Contains("python3", StringComparison.OrdinalIgnoreCase))
{
return "3";
}
if (shebang.Contains("python", StringComparison.OrdinalIgnoreCase))
{
return null; // Could be Python 2 or 3
}
return null;
}
private static string? TryExtractEntryModule(ZipArchiveEntry entry)
{
try
{
using var stream = entry.Open();
using var reader = new StreamReader(stream);
var content = reader.ReadToEnd();
// Look for common patterns:
// from package.module import main
// import package.main
// runpy.run_module('package')
var runpyMatch = RunpyPattern().Match(content);
if (runpyMatch.Success)
{
return runpyMatch.Groups["module"].Value;
}
var fromImportMatch = FromImportPattern().Match(content);
if (fromImportMatch.Success)
{
return fromImportMatch.Groups["module"].Value;
}
return null;
}
catch (IOException)
{
return null;
}
}
private static List<string> ExtractRequirements(ZipArchiveEntry entry)
{
var results = new List<string>();
try
{
using var stream = entry.Open();
using var reader = new StreamReader(stream);
var content = reader.ReadToEnd();
var lines = content.Split('\n', StringSplitOptions.RemoveEmptyEntries);
foreach (var line in lines)
{
var trimmed = line.Trim();
if (string.IsNullOrEmpty(trimmed) || trimmed.StartsWith('#') || trimmed.StartsWith('-'))
{
continue;
}
// Extract package name (before any version specifier)
var match = PackageNamePattern().Match(trimmed);
if (match.Success)
{
results.Add(match.Groups["name"].Value);
}
}
}
catch (IOException)
{
// Ignore read errors
}
return results;
}
private static void DiscoverInDirectory(string directory, HashSet<string> discovered)
{
if (!Directory.Exists(directory))
{
return;
}
try
{
foreach (var file in Directory.EnumerateFiles(directory, "*.pyz"))
{
discovered.Add(file);
}
foreach (var file in Directory.EnumerateFiles(directory, "*.pyzw"))
{
discovered.Add(file);
}
// Also check in subdirectories (up to 3 levels)
foreach (var subdir in Directory.EnumerateDirectories(directory))
{
DiscoverInSubdirectory(subdir, discovered, 1);
}
}
catch (IOException)
{
// Ignore
}
catch (UnauthorizedAccessException)
{
// Ignore
}
}
private static void DiscoverInSubdirectory(string directory, HashSet<string> discovered, int depth)
{
if (depth > 3)
{
return;
}
try
{
foreach (var file in Directory.EnumerateFiles(directory, "*.pyz"))
{
discovered.Add(file);
}
foreach (var file in Directory.EnumerateFiles(directory, "*.pyzw"))
{
discovered.Add(file);
}
foreach (var subdir in Directory.EnumerateDirectories(directory))
{
var dirName = Path.GetFileName(subdir);
// Skip common non-relevant directories
if (dirName.StartsWith('.') ||
string.Equals(dirName, "node_modules", StringComparison.OrdinalIgnoreCase) ||
string.Equals(dirName, "__pycache__", StringComparison.OrdinalIgnoreCase) ||
string.Equals(dirName, "venv", StringComparison.OrdinalIgnoreCase) ||
string.Equals(dirName, ".venv", StringComparison.OrdinalIgnoreCase))
{
continue;
}
DiscoverInSubdirectory(subdir, discovered, depth + 1);
}
}
catch (IOException)
{
// Ignore
}
catch (UnauthorizedAccessException)
{
// Ignore
}
}
private static IEnumerable<string> EnumerateLayerRoots(string workspaceRoot)
{
foreach (var candidate in LayerRootCandidates)
{
var root = Path.Combine(workspaceRoot, candidate);
if (!Directory.Exists(root))
{
continue;
}
IEnumerable<string>? directories;
try
{
directories = Directory.EnumerateDirectories(root);
}
catch (IOException)
{
continue;
}
catch (UnauthorizedAccessException)
{
continue;
}
foreach (var layerDirectory in directories)
{
var fsDirectory = Path.Combine(layerDirectory, "fs");
yield return Directory.Exists(fsDirectory) ? fsDirectory : layerDirectory;
}
}
}
[GeneratedRegex(@"python(?<version>\d+\.\d+)", RegexOptions.IgnoreCase)]
private static partial Regex PythonVersionPattern();
[GeneratedRegex(@"runpy\.run_module\(['""](?<module>[^'""]+)['""]", RegexOptions.IgnoreCase)]
private static partial Regex RunpyPattern();
[GeneratedRegex(@"from\s+(?<module>[\w.]+)\s+import", RegexOptions.IgnoreCase)]
private static partial Regex FromImportPattern();
[GeneratedRegex(@"^(?<name>[\w\-_.]+)", RegexOptions.IgnoreCase)]
private static partial Regex PackageNamePattern();
}
/// <summary>
/// Information about a Python zipapp archive.
/// </summary>
internal sealed record PythonZipappInfo(
string Path,
string FileName,
string? Shebang,
string? PythonVersion,
bool HasMainPy,
string? EntryModule,
bool IsWindowsApp,
IReadOnlyCollection<string> EmbeddedDependencies,
IReadOnlyCollection<string> Warnings)
{
public IReadOnlyCollection<KeyValuePair<string, string?>> ToMetadata()
{
var entries = new List<KeyValuePair<string, string?>>
{
new("zipapp.path", Path),
new("zipapp.hasMain", HasMainPy.ToString().ToLowerInvariant())
};
if (Shebang != null)
{
entries.Add(new("zipapp.shebang", Shebang));
}
if (PythonVersion != null)
{
entries.Add(new("zipapp.pythonVersion", PythonVersion));
}
if (EntryModule != null)
{
entries.Add(new("zipapp.entryModule", EntryModule));
}
if (IsWindowsApp)
{
entries.Add(new("zipapp.windowsApp", "true"));
}
if (EmbeddedDependencies.Count > 0)
{
entries.Add(new("zipapp.embeddedDeps.count", EmbeddedDependencies.Count.ToString()));
}
return entries;
}
}
/// <summary>
/// Analysis results for all zipapps in a workspace.
/// </summary>
internal sealed class PythonZipappAnalysis
{
public PythonZipappAnalysis(
IReadOnlyCollection<PythonZipappInfo> zipapps,
IReadOnlyCollection<string> warnings)
{
Zipapps = zipapps;
Warnings = warnings;
}
public IReadOnlyCollection<PythonZipappInfo> Zipapps { get; }
public IReadOnlyCollection<string> Warnings { get; }
public bool HasZipapps => Zipapps.Count > 0;
public bool HasWarnings => Warnings.Count > 0;
public IReadOnlyCollection<KeyValuePair<string, string?>> ToMetadata()
{
var entries = new List<KeyValuePair<string, string?>>();
if (Zipapps.Count > 0)
{
entries.Add(new("zipapps.count", Zipapps.Count.ToString()));
var withShebang = Zipapps.Count(z => z.Shebang != null);
if (withShebang > 0)
{
entries.Add(new("zipapps.withShebang", withShebang.ToString()));
}
var windowsApps = Zipapps.Count(z => z.IsWindowsApp);
if (windowsApps > 0)
{
entries.Add(new("zipapps.windowsApps", windowsApps.ToString()));
}
}
for (var i = 0; i < Warnings.Count; i++)
{
entries.Add(new($"zipapps.warning[{i}]", Warnings.ElementAt(i)));
}
return entries;
}
}

View File

@@ -0,0 +1,194 @@
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.RuntimeEvidence;
/// <summary>
/// Provides the Python import hook script for runtime evidence collection.
/// </summary>
internal static class PythonImportHookScript
{
/// <summary>
/// Gets the Python import hook script that captures module load events.
/// This script outputs NDJSON to stdout.
/// </summary>
public static string GetScript() => Script;
/// <summary>
/// Gets the Python import hook script that writes to a file.
/// </summary>
/// <param name="outputPath">The path to write output to.</param>
/// <returns>The modified script.</returns>
public static string GetFileScript(string outputPath)
{
var escapedPath = outputPath.Replace("\\", "\\\\").Replace("'", "\\'");
return Script.Replace(
"_stellaops_output = None",
$"_stellaops_output = open('{escapedPath}', 'w', buffering=1)");
}
// The Python script is stored as a verbatim string to avoid issues with # characters
private const string Script = @"
# StellaOps Python Import Hook
# This script captures module import events for static analysis validation.
# Output format: NDJSON (Newline-Delimited JSON)
import sys
import json
import threading
import os
from datetime import datetime, timezone
_stellaops_output = None
_stellaops_lock = threading.Lock()
_stellaops_seen = set()
def _stellaops_emit(event_type, **kwargs):
""""""Emit an event as JSON.""""""
event = {
'type': event_type,
'timestamp': datetime.now(timezone.utc).isoformat(),
'pid': os.getpid(),
**kwargs
}
with _stellaops_lock:
line = json.dumps(event, default=str)
if _stellaops_output:
_stellaops_output.write(line + '\n')
_stellaops_output.flush()
else:
print(line, flush=True)
def _stellaops_get_module_path(module):
""""""Get the file path for a module if available.""""""
try:
if hasattr(module, '__file__') and module.__file__:
return module.__file__
if hasattr(module, '__spec__') and module.__spec__:
if hasattr(module.__spec__, 'origin') and module.__spec__.origin:
return module.__spec__.origin
except Exception:
pass
return None
class StellaOpsMetaPathFinder:
""""""Meta path finder that logs all import attempts.""""""
def find_module(self, fullname, path=None):
if fullname not in _stellaops_seen:
_stellaops_seen.add(fullname)
_stellaops_emit('import_attempt', module=fullname, path=str(path) if path else None)
return None
def find_spec(self, fullname, path, target=None):
return None
def _stellaops_wrap_import():
""""""Wrap the built-in __import__ to capture all imports.""""""
original_import = __builtins__.__import__ if hasattr(__builtins__, '__import__') else __builtins__['__import__']
def wrapped_import(name, globals=None, locals=None, fromlist=(), level=0):
module = original_import(name, globals, locals, fromlist, level)
if name not in _stellaops_seen:
_stellaops_seen.add(name)
path = _stellaops_get_module_path(module)
is_native = path and (path.endswith('.so') or path.endswith('.pyd'))
event_type = 'native_load' if is_native else 'module_import'
_stellaops_emit(
event_type,
module=name,
path=path,
parent=module.__package__ if hasattr(module, '__package__') else None,
tid=threading.get_ident()
)
return module
if hasattr(__builtins__, '__import__'):
__builtins__.__import__ = wrapped_import
else:
__builtins__['__import__'] = wrapped_import
def _stellaops_wrap_subprocess():
""""""Wrap subprocess module to capture process spawns.""""""
try:
import subprocess
original_popen = subprocess.Popen
class WrappedPopen(original_popen):
def __init__(self, *args, **kwargs):
_stellaops_emit('process_spawn', spawn_type='subprocess', args=str(args[0]) if args else None)
super().__init__(*args, **kwargs)
subprocess.Popen = WrappedPopen
except Exception:
pass
def _stellaops_wrap_multiprocessing():
""""""Wrap multiprocessing module to capture process spawns.""""""
try:
import multiprocessing
original_process = multiprocessing.Process
class WrappedProcess(original_process):
def __init__(self, *args, **kwargs):
_stellaops_emit('process_spawn', spawn_type='multiprocessing', target=str(kwargs.get('target')))
super().__init__(*args, **kwargs)
multiprocessing.Process = WrappedProcess
except Exception:
pass
def stellaops_start_tracing():
""""""Initialize runtime evidence collection.""""""
# Emit interpreter start event
_stellaops_emit(
'interpreter_start',
python_version=sys.version,
platform=sys.platform,
executable=sys.executable
)
# Install import hook
_stellaops_wrap_import()
# Install meta path finder
sys.meta_path.insert(0, StellaOpsMetaPathFinder())
# Wrap subprocess and multiprocessing (optional)
_stellaops_wrap_subprocess()
_stellaops_wrap_multiprocessing()
# Record already-loaded modules
for name, module in list(sys.modules.items()):
if module is not None and name not in _stellaops_seen:
_stellaops_seen.add(name)
path = _stellaops_get_module_path(module)
if path:
is_native = path.endswith('.so') or path.endswith('.pyd')
event_type = 'native_load' if is_native else 'module_import'
_stellaops_emit(
event_type,
module=name,
path=path,
preloaded=True
)
# Auto-start if this module is imported
if __name__ != '__main__':
stellaops_start_tracing()
else:
# If run directly, start tracing and wait
stellaops_start_tracing()
import sys
print('StellaOps import hook active. Press Ctrl+C to stop.', file=sys.stderr)
try:
import time
while True:
time.sleep(1)
except KeyboardInterrupt:
pass
";
}

View File

@@ -0,0 +1,194 @@
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.RuntimeEvidence;
/// <summary>
/// Provides path scrubbing and hashing for privacy-preserving runtime evidence.
/// </summary>
internal static partial class PythonPathHasher
{
private static readonly string[] SensitivePatterns =
[
@"/home/[^/]+",
@"/Users/[^/]+",
@"C:\\Users\\[^\\]+",
@"/root",
@"/tmp/[^/]+",
@"/var/[^/]+/[^/]+",
];
/// <summary>
/// Scrubs sensitive path components and returns a normalized path.
/// </summary>
/// <param name="path">The path to scrub.</param>
/// <returns>A scrubbed path with sensitive components replaced.</returns>
public static string ScrubPath(string? path)
{
if (string.IsNullOrEmpty(path))
{
return string.Empty;
}
var result = path;
// Replace home directories
result = HomeDirectoryPattern().Replace(result, "[HOME]");
result = WindowsUserPattern().Replace(result, "[HOME]");
result = MacUserPattern().Replace(result, "[HOME]");
result = RootPattern().Replace(result, "[ROOT]");
// Replace temp directories
result = TempPattern().Replace(result, "[TEMP]");
// Replace container-specific paths
result = ContainerAppPattern().Replace(result, "[APP]");
// Normalize path separators
result = result.Replace('\\', '/');
return result;
}
/// <summary>
/// Computes a SHA-256 hash of a path for deterministic identification.
/// </summary>
/// <param name="path">The path to hash.</param>
/// <returns>The SHA-256 hash as a hex string.</returns>
public static string HashPath(string? path)
{
if (string.IsNullOrEmpty(path))
{
return string.Empty;
}
var normalizedPath = NormalizePath(path);
var bytes = Encoding.UTF8.GetBytes(normalizedPath);
var hash = SHA256.HashData(bytes);
return Convert.ToHexStringLower(hash);
}
/// <summary>
/// Creates a scrubbed and hashed mapping for a path.
/// </summary>
/// <param name="path">The original path.</param>
/// <returns>A tuple of (scrubbedPath, hash).</returns>
public static (string ScrubbedPath, string Hash) ScrubAndHash(string? path)
{
var scrubbed = ScrubPath(path);
var hash = HashPath(path);
return (scrubbed, hash);
}
/// <summary>
/// Normalizes a path for consistent hashing.
/// </summary>
private static string NormalizePath(string path)
{
// Convert to lowercase and normalize separators
var result = path.ToLowerInvariant().Replace('\\', '/');
// Remove trailing slashes
result = result.TrimEnd('/');
// Collapse multiple slashes
result = MultiSlashPattern().Replace(result, "/");
return result;
}
/// <summary>
/// Extracts the module name from a file path.
/// </summary>
/// <param name="path">The file path.</param>
/// <returns>The module name, or null if not determinable.</returns>
public static string? ExtractModuleName(string? path)
{
if (string.IsNullOrEmpty(path))
{
return null;
}
// Normalize path
var normalizedPath = path.Replace('\\', '/');
// Check for site-packages
var sitePackagesIndex = normalizedPath.IndexOf("site-packages/", StringComparison.OrdinalIgnoreCase);
if (sitePackagesIndex >= 0)
{
var relativePath = normalizedPath[(sitePackagesIndex + 14)..];
return ConvertPathToModule(relativePath);
}
// Check for dist-packages
var distPackagesIndex = normalizedPath.IndexOf("dist-packages/", StringComparison.OrdinalIgnoreCase);
if (distPackagesIndex >= 0)
{
var relativePath = normalizedPath[(distPackagesIndex + 14)..];
return ConvertPathToModule(relativePath);
}
// Fallback: extract filename
var fileName = Path.GetFileNameWithoutExtension(normalizedPath);
return string.IsNullOrEmpty(fileName) ? null : fileName;
}
/// <summary>
/// Converts a relative file path to a Python module name.
/// </summary>
private static string ConvertPathToModule(string relativePath)
{
// Remove file extension
if (relativePath.EndsWith(".py", StringComparison.OrdinalIgnoreCase))
{
relativePath = relativePath[..^3];
}
else if (relativePath.EndsWith(".so", StringComparison.OrdinalIgnoreCase) ||
relativePath.EndsWith(".pyd", StringComparison.OrdinalIgnoreCase))
{
// Handle native extensions like module.cpython-311-x86_64-linux-gnu.so
var lastDot = relativePath.LastIndexOf('.');
if (lastDot > 0)
{
relativePath = relativePath[..lastDot];
// Remove cpython version suffix
var cpythonIndex = relativePath.IndexOf(".cpython-", StringComparison.OrdinalIgnoreCase);
if (cpythonIndex > 0)
{
relativePath = relativePath[..cpythonIndex];
}
}
}
// Handle __init__ files
if (relativePath.EndsWith("/__init__", StringComparison.OrdinalIgnoreCase))
{
relativePath = relativePath[..^9];
}
// Convert path separators to dots
return relativePath.Replace('/', '.');
}
[GeneratedRegex(@"/home/[^/]+", RegexOptions.IgnoreCase)]
private static partial Regex HomeDirectoryPattern();
[GeneratedRegex(@"C:\\Users\\[^\\]+", RegexOptions.IgnoreCase)]
private static partial Regex WindowsUserPattern();
[GeneratedRegex(@"/Users/[^/]+", RegexOptions.IgnoreCase)]
private static partial Regex MacUserPattern();
[GeneratedRegex(@"/root\b")]
private static partial Regex RootPattern();
[GeneratedRegex(@"/tmp/[^/]+")]
private static partial Regex TempPattern();
[GeneratedRegex(@"^/app(?=/|$)")]
private static partial Regex ContainerAppPattern();
[GeneratedRegex(@"//+")]
private static partial Regex MultiSlashPattern();
}

View File

@@ -0,0 +1,51 @@
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.RuntimeEvidence;
/// <summary>
/// Represents a runtime event captured from Python execution.
/// </summary>
internal sealed record PythonRuntimeEvent(
PythonRuntimeEventKind Kind,
string Timestamp,
string? ModuleName,
string? ModulePath,
string? ModuleSpec,
string? ParentModule,
string? ErrorMessage,
string? ProcessId,
string? ThreadId);
/// <summary>
/// Kind of runtime event captured.
/// </summary>
internal enum PythonRuntimeEventKind
{
/// <summary>Module was imported.</summary>
ModuleImport,
/// <summary>Module load failed.</summary>
ModuleLoadError,
/// <summary>Native extension was loaded.</summary>
NativeExtensionLoad,
/// <summary>Dynamic import via importlib.</summary>
DynamicImport,
/// <summary>Process spawned via subprocess/multiprocessing.</summary>
ProcessSpawn,
/// <summary>New thread started.</summary>
ThreadStart,
/// <summary>Import hook was installed.</summary>
ImportHookInstall,
/// <summary>Path was added to sys.path.</summary>
PathModification,
/// <summary>Python interpreter started.</summary>
InterpreterStart,
/// <summary>Python interpreter stopped.</summary>
InterpreterStop
}

View File

@@ -0,0 +1,396 @@
using System.Collections.Immutable;
using System.Text.Json;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Observations;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Internal.RuntimeEvidence;
/// <summary>
/// Collects and processes runtime evidence from Python execution traces.
/// </summary>
internal sealed class PythonRuntimeEvidenceCollector
{
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNameCaseInsensitive = true,
AllowTrailingCommas = true
};
private readonly List<PythonRuntimeEvent> _events = [];
private readonly Dictionary<string, string> _pathHashes = new();
private readonly HashSet<string> _loadedModules = new(StringComparer.Ordinal);
private readonly HashSet<string> _loadedPackages = new(StringComparer.Ordinal);
private readonly HashSet<string> _runtimeCapabilities = new(StringComparer.Ordinal);
private readonly List<PythonObservationRuntimeError> _errors = [];
private string? _pythonVersion;
private string? _platform;
/// <summary>
/// Parses a JSON line from the runtime evidence output.
/// </summary>
/// <param name="jsonLine">A JSON line containing event data.</param>
public void ParseLine(string jsonLine)
{
if (string.IsNullOrWhiteSpace(jsonLine))
{
return;
}
try
{
using var doc = JsonDocument.Parse(jsonLine);
var root = doc.RootElement;
var eventType = root.GetProperty("type").GetString();
switch (eventType)
{
case "interpreter_start":
ParseInterpreterStart(root);
break;
case "module_import":
ParseModuleImport(root);
break;
case "module_error":
ParseModuleError(root);
break;
case "native_load":
ParseNativeLoad(root);
break;
case "dynamic_import":
ParseDynamicImport(root);
break;
case "process_spawn":
ParseProcessSpawn(root);
break;
case "path_modification":
ParsePathModification(root);
break;
}
}
catch (JsonException)
{
// Skip malformed JSON lines
}
}
/// <summary>
/// Parses multiple JSON lines from the runtime evidence output.
/// </summary>
/// <param name="output">The complete output string (NDJSON format).</param>
public void ParseOutput(string output)
{
if (string.IsNullOrEmpty(output))
{
return;
}
foreach (var line in output.Split('\n', StringSplitOptions.RemoveEmptyEntries))
{
ParseLine(line.Trim());
}
}
/// <summary>
/// Parses runtime evidence from a file.
/// </summary>
/// <param name="filePath">Path to the NDJSON evidence file.</param>
/// <param name="cancellationToken">Cancellation token.</param>
public async Task ParseFileAsync(string filePath, CancellationToken cancellationToken = default)
{
if (!File.Exists(filePath))
{
return;
}
await foreach (var line in File.ReadLinesAsync(filePath, cancellationToken).ConfigureAwait(false))
{
ParseLine(line);
}
}
/// <summary>
/// Builds the runtime evidence observation from collected data.
/// </summary>
/// <returns>The runtime evidence observation.</returns>
public PythonObservationRuntimeEvidence Build()
{
if (_events.Count == 0 && _loadedModules.Count == 0)
{
return PythonObservationRuntimeEvidence.Empty;
}
return new PythonObservationRuntimeEvidence(
HasEvidence: true,
RuntimePythonVersion: _pythonVersion,
RuntimePlatform: _platform,
LoadedModulesCount: _loadedModules.Count,
LoadedPackages: [.. _loadedPackages.OrderBy(p => p, StringComparer.Ordinal)],
LoadedModules: [.. _loadedModules.OrderBy(m => m, StringComparer.Ordinal)],
PathHashes: _pathHashes.ToImmutableDictionary(StringComparer.Ordinal),
RuntimeCapabilities: [.. _runtimeCapabilities.OrderBy(c => c, StringComparer.Ordinal)],
Errors: [.. _errors]);
}
/// <summary>
/// Gets all captured runtime events.
/// </summary>
public IReadOnlyList<PythonRuntimeEvent> Events => _events;
private void ParseInterpreterStart(JsonElement root)
{
_pythonVersion = root.TryGetProperty("python_version", out var version)
? version.GetString()
: null;
_platform = root.TryGetProperty("platform", out var platform)
? platform.GetString()
: null;
var timestamp = root.TryGetProperty("timestamp", out var ts)
? ts.GetString() ?? GetUtcTimestamp()
: GetUtcTimestamp();
_events.Add(new PythonRuntimeEvent(
Kind: PythonRuntimeEventKind.InterpreterStart,
Timestamp: timestamp,
ModuleName: null,
ModulePath: null,
ModuleSpec: null,
ParentModule: null,
ErrorMessage: null,
ProcessId: root.TryGetProperty("pid", out var pid) ? pid.GetInt32().ToString() : null,
ThreadId: null));
}
private void ParseModuleImport(JsonElement root)
{
var moduleName = root.TryGetProperty("module", out var module)
? module.GetString()
: null;
var modulePath = root.TryGetProperty("path", out var path)
? path.GetString()
: null;
var timestamp = root.TryGetProperty("timestamp", out var ts)
? ts.GetString() ?? GetUtcTimestamp()
: GetUtcTimestamp();
if (!string.IsNullOrEmpty(moduleName))
{
_loadedModules.Add(moduleName);
// Extract top-level package
var topLevelPackage = moduleName.Split('.')[0];
_loadedPackages.Add(topLevelPackage);
}
// Add path hash
if (!string.IsNullOrEmpty(modulePath))
{
var (scrubbed, hash) = PythonPathHasher.ScrubAndHash(modulePath);
if (!string.IsNullOrEmpty(hash))
{
_pathHashes.TryAdd(scrubbed, hash);
}
}
_events.Add(new PythonRuntimeEvent(
Kind: PythonRuntimeEventKind.ModuleImport,
Timestamp: timestamp,
ModuleName: moduleName,
ModulePath: PythonPathHasher.ScrubPath(modulePath),
ModuleSpec: root.TryGetProperty("spec", out var spec) ? spec.GetString() : null,
ParentModule: root.TryGetProperty("parent", out var parent) ? parent.GetString() : null,
ErrorMessage: null,
ProcessId: root.TryGetProperty("pid", out var pid) ? pid.GetInt32().ToString() : null,
ThreadId: root.TryGetProperty("tid", out var tid) ? tid.GetInt64().ToString() : null));
}
private void ParseModuleError(JsonElement root)
{
var moduleName = root.TryGetProperty("module", out var module)
? module.GetString()
: null;
var errorMessage = root.TryGetProperty("error", out var error)
? error.GetString()
: null;
var modulePath = root.TryGetProperty("path", out var path)
? path.GetString()
: null;
var timestamp = root.TryGetProperty("timestamp", out var ts)
? ts.GetString() ?? GetUtcTimestamp()
: GetUtcTimestamp();
_events.Add(new PythonRuntimeEvent(
Kind: PythonRuntimeEventKind.ModuleLoadError,
Timestamp: timestamp,
ModuleName: moduleName,
ModulePath: PythonPathHasher.ScrubPath(modulePath),
ModuleSpec: null,
ParentModule: null,
ErrorMessage: errorMessage,
ProcessId: root.TryGetProperty("pid", out var pid) ? pid.GetInt32().ToString() : null,
ThreadId: null));
// Add to errors
var scrubbed = PythonPathHasher.ScrubPath(modulePath);
var hash = PythonPathHasher.HashPath(modulePath);
_errors.Add(new PythonObservationRuntimeError(
Timestamp: timestamp,
Message: errorMessage ?? $"Failed to import {moduleName}",
Path: scrubbed,
PathSha256: string.IsNullOrEmpty(hash) ? null : hash));
}
private void ParseNativeLoad(JsonElement root)
{
var moduleName = root.TryGetProperty("module", out var module)
? module.GetString()
: null;
var modulePath = root.TryGetProperty("path", out var path)
? path.GetString()
: null;
var timestamp = root.TryGetProperty("timestamp", out var ts)
? ts.GetString() ?? GetUtcTimestamp()
: GetUtcTimestamp();
if (!string.IsNullOrEmpty(moduleName))
{
_loadedModules.Add(moduleName);
}
// Track native code capability
_runtimeCapabilities.Add("native_code");
// Add path hash
if (!string.IsNullOrEmpty(modulePath))
{
var (scrubbed, hash) = PythonPathHasher.ScrubAndHash(modulePath);
if (!string.IsNullOrEmpty(hash))
{
_pathHashes.TryAdd(scrubbed, hash);
}
}
_events.Add(new PythonRuntimeEvent(
Kind: PythonRuntimeEventKind.NativeExtensionLoad,
Timestamp: timestamp,
ModuleName: moduleName,
ModulePath: PythonPathHasher.ScrubPath(modulePath),
ModuleSpec: null,
ParentModule: null,
ErrorMessage: null,
ProcessId: root.TryGetProperty("pid", out var pid) ? pid.GetInt32().ToString() : null,
ThreadId: null));
}
private void ParseDynamicImport(JsonElement root)
{
var moduleName = root.TryGetProperty("module", out var module)
? module.GetString()
: null;
var timestamp = root.TryGetProperty("timestamp", out var ts)
? ts.GetString() ?? GetUtcTimestamp()
: GetUtcTimestamp();
if (!string.IsNullOrEmpty(moduleName))
{
_loadedModules.Add(moduleName);
}
// Track dynamic import capability
_runtimeCapabilities.Add("dynamic_import");
_events.Add(new PythonRuntimeEvent(
Kind: PythonRuntimeEventKind.DynamicImport,
Timestamp: timestamp,
ModuleName: moduleName,
ModulePath: null,
ModuleSpec: null,
ParentModule: null,
ErrorMessage: null,
ProcessId: root.TryGetProperty("pid", out var pid) ? pid.GetInt32().ToString() : null,
ThreadId: null));
}
private void ParseProcessSpawn(JsonElement root)
{
var timestamp = root.TryGetProperty("timestamp", out var ts)
? ts.GetString() ?? GetUtcTimestamp()
: GetUtcTimestamp();
// Track process spawn capability
_runtimeCapabilities.Add("process_spawn");
var spawnType = root.TryGetProperty("spawn_type", out var st)
? st.GetString()
: "unknown";
if (spawnType == "multiprocessing")
{
_runtimeCapabilities.Add("multiprocessing");
}
_events.Add(new PythonRuntimeEvent(
Kind: PythonRuntimeEventKind.ProcessSpawn,
Timestamp: timestamp,
ModuleName: null,
ModulePath: null,
ModuleSpec: spawnType,
ParentModule: null,
ErrorMessage: null,
ProcessId: root.TryGetProperty("pid", out var pid) ? pid.GetInt32().ToString() : null,
ThreadId: null));
}
private void ParsePathModification(JsonElement root)
{
var path = root.TryGetProperty("path", out var p)
? p.GetString()
: null;
var timestamp = root.TryGetProperty("timestamp", out var ts)
? ts.GetString() ?? GetUtcTimestamp()
: GetUtcTimestamp();
// Add path hash
if (!string.IsNullOrEmpty(path))
{
var (scrubbed, hash) = PythonPathHasher.ScrubAndHash(path);
if (!string.IsNullOrEmpty(hash))
{
_pathHashes.TryAdd(scrubbed, hash);
}
}
_events.Add(new PythonRuntimeEvent(
Kind: PythonRuntimeEventKind.PathModification,
Timestamp: timestamp,
ModuleName: null,
ModulePath: PythonPathHasher.ScrubPath(path),
ModuleSpec: null,
ParentModule: null,
ErrorMessage: null,
ProcessId: root.TryGetProperty("pid", out var pid) ? pid.GetInt32().ToString() : null,
ThreadId: null));
}
private static string GetUtcTimestamp()
{
return DateTime.UtcNow.ToString("O");
}
}

View File

@@ -40,6 +40,9 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
// Detect startup hooks (sitecustomize.py, usercustomize.py, .pth files)
var startupHooks = PythonStartupHookDetector.Detect(context.RootPath);
// Analyze zipapps in workspace and container layers
var zipappAnalysis = PythonZipappAdapter.AnalyzeAll(context.RootPath);
// Collect dist-info directories from both root and container layers
var distInfoDirectories = CollectDistInfoDirectories(context.RootPath);
@@ -91,6 +94,9 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
// Append startup hooks warnings
AppendStartupHooksMetadata(metadata, startupHooks);
// Append zipapp analysis
AppendZipappMetadata(metadata, zipappAnalysis);
// Collect evidence including startup hooks
var evidence = distribution.SortedEvidence.ToList();
evidence.AddRange(startupHooks.ToEvidence(context));
@@ -242,6 +248,45 @@ public sealed class PythonLanguageAnalyzer : ILanguageAnalyzer
}
}
private static void AppendZipappMetadata(List<KeyValuePair<string, string?>> metadata, PythonZipappAnalysis zipappAnalysis)
{
if (!zipappAnalysis.HasZipapps)
{
return;
}
metadata.Add(new KeyValuePair<string, string?>("zipapps.detected", "true"));
metadata.Add(new KeyValuePair<string, string?>("zipapps.count", zipappAnalysis.Zipapps.Count.ToString()));
// Add version information from zipapp shebangs
var versions = zipappAnalysis.Zipapps
.Where(z => z.PythonVersion != null)
.Select(z => z.PythonVersion!)
.Distinct()
.OrderBy(v => v, StringComparer.Ordinal)
.ToArray();
if (versions.Length > 0)
{
metadata.Add(new KeyValuePair<string, string?>("zipapps.pythonVersions", string.Join(';', versions)));
}
// Add warnings
foreach (var warning in zipappAnalysis.Warnings)
{
metadata.Add(new KeyValuePair<string, string?>("zipapps.warning", warning));
}
// Add individual zipapp warnings
foreach (var zipapp in zipappAnalysis.Zipapps)
{
foreach (var warning in zipapp.Warnings)
{
metadata.Add(new KeyValuePair<string, string?>($"zipapps.{zipapp.FileName}.warning", warning));
}
}
}
private static IReadOnlyCollection<string> CollectDistInfoDirectories(string rootPath)
{
var directories = new HashSet<string>(StringComparer.OrdinalIgnoreCase);

View File

@@ -0,0 +1,26 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Analyzers.OS.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Plugin;
namespace StellaOps.Scanner.Analyzers.OS.Homebrew;
/// <summary>
/// Plugin that registers the Homebrew package analyzer for macOS Cellar discovery.
/// </summary>
public sealed class HomebrewAnalyzerPlugin : IOSAnalyzerPlugin
{
/// <inheritdoc />
public string Name => "StellaOps.Scanner.Analyzers.OS.Homebrew";
/// <inheritdoc />
public bool IsAvailable(IServiceProvider services) => services is not null;
/// <inheritdoc />
public IOSPackageAnalyzer CreateAnalyzer(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
var loggerFactory = services.GetRequiredService<ILoggerFactory>();
return new HomebrewPackageAnalyzer(loggerFactory.CreateLogger<HomebrewPackageAnalyzer>());
}
}

View File

@@ -0,0 +1,366 @@
using System.Collections.ObjectModel;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Analyzers.OS.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Analyzers;
using StellaOps.Scanner.Analyzers.OS.Helpers;
namespace StellaOps.Scanner.Analyzers.OS.Homebrew;
/// <summary>
/// Analyzes Homebrew Cellar directories to extract installed formula information.
/// Scans /usr/local/Cellar (Intel) and /opt/homebrew/Cellar (Apple Silicon) directories.
/// </summary>
internal sealed class HomebrewPackageAnalyzer : OsPackageAnalyzerBase
{
private static readonly IReadOnlyList<OSPackageRecord> EmptyPackages =
new ReadOnlyCollection<OSPackageRecord>(Array.Empty<OSPackageRecord>());
/// <summary>
/// Default paths to scan for Homebrew Cellar directories.
/// </summary>
private static readonly string[] CellarPaths =
[
"usr/local/Cellar", // Intel Macs
"opt/homebrew/Cellar", // Apple Silicon Macs
];
/// <summary>
/// Maximum traversal depth within Cellar to prevent runaway scanning.
/// Formula structure: Cellar/{formula}/{version}/...
/// </summary>
private const int MaxTraversalDepth = 3;
/// <summary>
/// Maximum formula size in bytes (200MB default per design spec).
/// </summary>
private const long MaxFormulaSizeBytes = 200L * 1024L * 1024L;
private readonly HomebrewReceiptParser _parser = new();
public HomebrewPackageAnalyzer(ILogger<HomebrewPackageAnalyzer> logger)
: base(logger)
{
}
public override string AnalyzerId => "homebrew";
protected override ValueTask<IReadOnlyList<OSPackageRecord>> ExecuteCoreAsync(
OSPackageAnalyzerContext context,
CancellationToken cancellationToken)
{
var records = new List<OSPackageRecord>();
var warnings = new List<string>();
foreach (var cellarRelativePath in CellarPaths)
{
var cellarPath = Path.Combine(context.RootPath, cellarRelativePath);
if (!Directory.Exists(cellarPath))
{
continue;
}
Logger.LogInformation("Scanning Homebrew Cellar at {Path}", cellarPath);
try
{
DiscoverFormulas(cellarPath, records, warnings, cancellationToken);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
Logger.LogWarning(ex, "Failed to scan Homebrew Cellar at {Path}", cellarPath);
}
}
if (records.Count == 0)
{
Logger.LogInformation("No Homebrew formulas found; skipping analyzer.");
return ValueTask.FromResult<IReadOnlyList<OSPackageRecord>>(EmptyPackages);
}
foreach (var warning in warnings)
{
Logger.LogWarning("Homebrew scan warning: {Warning}", warning);
}
Logger.LogInformation("Discovered {Count} Homebrew formulas", records.Count);
// Sort for deterministic output
records.Sort();
return ValueTask.FromResult<IReadOnlyList<OSPackageRecord>>(records);
}
private void DiscoverFormulas(
string cellarPath,
List<OSPackageRecord> records,
List<string> warnings,
CancellationToken cancellationToken)
{
// Enumerate formula directories (e.g., /usr/local/Cellar/openssl@3)
foreach (var formulaDir in EnumerateDirectoriesSafe(cellarPath))
{
cancellationToken.ThrowIfCancellationRequested();
var formulaName = Path.GetFileName(formulaDir);
if (string.IsNullOrWhiteSpace(formulaName) || formulaName.StartsWith('.'))
{
continue;
}
// Enumerate version directories (e.g., /usr/local/Cellar/openssl@3/3.1.0)
foreach (var versionDir in EnumerateDirectoriesSafe(formulaDir))
{
cancellationToken.ThrowIfCancellationRequested();
var versionName = Path.GetFileName(versionDir);
if (string.IsNullOrWhiteSpace(versionName) || versionName.StartsWith('.'))
{
continue;
}
// Check size guardrail
if (!CheckFormulaSizeGuardrail(versionDir, out var sizeWarning))
{
warnings.Add(sizeWarning!);
continue;
}
// Look for INSTALL_RECEIPT.json
var receiptPath = Path.Combine(versionDir, "INSTALL_RECEIPT.json");
if (File.Exists(receiptPath))
{
var record = ParseReceiptAndCreateRecord(receiptPath, formulaName, versionName, versionDir);
if (record is not null)
{
records.Add(record);
}
}
else
{
// Fallback: create record from directory structure
var record = CreateRecordFromDirectory(formulaName, versionName, versionDir);
if (record is not null)
{
records.Add(record);
warnings.Add($"No INSTALL_RECEIPT.json for {formulaName}@{versionName}; using directory-based discovery.");
}
}
}
}
}
private OSPackageRecord? ParseReceiptAndCreateRecord(
string receiptPath,
string formulaName,
string versionFromDir,
string versionDir)
{
var receipt = _parser.Parse(receiptPath);
if (receipt is null)
{
Logger.LogWarning("Failed to parse INSTALL_RECEIPT.json at {Path}", receiptPath);
return null;
}
// Use receipt version if available, fallback to directory name
var version = !string.IsNullOrWhiteSpace(receipt.Version) ? receipt.Version : versionFromDir;
// Build PURL per spec: pkg:brew/<tap>/<formula>@<version>?revision=<revision>
var purl = PackageUrlBuilder.BuildHomebrew(
receipt.Tap ?? "homebrew/core",
receipt.Name ?? formulaName,
version,
receipt.Revision);
var vendorMetadata = BuildVendorMetadata(receipt, versionDir);
var files = DiscoverFormulaFiles(versionDir);
return new OSPackageRecord(
AnalyzerId,
purl,
receipt.Name ?? formulaName,
version,
receipt.Architecture,
PackageEvidenceSource.HomebrewCellar,
epoch: null,
release: receipt.Revision > 0 ? receipt.Revision.ToString() : null,
sourcePackage: receipt.Tap,
license: receipt.License,
cveHints: null,
provides: null,
depends: receipt.RuntimeDependencies,
files: files,
vendorMetadata: vendorMetadata);
}
private OSPackageRecord? CreateRecordFromDirectory(
string formulaName,
string version,
string versionDir)
{
if (string.IsNullOrWhiteSpace(formulaName) || string.IsNullOrWhiteSpace(version))
{
return null;
}
var purl = PackageUrlBuilder.BuildHomebrew("homebrew/core", formulaName, version, revision: 0);
var architecture = DetectArchitectureFromPath(versionDir);
var files = DiscoverFormulaFiles(versionDir);
var vendorMetadata = new Dictionary<string, string?>(StringComparer.Ordinal)
{
["brew:discovery_method"] = "directory",
["brew:install_path"] = versionDir,
};
return new OSPackageRecord(
AnalyzerId,
purl,
formulaName,
version,
architecture,
PackageEvidenceSource.HomebrewCellar,
epoch: null,
release: null,
sourcePackage: "homebrew/core",
license: null,
cveHints: null,
provides: null,
depends: null,
files: files,
vendorMetadata: vendorMetadata);
}
private static Dictionary<string, string?> BuildVendorMetadata(HomebrewReceipt receipt, string versionDir)
{
var metadata = new Dictionary<string, string?>(StringComparer.Ordinal)
{
["brew:tap"] = receipt.Tap,
["brew:poured_from_bottle"] = receipt.PouredFromBottle.ToString().ToLowerInvariant(),
["brew:installed_as_dependency"] = receipt.InstalledAsDependency.ToString().ToLowerInvariant(),
["brew:installed_on_request"] = receipt.InstalledOnRequest.ToString().ToLowerInvariant(),
["brew:install_path"] = versionDir,
};
if (receipt.InstallTime.HasValue)
{
var installTime = DateTimeOffset.FromUnixTimeSeconds(receipt.InstallTime.Value);
metadata["brew:install_time"] = installTime.ToString("o");
}
if (!string.IsNullOrWhiteSpace(receipt.Description))
{
metadata["description"] = receipt.Description;
}
if (!string.IsNullOrWhiteSpace(receipt.Homepage))
{
metadata["homepage"] = receipt.Homepage;
}
if (!string.IsNullOrWhiteSpace(receipt.SourceUrl))
{
metadata["brew:source_url"] = receipt.SourceUrl;
}
if (!string.IsNullOrWhiteSpace(receipt.SourceChecksum))
{
metadata["brew:source_checksum"] = receipt.SourceChecksum;
}
if (!string.IsNullOrWhiteSpace(receipt.BottleChecksum))
{
metadata["brew:bottle_checksum"] = receipt.BottleChecksum;
}
return metadata;
}
private List<OSPackageFileEvidence> DiscoverFormulaFiles(string versionDir)
{
var files = new List<OSPackageFileEvidence>();
try
{
// Only discover key files to avoid excessive enumeration
// Focus on bin/, lib/, include/, share/man directories
var keyDirs = new[] { "bin", "lib", "include", "sbin" };
foreach (var keyDir in keyDirs)
{
var keyPath = Path.Combine(versionDir, keyDir);
if (!Directory.Exists(keyPath))
{
continue;
}
foreach (var file in Directory.EnumerateFiles(keyPath, "*", SearchOption.TopDirectoryOnly))
{
var relativePath = Path.GetRelativePath(versionDir, file);
files.Add(new OSPackageFileEvidence(
relativePath,
layerDigest: null,
sha256: null,
sizeBytes: null,
isConfigFile: false));
}
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Ignore file enumeration errors
}
return files;
}
private static bool CheckFormulaSizeGuardrail(string versionDir, out string? warning)
{
warning = null;
try
{
long totalSize = 0;
foreach (var file in Directory.EnumerateFiles(versionDir, "*", SearchOption.AllDirectories))
{
var info = new FileInfo(file);
totalSize += info.Length;
if (totalSize > MaxFormulaSizeBytes)
{
warning = $"Formula at {versionDir} exceeds {MaxFormulaSizeBytes / (1024 * 1024)}MB size limit; skipping.";
return false;
}
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Allow if we can't determine size
}
return true;
}
private static string DetectArchitectureFromPath(string path)
{
// /opt/homebrew is Apple Silicon (arm64)
// /usr/local is Intel (x86_64)
if (path.Contains("/opt/homebrew/", StringComparison.OrdinalIgnoreCase))
{
return "arm64";
}
return "x86_64";
}
private static IEnumerable<string> EnumerateDirectoriesSafe(string path)
{
try
{
return Directory.EnumerateDirectories(path);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return Array.Empty<string>();
}
}
}

View File

@@ -0,0 +1,237 @@
using System.Text.Json;
using System.Text.Json.Serialization;
namespace StellaOps.Scanner.Analyzers.OS.Homebrew;
/// <summary>
/// Parses Homebrew INSTALL_RECEIPT.json files to extract formula metadata.
/// </summary>
internal sealed class HomebrewReceiptParser
{
private static readonly JsonSerializerOptions SerializerOptions = new()
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true,
};
/// <summary>
/// Parses a Homebrew INSTALL_RECEIPT.json stream.
/// </summary>
public HomebrewReceipt? Parse(Stream stream, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(stream);
try
{
var rawReceipt = JsonSerializer.Deserialize<RawHomebrewReceipt>(stream, SerializerOptions);
if (rawReceipt is null || string.IsNullOrWhiteSpace(rawReceipt.Name))
{
return null;
}
return new HomebrewReceipt(
Name: rawReceipt.Name.Trim(),
Version: rawReceipt.Versions?.Stable?.Trim() ?? rawReceipt.Version?.Trim() ?? string.Empty,
Revision: rawReceipt.Revision ?? 0,
Tap: rawReceipt.TappedFrom?.Trim() ?? rawReceipt.Tap?.Trim() ?? "homebrew/core",
PouredFromBottle: rawReceipt.PouredFromBottle ?? false,
InstallTime: rawReceipt.InstallTime ?? rawReceipt.Time,
InstalledAsDependency: rawReceipt.InstalledAsDependency ?? false,
InstalledOnRequest: rawReceipt.InstalledOnRequest ?? true,
RuntimeDependencies: ExtractDependencies(rawReceipt.RuntimeDependencies),
BuildDependencies: ExtractDependencies(rawReceipt.BuildDependencies),
SourceUrl: rawReceipt.Source?.Url?.Trim(),
SourceChecksum: rawReceipt.Source?.Checksum?.Trim(),
BottleChecksum: rawReceipt.BottleChecksum?.Trim(),
Description: rawReceipt.Description?.Trim(),
Homepage: rawReceipt.Homepage?.Trim(),
License: rawReceipt.License?.Trim(),
Architecture: NormalizeArchitecture(rawReceipt.TabJson?.Arch ?? rawReceipt.Arch),
TabPath: rawReceipt.TabPath?.Trim());
}
catch (JsonException)
{
return null;
}
}
/// <summary>
/// Parses a Homebrew INSTALL_RECEIPT.json from a file path.
/// </summary>
public HomebrewReceipt? Parse(string receiptPath, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(receiptPath);
if (!File.Exists(receiptPath))
{
return null;
}
using var stream = File.OpenRead(receiptPath);
return Parse(stream, cancellationToken);
}
private static IReadOnlyList<string> ExtractDependencies(RawDependency[]? dependencies)
{
if (dependencies is null or { Length: 0 })
{
return Array.Empty<string>();
}
var result = new List<string>(dependencies.Length);
foreach (var dep in dependencies)
{
var name = dep.FullName?.Trim() ?? dep.Name?.Trim();
if (!string.IsNullOrWhiteSpace(name))
{
result.Add(name);
}
}
result.Sort(StringComparer.Ordinal);
return result;
}
private static string NormalizeArchitecture(string? arch)
{
if (string.IsNullOrWhiteSpace(arch))
{
return "x86_64"; // Default for Intel Macs
}
var normalized = arch.Trim().ToLowerInvariant();
return normalized switch
{
"arm64" or "aarch64" => "arm64",
"x86_64" or "amd64" or "x64" => "x86_64",
_ => normalized,
};
}
// Raw JSON models for deserialization
private sealed class RawHomebrewReceipt
{
[JsonPropertyName("name")]
public string? Name { get; set; }
[JsonPropertyName("version")]
public string? Version { get; set; }
[JsonPropertyName("versions")]
public RawVersions? Versions { get; set; }
[JsonPropertyName("revision")]
public int? Revision { get; set; }
[JsonPropertyName("tap")]
public string? Tap { get; set; }
[JsonPropertyName("tapped_from")]
public string? TappedFrom { get; set; }
[JsonPropertyName("poured_from_bottle")]
public bool? PouredFromBottle { get; set; }
[JsonPropertyName("time")]
public long? Time { get; set; }
[JsonPropertyName("install_time")]
public long? InstallTime { get; set; }
[JsonPropertyName("installed_as_dependency")]
public bool? InstalledAsDependency { get; set; }
[JsonPropertyName("installed_on_request")]
public bool? InstalledOnRequest { get; set; }
[JsonPropertyName("runtime_dependencies")]
public RawDependency[]? RuntimeDependencies { get; set; }
[JsonPropertyName("build_dependencies")]
public RawDependency[]? BuildDependencies { get; set; }
[JsonPropertyName("source")]
public RawSource? Source { get; set; }
[JsonPropertyName("bottle_checksum")]
public string? BottleChecksum { get; set; }
[JsonPropertyName("desc")]
public string? Description { get; set; }
[JsonPropertyName("homepage")]
public string? Homepage { get; set; }
[JsonPropertyName("license")]
public string? License { get; set; }
[JsonPropertyName("arch")]
public string? Arch { get; set; }
[JsonPropertyName("HOMEBREW_INSTALL_PATH")]
public string? TabPath { get; set; }
[JsonPropertyName("tab")]
public RawTab? TabJson { get; set; }
}
private sealed class RawVersions
{
[JsonPropertyName("stable")]
public string? Stable { get; set; }
[JsonPropertyName("head")]
public string? Head { get; set; }
}
private sealed class RawDependency
{
[JsonPropertyName("full_name")]
public string? FullName { get; set; }
[JsonPropertyName("name")]
public string? Name { get; set; }
[JsonPropertyName("version")]
public string? Version { get; set; }
}
private sealed class RawSource
{
[JsonPropertyName("url")]
public string? Url { get; set; }
[JsonPropertyName("checksum")]
public string? Checksum { get; set; }
}
private sealed class RawTab
{
[JsonPropertyName("arch")]
public string? Arch { get; set; }
}
}
/// <summary>
/// Represents parsed Homebrew formula receipt metadata.
/// </summary>
internal sealed record HomebrewReceipt(
string Name,
string Version,
int Revision,
string Tap,
bool PouredFromBottle,
long? InstallTime,
bool InstalledAsDependency,
bool InstalledOnRequest,
IReadOnlyList<string> RuntimeDependencies,
IReadOnlyList<string> BuildDependencies,
string? SourceUrl,
string? SourceChecksum,
string? BottleChecksum,
string? Description,
string? Homepage,
string? License,
string Architecture,
string? TabPath);

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Scanner.Analyzers.OS.Homebrew.Tests")]

View File

@@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Scanner.Analyzers.OS\StellaOps.Scanner.Analyzers.OS.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,230 @@
using Claunia.PropertyList;
namespace StellaOps.Scanner.Analyzers.OS.MacOsBundle;
/// <summary>
/// Parses macOS entitlements from embedded plist files or code signature data.
/// </summary>
internal sealed class EntitlementsParser
{
/// <summary>
/// Well-known entitlement categories for capability classification.
/// </summary>
private static readonly Dictionary<string, string> EntitlementCategories = new(StringComparer.OrdinalIgnoreCase)
{
// Network
["com.apple.security.network.client"] = "network",
["com.apple.security.network.server"] = "network",
// File System
["com.apple.security.files.user-selected.read-only"] = "filesystem",
["com.apple.security.files.user-selected.read-write"] = "filesystem",
["com.apple.security.files.downloads.read-only"] = "filesystem",
["com.apple.security.files.downloads.read-write"] = "filesystem",
["com.apple.security.files.all"] = "filesystem",
// Hardware
["com.apple.security.device.camera"] = "camera",
["com.apple.security.device.microphone"] = "microphone",
["com.apple.security.device.usb"] = "hardware",
["com.apple.security.device.bluetooth"] = "hardware",
["com.apple.security.device.serial"] = "hardware",
// Privacy
["com.apple.security.personal-information.addressbook"] = "privacy",
["com.apple.security.personal-information.calendars"] = "privacy",
["com.apple.security.personal-information.location"] = "privacy",
["com.apple.security.personal-information.photos-library"] = "privacy",
// System
["com.apple.security.automation.apple-events"] = "automation",
["com.apple.security.scripting-targets"] = "automation",
["com.apple.security.cs.allow-jit"] = "code-execution",
["com.apple.security.cs.allow-unsigned-executable-memory"] = "code-execution",
["com.apple.security.cs.disable-library-validation"] = "code-execution",
["com.apple.security.cs.allow-dyld-environment-variables"] = "code-execution",
["com.apple.security.get-task-allow"] = "debugging",
// App Sandbox
["com.apple.security.app-sandbox"] = "sandbox",
["com.apple.security.inherit"] = "sandbox",
};
/// <summary>
/// High-risk entitlements that warrant policy attention.
/// </summary>
private static readonly HashSet<string> HighRiskEntitlements = new(StringComparer.OrdinalIgnoreCase)
{
"com.apple.security.device.camera",
"com.apple.security.device.microphone",
"com.apple.security.cs.allow-unsigned-executable-memory",
"com.apple.security.cs.disable-library-validation",
"com.apple.security.get-task-allow",
"com.apple.security.files.all",
"com.apple.security.automation.apple-events",
};
/// <summary>
/// Parses entitlements from a plist file (e.g., embedded entitlements or xcent file).
/// </summary>
public BundleEntitlements Parse(string entitlementsPath, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(entitlementsPath);
if (!File.Exists(entitlementsPath))
{
return BundleEntitlements.Empty;
}
try
{
using var stream = File.OpenRead(entitlementsPath);
return Parse(stream, cancellationToken);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or FormatException)
{
return BundleEntitlements.Empty;
}
}
/// <summary>
/// Parses entitlements from a stream.
/// </summary>
public BundleEntitlements Parse(Stream stream, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(stream);
try
{
var plist = PropertyListParser.Parse(stream);
if (plist is not NSDictionary root)
{
return BundleEntitlements.Empty;
}
var entitlements = new Dictionary<string, object?>(StringComparer.Ordinal);
var categories = new HashSet<string>(StringComparer.Ordinal);
var highRisk = new List<string>();
foreach (var kvp in root)
{
var key = kvp.Key;
var value = ConvertValue(kvp.Value);
entitlements[key] = value;
// Classify the entitlement
if (EntitlementCategories.TryGetValue(key, out var category))
{
categories.Add(category);
}
// Check if high risk
if (HighRiskEntitlements.Contains(key) && IsTrueValue(value))
{
highRisk.Add(key);
}
}
return new BundleEntitlements(
Entitlements: entitlements,
Categories: categories.OrderBy(c => c, StringComparer.Ordinal).ToList(),
HighRiskEntitlements: highRisk.OrderBy(e => e, StringComparer.Ordinal).ToList(),
IsSandboxed: entitlements.TryGetValue("com.apple.security.app-sandbox", out var sandbox) && IsTrueValue(sandbox),
HasHardenedRuntime: entitlements.TryGetValue("com.apple.security.cs.allow-unsigned-executable-memory", out var hr) && !IsTrueValue(hr));
}
catch (Exception ex) when (ex is FormatException or InvalidOperationException or ArgumentException)
{
return BundleEntitlements.Empty;
}
}
/// <summary>
/// Discovers entitlements file within an app bundle.
/// </summary>
public string? FindEntitlementsFile(string bundlePath)
{
if (string.IsNullOrWhiteSpace(bundlePath))
{
return null;
}
// Look for xcent files first (highest priority for actual entitlements)
var codeSignPath = Path.Combine(bundlePath, "Contents", "_CodeSignature");
if (Directory.Exists(codeSignPath))
{
try
{
var xcentFiles = Directory.GetFiles(codeSignPath, "*.xcent");
if (xcentFiles.Length > 0)
{
return xcentFiles[0];
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Ignore
}
}
// Check other common entitlements locations
var candidates = new[]
{
Path.Combine(bundlePath, "Contents", "embedded.provisionprofile"),
Path.Combine(bundlePath, "embedded.mobileprovision"),
};
foreach (var candidate in candidates)
{
if (File.Exists(candidate))
{
return candidate;
}
}
return null;
}
private static object? ConvertValue(NSObject? obj)
{
return obj switch
{
NSString s => s.Content,
NSNumber n => n.ToObject(),
NSData d => Convert.ToBase64String(d.Bytes),
NSArray a => a.Select(ConvertValue).ToArray(),
NSDictionary d => d.ToDictionary(kvp => kvp.Key, kvp => ConvertValue(kvp.Value)),
_ => obj?.ToString()
};
}
private static bool IsTrueValue(object? value)
{
return value switch
{
bool b => b,
int i => i != 0,
string s => s.Equals("true", StringComparison.OrdinalIgnoreCase) || s == "1",
_ => false
};
}
}
/// <summary>
/// Represents parsed macOS bundle entitlements.
/// </summary>
internal sealed record BundleEntitlements(
IReadOnlyDictionary<string, object?> Entitlements,
IReadOnlyList<string> Categories,
IReadOnlyList<string> HighRiskEntitlements,
bool IsSandboxed,
bool HasHardenedRuntime)
{
public static BundleEntitlements Empty { get; } = new(
new Dictionary<string, object?>(),
Array.Empty<string>(),
Array.Empty<string>(),
IsSandboxed: false,
HasHardenedRuntime: false);
public bool HasEntitlement(string key) => Entitlements.ContainsKey(key);
}

View File

@@ -0,0 +1,132 @@
using Claunia.PropertyList;
namespace StellaOps.Scanner.Analyzers.OS.MacOsBundle;
/// <summary>
/// Parses macOS application bundle Info.plist files.
/// </summary>
internal sealed class InfoPlistParser
{
/// <summary>
/// Parses an Info.plist file from the given path.
/// </summary>
public BundleInfo? Parse(string plistPath, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(plistPath);
if (!File.Exists(plistPath))
{
return null;
}
try
{
using var stream = File.OpenRead(plistPath);
return Parse(stream, plistPath, cancellationToken);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or FormatException)
{
return null;
}
}
/// <summary>
/// Parses an Info.plist from a stream.
/// </summary>
public BundleInfo? Parse(Stream stream, string? sourcePath = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(stream);
try
{
var plist = PropertyListParser.Parse(stream);
if (plist is not NSDictionary root)
{
return null;
}
var bundleId = GetString(root, "CFBundleIdentifier");
if (string.IsNullOrWhiteSpace(bundleId))
{
return null;
}
return new BundleInfo(
BundleIdentifier: bundleId.Trim(),
BundleName: GetString(root, "CFBundleName")?.Trim() ?? ExtractNameFromBundleId(bundleId),
BundleDisplayName: GetString(root, "CFBundleDisplayName")?.Trim(),
Version: GetString(root, "CFBundleVersion")?.Trim() ?? "0",
ShortVersion: GetString(root, "CFBundleShortVersionString")?.Trim(),
MinimumSystemVersion: GetString(root, "LSMinimumSystemVersion")?.Trim(),
Executable: GetString(root, "CFBundleExecutable")?.Trim(),
BundlePackageType: GetString(root, "CFBundlePackageType")?.Trim() ?? "APPL",
SupportedPlatforms: GetStringArray(root, "CFBundleSupportedPlatforms"),
RequiredCapabilities: GetStringArray(root, "UIRequiredDeviceCapabilities"),
SourcePath: sourcePath);
}
catch (Exception ex) when (ex is FormatException or InvalidOperationException or ArgumentException)
{
return null;
}
}
private static string? GetString(NSDictionary dict, string key)
{
if (dict.TryGetValue(key, out var value) && value is NSString nsString)
{
return nsString.Content;
}
return null;
}
private static IReadOnlyList<string> GetStringArray(NSDictionary dict, string key)
{
if (!dict.TryGetValue(key, out var value))
{
return Array.Empty<string>();
}
var result = new List<string>();
if (value is NSArray array)
{
foreach (var item in array)
{
if (item is NSString nsString && !string.IsNullOrWhiteSpace(nsString.Content))
{
result.Add(nsString.Content.Trim());
}
}
}
else if (value is NSString singleString && !string.IsNullOrWhiteSpace(singleString.Content))
{
result.Add(singleString.Content.Trim());
}
result.Sort(StringComparer.Ordinal);
return result;
}
private static string ExtractNameFromBundleId(string bundleId)
{
var parts = bundleId.Split('.', StringSplitOptions.RemoveEmptyEntries);
return parts.Length > 0 ? parts[^1] : bundleId;
}
}
/// <summary>
/// Represents parsed macOS bundle Info.plist metadata.
/// </summary>
internal sealed record BundleInfo(
string BundleIdentifier,
string BundleName,
string? BundleDisplayName,
string Version,
string? ShortVersion,
string? MinimumSystemVersion,
string? Executable,
string BundlePackageType,
IReadOnlyList<string> SupportedPlatforms,
IReadOnlyList<string> RequiredCapabilities,
string? SourcePath);

View File

@@ -0,0 +1,368 @@
using System.Collections.ObjectModel;
using System.Security.Cryptography;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Analyzers.OS.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Analyzers;
using StellaOps.Scanner.Analyzers.OS.Helpers;
namespace StellaOps.Scanner.Analyzers.OS.MacOsBundle;
/// <summary>
/// Analyzes macOS application bundles (.app) to extract metadata and capability information.
/// Scans /Applications, /System/Applications, and user application directories.
/// </summary>
internal sealed class MacOsBundleAnalyzer : OsPackageAnalyzerBase
{
private static readonly IReadOnlyList<OSPackageRecord> EmptyPackages =
new ReadOnlyCollection<OSPackageRecord>(Array.Empty<OSPackageRecord>());
/// <summary>
/// Standard paths to scan for application bundles.
/// </summary>
private static readonly string[] ApplicationPaths =
[
"Applications",
"System/Applications",
"Library/Application Support",
];
/// <summary>
/// Maximum traversal depth within application directories.
/// </summary>
private const int MaxTraversalDepth = 3;
/// <summary>
/// Maximum bundle size to process (500MB).
/// </summary>
private const long MaxBundleSizeBytes = 500L * 1024L * 1024L;
private readonly InfoPlistParser _infoPlistParser = new();
private readonly EntitlementsParser _entitlementsParser = new();
public MacOsBundleAnalyzer(ILogger<MacOsBundleAnalyzer> logger)
: base(logger)
{
}
public override string AnalyzerId => "macos-bundle";
protected override ValueTask<IReadOnlyList<OSPackageRecord>> ExecuteCoreAsync(
OSPackageAnalyzerContext context,
CancellationToken cancellationToken)
{
var records = new List<OSPackageRecord>();
var warnings = new List<string>();
// Scan standard application paths
foreach (var appPath in ApplicationPaths)
{
var fullPath = Path.Combine(context.RootPath, appPath);
if (!Directory.Exists(fullPath))
{
continue;
}
Logger.LogInformation("Scanning for application bundles in {Path}", fullPath);
try
{
DiscoverBundles(fullPath, records, warnings, 0, cancellationToken);
}
catch (Exception ex) when (ex is not OperationCanceledException)
{
Logger.LogWarning(ex, "Failed to scan application path {Path}", fullPath);
}
}
// Scan user directories
var usersPath = Path.Combine(context.RootPath, "Users");
if (Directory.Exists(usersPath))
{
try
{
foreach (var userDir in Directory.EnumerateDirectories(usersPath))
{
cancellationToken.ThrowIfCancellationRequested();
var userAppsPath = Path.Combine(userDir, "Applications");
if (Directory.Exists(userAppsPath))
{
DiscoverBundles(userAppsPath, records, warnings, 0, cancellationToken);
}
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
Logger.LogDebug(ex, "Could not enumerate user directories");
}
}
if (records.Count == 0)
{
Logger.LogInformation("No application bundles found; skipping analyzer.");
return ValueTask.FromResult<IReadOnlyList<OSPackageRecord>>(EmptyPackages);
}
foreach (var warning in warnings.Take(10)) // Limit warning output
{
Logger.LogWarning("Bundle scan warning: {Warning}", warning);
}
Logger.LogInformation("Discovered {Count} application bundles", records.Count);
// Sort for deterministic output
records.Sort();
return ValueTask.FromResult<IReadOnlyList<OSPackageRecord>>(records);
}
private void DiscoverBundles(
string searchPath,
List<OSPackageRecord> records,
List<string> warnings,
int depth,
CancellationToken cancellationToken)
{
if (depth > MaxTraversalDepth)
{
return;
}
IEnumerable<string> entries;
try
{
entries = Directory.EnumerateDirectories(searchPath);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return;
}
foreach (var entry in entries)
{
cancellationToken.ThrowIfCancellationRequested();
var name = Path.GetFileName(entry);
if (string.IsNullOrWhiteSpace(name) || name.StartsWith('.'))
{
continue;
}
// Check if this is an app bundle
if (name.EndsWith(".app", StringComparison.OrdinalIgnoreCase))
{
var record = AnalyzeBundle(entry, warnings, cancellationToken);
if (record is not null)
{
records.Add(record);
}
}
else
{
// Recurse into subdirectories (e.g., for nested apps)
DiscoverBundles(entry, records, warnings, depth + 1, cancellationToken);
}
}
}
private OSPackageRecord? AnalyzeBundle(
string bundlePath,
List<string> warnings,
CancellationToken cancellationToken)
{
// Find and parse Info.plist
var infoPlistPath = Path.Combine(bundlePath, "Contents", "Info.plist");
if (!File.Exists(infoPlistPath))
{
// Try iOS-style location
infoPlistPath = Path.Combine(bundlePath, "Info.plist");
}
if (!File.Exists(infoPlistPath))
{
warnings.Add($"No Info.plist found in {bundlePath}");
return null;
}
var bundleInfo = _infoPlistParser.Parse(infoPlistPath, cancellationToken);
if (bundleInfo is null)
{
warnings.Add($"Failed to parse Info.plist in {bundlePath}");
return null;
}
// Parse entitlements if available
var entitlementsPath = _entitlementsParser.FindEntitlementsFile(bundlePath);
var entitlements = entitlementsPath is not null
? _entitlementsParser.Parse(entitlementsPath, cancellationToken)
: BundleEntitlements.Empty;
// Compute CodeResources hash if available
var codeResourcesHash = ComputeCodeResourcesHash(bundlePath);
// Determine version (prefer short version, fallback to bundle version)
var version = !string.IsNullOrWhiteSpace(bundleInfo.ShortVersion)
? bundleInfo.ShortVersion
: bundleInfo.Version;
// Build PURL
var purl = PackageUrlBuilder.BuildMacOsBundle(bundleInfo.BundleIdentifier, version);
// Build vendor metadata
var vendorMetadata = BuildVendorMetadata(bundleInfo, entitlements, codeResourcesHash, bundlePath);
// Discover key files
var files = DiscoverBundleFiles(bundlePath, bundleInfo);
// Extract display name
var displayName = bundleInfo.BundleDisplayName ?? bundleInfo.BundleName;
return new OSPackageRecord(
AnalyzerId,
purl,
displayName,
version,
DetermineArchitecture(bundlePath),
PackageEvidenceSource.MacOsBundle,
epoch: null,
release: bundleInfo.Version != version ? bundleInfo.Version : null,
sourcePackage: ExtractVendorFromBundleId(bundleInfo.BundleIdentifier),
license: null,
cveHints: null,
provides: null,
depends: null,
files: files,
vendorMetadata: vendorMetadata);
}
private static Dictionary<string, string?> BuildVendorMetadata(
BundleInfo bundleInfo,
BundleEntitlements entitlements,
string? codeResourcesHash,
string bundlePath)
{
var metadata = new Dictionary<string, string?>(StringComparer.Ordinal)
{
["macos:bundle_id"] = bundleInfo.BundleIdentifier,
["macos:bundle_type"] = bundleInfo.BundlePackageType,
["macos:bundle_path"] = bundlePath,
};
if (!string.IsNullOrWhiteSpace(bundleInfo.MinimumSystemVersion))
{
metadata["macos:min_os_version"] = bundleInfo.MinimumSystemVersion;
}
if (!string.IsNullOrWhiteSpace(bundleInfo.Executable))
{
metadata["macos:executable"] = bundleInfo.Executable;
}
if (bundleInfo.SupportedPlatforms.Count > 0)
{
metadata["macos:platforms"] = string.Join(",", bundleInfo.SupportedPlatforms);
}
// Entitlements metadata
metadata["macos:sandboxed"] = entitlements.IsSandboxed.ToString().ToLowerInvariant();
metadata["macos:hardened_runtime"] = entitlements.HasHardenedRuntime.ToString().ToLowerInvariant();
if (entitlements.Categories.Count > 0)
{
metadata["macos:capability_categories"] = string.Join(",", entitlements.Categories);
}
if (entitlements.HighRiskEntitlements.Count > 0)
{
metadata["macos:high_risk_entitlements"] = string.Join(",", entitlements.HighRiskEntitlements);
}
if (!string.IsNullOrWhiteSpace(codeResourcesHash))
{
metadata["macos:code_resources_hash"] = codeResourcesHash;
}
return metadata;
}
private static string? ComputeCodeResourcesHash(string bundlePath)
{
var codeResourcesPath = Path.Combine(bundlePath, "Contents", "_CodeSignature", "CodeResources");
if (!File.Exists(codeResourcesPath))
{
return null;
}
try
{
using var stream = File.OpenRead(codeResourcesPath);
var hash = SHA256.HashData(stream);
return $"sha256:{Convert.ToHexStringLower(hash)}";
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return null;
}
}
private static List<OSPackageFileEvidence> DiscoverBundleFiles(string bundlePath, BundleInfo bundleInfo)
{
var files = new List<OSPackageFileEvidence>();
// Add key bundle files
var contentsPath = Path.Combine(bundlePath, "Contents");
// Executable
if (!string.IsNullOrWhiteSpace(bundleInfo.Executable))
{
var execPath = Path.Combine(contentsPath, "MacOS", bundleInfo.Executable);
if (File.Exists(execPath))
{
files.Add(new OSPackageFileEvidence(
$"Contents/MacOS/{bundleInfo.Executable}",
layerDigest: null,
sha256: null,
sizeBytes: null,
isConfigFile: false));
}
}
// Info.plist
var infoPlistRelative = "Contents/Info.plist";
if (File.Exists(Path.Combine(bundlePath, infoPlistRelative)))
{
files.Add(new OSPackageFileEvidence(
infoPlistRelative,
layerDigest: null,
sha256: null,
sizeBytes: null,
isConfigFile: true));
}
return files;
}
private static string DetermineArchitecture(string bundlePath)
{
// Check for universal binary indicators
var macosPath = Path.Combine(bundlePath, "Contents", "MacOS");
if (Directory.Exists(macosPath))
{
// Look for architecture-specific subdirectories or lipo info
// For now, default to universal
return "universal";
}
return "universal";
}
private static string? ExtractVendorFromBundleId(string bundleId)
{
var parts = bundleId.Split('.', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 2)
{
return parts[1];
}
return null;
}
}

View File

@@ -0,0 +1,26 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Analyzers.OS.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Plugin;
namespace StellaOps.Scanner.Analyzers.OS.MacOsBundle;
/// <summary>
/// Plugin that registers the macOS bundle analyzer for application bundle discovery.
/// </summary>
public sealed class MacOsBundleAnalyzerPlugin : IOSAnalyzerPlugin
{
/// <inheritdoc />
public string Name => "StellaOps.Scanner.Analyzers.OS.MacOsBundle";
/// <inheritdoc />
public bool IsAvailable(IServiceProvider services) => services is not null;
/// <inheritdoc />
public IOSPackageAnalyzer CreateAnalyzer(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
var loggerFactory = services.GetRequiredService<ILoggerFactory>();
return new MacOsBundleAnalyzer(loggerFactory.CreateLogger<MacOsBundleAnalyzer>());
}
}

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Scanner.Analyzers.OS.MacOsBundle.Tests")]

View File

@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="plist-cil" Version="2.2.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Scanner.Analyzers.OS\StellaOps.Scanner.Analyzers.OS.csproj" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,198 @@
using System.Buffers.Binary;
namespace StellaOps.Scanner.Analyzers.OS.Pkgutil;
/// <summary>
/// Parses macOS BOM (Bill of Materials) files to enumerate installed files.
/// BOM files are used by pkgutil to track which files were installed by a package.
/// </summary>
internal sealed class BomParser
{
/// <summary>
/// BOM file magic header: "BOMStore"
/// </summary>
private static ReadOnlySpan<byte> BomMagic => "BOMStore"u8;
/// <summary>
/// Extracts the list of installed file paths from a BOM file.
/// </summary>
/// <remarks>
/// BOM files have a complex binary format. This implementation extracts
/// the file paths from the BOM tree structure, focusing on the Paths tree.
/// </remarks>
public IReadOnlyList<BomFileEntry> Parse(string bomPath, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(bomPath);
if (!File.Exists(bomPath))
{
return Array.Empty<BomFileEntry>();
}
try
{
using var stream = File.OpenRead(bomPath);
return Parse(stream, cancellationToken);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
return Array.Empty<BomFileEntry>();
}
}
/// <summary>
/// Extracts file paths from a BOM stream.
/// </summary>
public IReadOnlyList<BomFileEntry> Parse(Stream stream, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(stream);
var results = new List<BomFileEntry>();
try
{
// Read header (512+ bytes)
var header = new byte[512];
if (stream.Read(header, 0, 512) < 512)
{
return results;
}
// Verify magic
if (!header.AsSpan(0, 8).SequenceEqual(BomMagic))
{
return results;
}
// BOM format is complex - we'll do a simplified extraction
// by scanning for null-terminated strings that look like paths
stream.Position = 0;
using var reader = new BinaryReader(stream);
var content = reader.ReadBytes((int)Math.Min(stream.Length, 10 * 1024 * 1024)); // Max 10MB
var paths = ExtractPaths(content, cancellationToken);
foreach (var path in paths)
{
results.Add(new BomFileEntry(path, IsDirectory: path.EndsWith('/')));
}
}
catch (Exception ex) when (ex is IOException or EndOfStreamException)
{
// Return partial results
}
return results;
}
/// <summary>
/// Finds the corresponding BOM file for a receipt plist.
/// </summary>
public string? FindBomForReceipt(string plistPath)
{
if (string.IsNullOrWhiteSpace(plistPath))
{
return null;
}
// BOM files are named with same base name as plist
// e.g., com.apple.pkg.Safari.plist -> com.apple.pkg.Safari.bom
var directory = Path.GetDirectoryName(plistPath);
var baseName = Path.GetFileNameWithoutExtension(plistPath);
if (string.IsNullOrEmpty(directory) || string.IsNullOrEmpty(baseName))
{
return null;
}
var bomPath = Path.Combine(directory, baseName + ".bom");
return File.Exists(bomPath) ? bomPath : null;
}
private static IEnumerable<string> ExtractPaths(byte[] content, CancellationToken cancellationToken)
{
var paths = new HashSet<string>(StringComparer.Ordinal);
// Scan for null-terminated strings that look like Unix paths
int start = -1;
for (int i = 0; i < content.Length; i++)
{
cancellationToken.ThrowIfCancellationRequested();
byte b = content[i];
if (start == -1)
{
// Look for path start indicators
if (b == '/' || b == '.')
{
start = i;
}
}
else
{
if (b == 0) // Null terminator
{
var length = i - start;
if (length > 1 && length < 4096)
{
var potential = System.Text.Encoding.UTF8.GetString(content, start, length);
if (IsValidPath(potential))
{
paths.Add(potential);
}
}
start = -1;
}
else if (!IsValidPathChar(b))
{
start = -1;
}
}
}
return paths.OrderBy(p => p, StringComparer.Ordinal);
}
private static bool IsValidPath(string path)
{
if (string.IsNullOrWhiteSpace(path))
{
return false;
}
// Must start with / or .
if (!path.StartsWith('/') && !path.StartsWith('.'))
{
return false;
}
// Must not contain control characters or obviously invalid sequences
foreach (char c in path)
{
if (c < 32 && c != '\t')
{
return false;
}
}
// Filter out common false positives
if (path.Contains("//") || path.EndsWith("/.") || path.Contains("/../"))
{
return false;
}
return true;
}
private static bool IsValidPathChar(byte b)
{
// Allow printable ASCII and common path characters
return b >= 32 && b < 127;
}
}
/// <summary>
/// Represents a file entry from a BOM file.
/// </summary>
internal sealed record BomFileEntry(string Path, bool IsDirectory);

View File

@@ -0,0 +1,26 @@
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Analyzers.OS.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Plugin;
namespace StellaOps.Scanner.Analyzers.OS.Pkgutil;
/// <summary>
/// Plugin that registers the pkgutil package analyzer for macOS receipt discovery.
/// </summary>
public sealed class PkgutilAnalyzerPlugin : IOSAnalyzerPlugin
{
/// <inheritdoc />
public string Name => "StellaOps.Scanner.Analyzers.OS.Pkgutil";
/// <inheritdoc />
public bool IsAvailable(IServiceProvider services) => services is not null;
/// <inheritdoc />
public IOSPackageAnalyzer CreateAnalyzer(IServiceProvider services)
{
ArgumentNullException.ThrowIfNull(services);
var loggerFactory = services.GetRequiredService<ILoggerFactory>();
return new PkgutilPackageAnalyzer(loggerFactory.CreateLogger<PkgutilPackageAnalyzer>());
}
}

View File

@@ -0,0 +1,229 @@
using System.Collections.ObjectModel;
using Microsoft.Extensions.Logging;
using StellaOps.Scanner.Analyzers.OS.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Analyzers;
using StellaOps.Scanner.Analyzers.OS.Helpers;
namespace StellaOps.Scanner.Analyzers.OS.Pkgutil;
/// <summary>
/// Analyzes macOS pkgutil receipts to extract installed package information.
/// Parses receipt plists from /var/db/receipts/ and optionally enumerates
/// installed files from corresponding BOM files.
/// </summary>
internal sealed class PkgutilPackageAnalyzer : OsPackageAnalyzerBase
{
private static readonly IReadOnlyList<OSPackageRecord> EmptyPackages =
new ReadOnlyCollection<OSPackageRecord>(Array.Empty<OSPackageRecord>());
private readonly PkgutilReceiptParser _receiptParser = new();
private readonly BomParser _bomParser = new();
/// <summary>
/// Maximum number of files to enumerate from BOM per package.
/// </summary>
private const int MaxFilesPerPackage = 1000;
public PkgutilPackageAnalyzer(ILogger<PkgutilPackageAnalyzer> logger)
: base(logger)
{
}
public override string AnalyzerId => "pkgutil";
protected override ValueTask<IReadOnlyList<OSPackageRecord>> ExecuteCoreAsync(
OSPackageAnalyzerContext context,
CancellationToken cancellationToken)
{
var receiptsPath = Path.Combine(context.RootPath, "var", "db", "receipts");
if (!Directory.Exists(receiptsPath))
{
Logger.LogInformation("pkgutil receipts directory not found at {Path}; skipping analyzer.", receiptsPath);
return ValueTask.FromResult<IReadOnlyList<OSPackageRecord>>(EmptyPackages);
}
var receipts = _receiptParser.DiscoverReceipts(context.RootPath, cancellationToken);
if (receipts.Count == 0)
{
Logger.LogInformation("No pkgutil receipts found; skipping analyzer.");
return ValueTask.FromResult<IReadOnlyList<OSPackageRecord>>(EmptyPackages);
}
Logger.LogInformation("Discovered {Count} pkgutil receipts", receipts.Count);
var records = new List<OSPackageRecord>(receipts.Count);
foreach (var receipt in receipts)
{
cancellationToken.ThrowIfCancellationRequested();
var record = CreateRecordFromReceipt(receipt, cancellationToken);
if (record is not null)
{
records.Add(record);
}
}
Logger.LogInformation("Created {Count} package records from pkgutil receipts", records.Count);
// Sort for deterministic output
records.Sort();
return ValueTask.FromResult<IReadOnlyList<OSPackageRecord>>(records);
}
private OSPackageRecord? CreateRecordFromReceipt(
PkgutilReceipt receipt,
CancellationToken cancellationToken)
{
if (string.IsNullOrWhiteSpace(receipt.Identifier) ||
string.IsNullOrWhiteSpace(receipt.Version))
{
return null;
}
// Build PURL
var purl = PackageUrlBuilder.BuildPkgutil(receipt.Identifier, receipt.Version);
// Determine architecture from identifier or install path heuristics
var architecture = DetectArchitecture(receipt);
// Extract files from BOM if available
var files = new List<OSPackageFileEvidence>();
if (!string.IsNullOrWhiteSpace(receipt.SourcePath))
{
var bomPath = _bomParser.FindBomForReceipt(receipt.SourcePath);
if (bomPath is not null)
{
var bomEntries = _bomParser.Parse(bomPath, cancellationToken);
var count = 0;
foreach (var entry in bomEntries)
{
if (count >= MaxFilesPerPackage)
{
break;
}
if (!entry.IsDirectory)
{
files.Add(new OSPackageFileEvidence(
entry.Path,
layerDigest: null,
sha256: null,
sizeBytes: null,
isConfigFile: IsConfigPath(entry.Path)));
count++;
}
}
}
}
// Build vendor metadata
var vendorMetadata = new Dictionary<string, string?>(StringComparer.Ordinal)
{
["pkgutil:identifier"] = receipt.Identifier,
["pkgutil:volume"] = receipt.VolumePath,
};
if (receipt.InstallDate.HasValue)
{
vendorMetadata["pkgutil:install_date"] = receipt.InstallDate.Value.ToString("o");
}
if (!string.IsNullOrWhiteSpace(receipt.InstallPrefixPath))
{
vendorMetadata["pkgutil:install_prefix"] = receipt.InstallPrefixPath;
}
if (!string.IsNullOrWhiteSpace(receipt.InstallProcessName))
{
vendorMetadata["pkgutil:installer"] = receipt.InstallProcessName;
}
// Extract package name from identifier (last component typically)
var name = ExtractNameFromIdentifier(receipt.Identifier);
return new OSPackageRecord(
AnalyzerId,
purl,
name,
receipt.Version,
architecture,
PackageEvidenceSource.PkgutilReceipt,
epoch: null,
release: null,
sourcePackage: ExtractVendorFromIdentifier(receipt.Identifier),
license: null,
cveHints: null,
provides: null,
depends: null,
files: files,
vendorMetadata: vendorMetadata);
}
private static string ExtractNameFromIdentifier(string identifier)
{
// Identifier format is typically: com.vendor.product or com.apple.pkg.Safari
var parts = identifier.Split('.', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length == 0)
{
return identifier;
}
// Return the last meaningful part
var last = parts[^1];
// Skip common suffixes
if (parts.Length > 1 &&
(last.Equals("pkg", StringComparison.OrdinalIgnoreCase) ||
last.Equals("app", StringComparison.OrdinalIgnoreCase)))
{
return parts[^2];
}
return last;
}
private static string? ExtractVendorFromIdentifier(string identifier)
{
// Extract vendor from identifier (e.g., "com.apple.pkg.Safari" -> "apple")
var parts = identifier.Split('.', StringSplitOptions.RemoveEmptyEntries);
if (parts.Length >= 2)
{
return parts[1];
}
return null;
}
private static string DetectArchitecture(PkgutilReceipt receipt)
{
// Check install path for architecture hints
var prefix = receipt.InstallPrefixPath ?? receipt.VolumePath;
if (!string.IsNullOrWhiteSpace(prefix))
{
if (prefix.Contains("/arm64/", StringComparison.OrdinalIgnoreCase) ||
prefix.Contains("/aarch64/", StringComparison.OrdinalIgnoreCase))
{
return "arm64";
}
if (prefix.Contains("/x86_64/", StringComparison.OrdinalIgnoreCase) ||
prefix.Contains("/amd64/", StringComparison.OrdinalIgnoreCase))
{
return "x86_64";
}
}
// Default to universal (noarch) for macOS packages
return "universal";
}
private static bool IsConfigPath(string path)
{
// Common macOS configuration paths
return path.Contains("/Preferences/", StringComparison.OrdinalIgnoreCase) ||
path.Contains("/etc/", StringComparison.OrdinalIgnoreCase) ||
path.EndsWith(".plist", StringComparison.OrdinalIgnoreCase) ||
path.EndsWith(".conf", StringComparison.OrdinalIgnoreCase) ||
path.EndsWith(".cfg", StringComparison.OrdinalIgnoreCase);
}
}

View File

@@ -0,0 +1,144 @@
using Claunia.PropertyList;
namespace StellaOps.Scanner.Analyzers.OS.Pkgutil;
/// <summary>
/// Parses macOS pkgutil receipt .plist files from /var/db/receipts/.
/// </summary>
internal sealed class PkgutilReceiptParser
{
/// <summary>
/// Parses a pkgutil receipt plist file.
/// </summary>
public PkgutilReceipt? Parse(string plistPath, CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(plistPath);
if (!File.Exists(plistPath))
{
return null;
}
try
{
using var stream = File.OpenRead(plistPath);
return Parse(stream, plistPath, cancellationToken);
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException or FormatException)
{
return null;
}
}
/// <summary>
/// Parses a pkgutil receipt plist from a stream.
/// </summary>
public PkgutilReceipt? Parse(Stream stream, string? sourcePath = null, CancellationToken cancellationToken = default)
{
ArgumentNullException.ThrowIfNull(stream);
try
{
var plist = PropertyListParser.Parse(stream);
if (plist is not NSDictionary root)
{
return null;
}
var identifier = GetString(root, "PackageIdentifier") ?? GetString(root, "packageIdentifier");
if (string.IsNullOrWhiteSpace(identifier))
{
return null;
}
var version = GetString(root, "PackageVersion") ?? GetString(root, "packageVersion") ?? "0.0.0";
var installDate = GetDate(root, "InstallDate");
var installPrefixPath = GetString(root, "InstallPrefixPath");
var volumePath = GetString(root, "VolumePath") ?? "/";
var installProcessName = GetString(root, "InstallProcessName");
return new PkgutilReceipt(
Identifier: identifier.Trim(),
Version: version.Trim(),
InstallDate: installDate,
InstallPrefixPath: installPrefixPath?.Trim(),
VolumePath: volumePath.Trim(),
InstallProcessName: installProcessName?.Trim(),
SourcePath: sourcePath);
}
catch (Exception ex) when (ex is FormatException or InvalidOperationException or ArgumentException)
{
return null;
}
}
/// <summary>
/// Discovers and parses all receipt plist files in the receipts directory.
/// </summary>
public IReadOnlyList<PkgutilReceipt> DiscoverReceipts(
string rootPath,
CancellationToken cancellationToken = default)
{
ArgumentException.ThrowIfNullOrWhiteSpace(rootPath);
var receiptsPath = Path.Combine(rootPath, "var", "db", "receipts");
if (!Directory.Exists(receiptsPath))
{
return Array.Empty<PkgutilReceipt>();
}
var results = new List<PkgutilReceipt>();
try
{
foreach (var plistFile in Directory.EnumerateFiles(receiptsPath, "*.plist"))
{
cancellationToken.ThrowIfCancellationRequested();
var receipt = Parse(plistFile, cancellationToken);
if (receipt is not null)
{
results.Add(receipt);
}
}
}
catch (Exception ex) when (ex is IOException or UnauthorizedAccessException)
{
// Partial results are acceptable
}
return results;
}
private static string? GetString(NSDictionary dict, string key)
{
if (dict.TryGetValue(key, out var value) && value is NSString nsString)
{
return nsString.Content;
}
return null;
}
private static DateTimeOffset? GetDate(NSDictionary dict, string key)
{
if (dict.TryGetValue(key, out var value) && value is NSDate nsDate)
{
return new DateTimeOffset(nsDate.Date, TimeSpan.Zero);
}
return null;
}
}
/// <summary>
/// Represents parsed macOS pkgutil receipt metadata.
/// </summary>
internal sealed record PkgutilReceipt(
string Identifier,
string Version,
DateTimeOffset? InstallDate,
string? InstallPrefixPath,
string VolumePath,
string? InstallProcessName,
string? SourcePath);

View File

@@ -0,0 +1,3 @@
using System.Runtime.CompilerServices;
[assembly: InternalsVisibleTo("StellaOps.Scanner.Analyzers.OS.Pkgutil.Tests")]

View File

@@ -0,0 +1,16 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
<PackageReference Include="plist-cil" Version="2.2.0" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\StellaOps.Scanner.Analyzers.OS\StellaOps.Scanner.Analyzers.OS.csproj" />
</ItemGroup>
</Project>

View File

@@ -27,6 +27,58 @@ public static class PackageUrlBuilder
return $"pkg:rpm/{Escape(name)}@{versionComponent}{releaseComponent}?arch={EscapeQuery(architecture)}";
}
/// <summary>
/// Builds a PURL for a Homebrew formula.
/// Format: pkg:brew/{tap}/{formula}@{version}?revision={revision}
/// </summary>
public static string BuildHomebrew(string tap, string formula, string version, int revision)
{
ArgumentException.ThrowIfNullOrWhiteSpace(tap);
ArgumentException.ThrowIfNullOrWhiteSpace(formula);
ArgumentException.ThrowIfNullOrWhiteSpace(version);
var normalizedTap = tap.Trim().ToLowerInvariant();
var builder = new StringBuilder();
builder.Append("pkg:brew/");
builder.Append(Escape(normalizedTap));
builder.Append('/');
builder.Append(Escape(formula));
builder.Append('@');
builder.Append(Escape(version));
if (revision > 0)
{
builder.Append("?revision=");
builder.Append(revision);
}
return builder.ToString();
}
/// <summary>
/// Builds a PURL for a macOS pkgutil receipt.
/// Format: pkg:generic/apple/{identifier}@{version}
/// </summary>
public static string BuildPkgutil(string identifier, string version)
{
ArgumentException.ThrowIfNullOrWhiteSpace(identifier);
ArgumentException.ThrowIfNullOrWhiteSpace(version);
return $"pkg:generic/apple/{Escape(identifier)}@{Escape(version)}";
}
/// <summary>
/// Builds a PURL for a macOS application bundle.
/// Format: pkg:generic/macos-app/{bundleId}@{version}
/// </summary>
public static string BuildMacOsBundle(string bundleId, string version)
{
ArgumentException.ThrowIfNullOrWhiteSpace(bundleId);
ArgumentException.ThrowIfNullOrWhiteSpace(version);
return $"pkg:generic/macos-app/{Escape(bundleId)}@{Escape(version)}";
}
private static string Escape(string value)
{
ArgumentException.ThrowIfNullOrWhiteSpace(value);

View File

@@ -6,4 +6,7 @@ public enum PackageEvidenceSource
ApkDatabase,
DpkgStatus,
RpmDatabase,
HomebrewCellar,
PkgutilReceipt,
MacOsBundle,
}

View File

@@ -0,0 +1,338 @@
using System.IO.Compression;
using System.Text;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests.Container;
public sealed class PythonZipappAdapterTests : IDisposable
{
private readonly string _tempDir;
public PythonZipappAdapterTests()
{
_tempDir = Path.Combine(Path.GetTempPath(), $"zipapp-tests-{Guid.NewGuid():N}");
Directory.CreateDirectory(_tempDir);
}
public void Dispose()
{
try
{
Directory.Delete(_tempDir, recursive: true);
}
catch
{
// Ignore cleanup errors
}
}
[Fact]
public void DiscoverZipapps_FindsPyzFiles()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "app.pyz");
CreateMinimalZipapp(pyzPath, "#!/usr/bin/env python3\n");
// Act
var discovered = PythonZipappAdapter.DiscoverZipapps(_tempDir);
// Assert
Assert.Single(discovered);
Assert.Contains(discovered, p => p.EndsWith("app.pyz"));
}
[Fact]
public void DiscoverZipapps_FindsPyzwFiles()
{
// Arrange
var pyzwPath = Path.Combine(_tempDir, "app.pyzw");
CreateMinimalZipapp(pyzwPath, "#!/usr/bin/env pythonw\n");
// Act
var discovered = PythonZipappAdapter.DiscoverZipapps(_tempDir);
// Assert
Assert.Single(discovered);
Assert.Contains(discovered, p => p.EndsWith("app.pyzw"));
}
[Fact]
public void DiscoverZipapps_FindsInContainerLayers()
{
// Arrange
var layersDir = Path.Combine(_tempDir, "layers", "layer1", "fs", "app");
Directory.CreateDirectory(layersDir);
var pyzPath = Path.Combine(layersDir, "container-app.pyz");
CreateMinimalZipapp(pyzPath, "#!/usr/bin/python3.11\n");
// Act
var discovered = PythonZipappAdapter.DiscoverZipapps(_tempDir);
// Assert
Assert.Single(discovered);
Assert.Contains(discovered, p => p.EndsWith("container-app.pyz"));
}
[Fact]
public void AnalyzeZipapp_ExtractsShebang()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "app.pyz");
CreateMinimalZipapp(pyzPath, "#!/usr/bin/python3.11\n");
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(pyzPath);
// Assert
Assert.NotNull(info);
Assert.Equal("/usr/bin/python3.11", info.Shebang);
Assert.Equal("3.11", info.PythonVersion);
}
[Fact]
public void AnalyzeZipapp_ExtractsEnvShebang()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "app.pyz");
CreateMinimalZipapp(pyzPath, "#!/usr/bin/env python3.10\n");
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(pyzPath);
// Assert
Assert.NotNull(info);
Assert.Contains("/usr/bin/env python3.10", info.Shebang);
Assert.Equal("3.10", info.PythonVersion);
}
[Fact]
public void AnalyzeZipapp_DetectsMainPy()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "app.pyz");
CreateZipappWithMain(pyzPath, "#!/usr/bin/python3\n", "print('Hello')");
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(pyzPath);
// Assert
Assert.NotNull(info);
Assert.True(info.HasMainPy);
}
[Fact]
public void AnalyzeZipapp_DetectsMissingMain_GeneratesWarning()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "app.pyz");
CreateMinimalZipapp(pyzPath, "#!/usr/bin/python3\n");
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(pyzPath);
// Assert
Assert.NotNull(info);
Assert.False(info.HasMainPy);
Assert.Contains(info.Warnings, w => w.Contains("missing __main__.py"));
}
[Fact]
public void AnalyzeZipapp_DetectsWindowsApp()
{
// Arrange
var pyzwPath = Path.Combine(_tempDir, "app.pyzw");
CreateZipappWithMain(pyzwPath, "#!/usr/bin/pythonw\n", "print('Hello')");
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(pyzwPath);
// Assert
Assert.NotNull(info);
Assert.True(info.IsWindowsApp);
Assert.Contains(info.Warnings, w => w.Contains("Windows-specific"));
}
[Fact]
public void AnalyzeZipapp_DetectsEnvShebangWarning()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "app.pyz");
CreateZipappWithMain(pyzPath, "#!/usr/bin/env python3\n", "print('Hello')");
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(pyzPath);
// Assert
Assert.NotNull(info);
Assert.Contains(info.Warnings, w => w.Contains("/usr/bin/env") && w.Contains("may vary"));
}
[Fact]
public void AnalyzeZipapp_ExtractsEmbeddedRequirements()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "app.pyz");
var requirements = "requests>=2.0\nflask==2.1.0\n# Comment\nnumpy";
CreateZipappWithRequirements(pyzPath, "#!/usr/bin/python3\n", requirements);
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(pyzPath);
// Assert
Assert.NotNull(info);
Assert.Contains("requests", info.EmbeddedDependencies);
Assert.Contains("flask", info.EmbeddedDependencies);
Assert.Contains("numpy", info.EmbeddedDependencies);
}
[Fact]
public void AnalyzeAll_ReturnsAnalysisForMultipleZipapps()
{
// Arrange
var pyz1 = Path.Combine(_tempDir, "app1.pyz");
var pyz2 = Path.Combine(_tempDir, "app2.pyz");
CreateZipappWithMain(pyz1, "#!/usr/bin/python3.10\n", "print('App1')");
CreateZipappWithMain(pyz2, "#!/usr/bin/python3.11\n", "print('App2')");
// Act
var analysis = PythonZipappAdapter.AnalyzeAll(_tempDir);
// Assert
Assert.Equal(2, analysis.Zipapps.Count);
Assert.True(analysis.HasZipapps);
Assert.Contains(analysis.Warnings, w => w.Contains("Multiple zipapps"));
}
[Fact]
public void AnalyzeAll_CollectsVersionsFromShebangs()
{
// Arrange
var pyz1 = Path.Combine(_tempDir, "app1.pyz");
var pyz2 = Path.Combine(_tempDir, "app2.pyz");
CreateZipappWithMain(pyz1, "#!/usr/bin/python3.10\n", "print('App1')");
CreateZipappWithMain(pyz2, "#!/usr/bin/python3.11\n", "print('App2')");
// Act
var analysis = PythonZipappAdapter.AnalyzeAll(_tempDir);
// Assert
var versioned = analysis.Zipapps.Where(z => z.PythonVersion != null).ToList();
Assert.Equal(2, versioned.Count);
Assert.Contains(versioned, z => z.PythonVersion == "3.10");
Assert.Contains(versioned, z => z.PythonVersion == "3.11");
}
[Fact]
public void AnalyzeZipapp_ExtractsEntryModuleFromRunpy()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "app.pyz");
var mainContent = @"
import runpy
runpy.run_module('mypackage.main')
";
CreateZipappWithMain(pyzPath, "#!/usr/bin/python3\n", mainContent);
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(pyzPath);
// Assert
Assert.NotNull(info);
Assert.Equal("mypackage.main", info.EntryModule);
}
[Fact]
public void ToMetadata_GeneratesExpectedKeys()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "app.pyz");
CreateZipappWithMain(pyzPath, "#!/usr/bin/python3.11\n", "print('Hello')");
var info = PythonZipappAdapter.AnalyzeZipapp(pyzPath);
// Act
var metadata = info!.ToMetadata();
// Assert
Assert.Contains(metadata, m => m.Key == "zipapp.path");
Assert.Contains(metadata, m => m.Key == "zipapp.hasMain" && m.Value == "true");
Assert.Contains(metadata, m => m.Key == "zipapp.shebang");
Assert.Contains(metadata, m => m.Key == "zipapp.pythonVersion" && m.Value == "3.11");
}
[Fact]
public void AnalyzeZipapp_ReturnsNull_ForNonExistentFile()
{
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(Path.Combine(_tempDir, "nonexistent.pyz"));
// Assert
Assert.Null(info);
}
[Fact]
public void AnalyzeZipapp_HandlesCorruptedArchive()
{
// Arrange
var pyzPath = Path.Combine(_tempDir, "corrupt.pyz");
File.WriteAllText(pyzPath, "#!/usr/bin/python3\nNot a valid zip archive");
// Act
var info = PythonZipappAdapter.AnalyzeZipapp(pyzPath);
// Assert - should return null for corrupted archives
Assert.Null(info);
}
private static void CreateMinimalZipapp(string path, string shebang)
{
using var fileStream = File.Create(path);
// Write shebang
var shebangBytes = Encoding.UTF8.GetBytes(shebang);
fileStream.Write(shebangBytes);
// Write minimal zip archive
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, leaveOpen: true);
var entry = archive.CreateEntry("placeholder.txt");
using var entryStream = entry.Open();
using var writer = new StreamWriter(entryStream);
writer.Write("placeholder");
}
private static void CreateZipappWithMain(string path, string shebang, string mainContent)
{
using var fileStream = File.Create(path);
// Write shebang
var shebangBytes = Encoding.UTF8.GetBytes(shebang);
fileStream.Write(shebangBytes);
// Write zip archive with __main__.py
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, leaveOpen: true);
var entry = archive.CreateEntry("__main__.py");
using var entryStream = entry.Open();
using var writer = new StreamWriter(entryStream);
writer.Write(mainContent);
}
private static void CreateZipappWithRequirements(string path, string shebang, string requirements)
{
using var fileStream = File.Create(path);
// Write shebang
var shebangBytes = Encoding.UTF8.GetBytes(shebang);
fileStream.Write(shebangBytes);
// Write zip archive with __main__.py and requirements.txt
using var archive = new ZipArchive(fileStream, ZipArchiveMode.Create, leaveOpen: true);
var mainEntry = archive.CreateEntry("__main__.py");
using (var mainStream = mainEntry.Open())
using (var mainWriter = new StreamWriter(mainStream))
{
mainWriter.Write("print('Hello')");
}
var reqEntry = archive.CreateEntry("requirements.txt");
using var reqStream = reqEntry.Open();
using var reqWriter = new StreamWriter(reqStream);
reqWriter.Write(requirements);
}
}

View File

@@ -0,0 +1,277 @@
using System.Collections.Immutable;
using System.Text.Json;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Capabilities;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Entrypoints;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Framework;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Imports;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Observations;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Packaging;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests.Fixtures;
/// <summary>
/// Fixture-based tests for Python analyzer covering various project structures.
/// </summary>
public sealed class PythonFixtureTests
{
private static readonly string FixturesPath = Path.Combine(
AppContext.BaseDirectory,
"Fixtures", "lang", "python");
/// <summary>
/// Tests that namespace packages (PEP 420) are correctly detected.
/// </summary>
[Fact]
public async Task NamespacePackage_DetectsMultipleSubpackages()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = Path.Combine(FixturesPath, "namespace-pkg");
if (!Directory.Exists(fixturePath))
{
// Fixture might not be in output yet
return;
}
var sitePackagesPath = Path.Combine(fixturePath, "lib", "python3.11", "site-packages");
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSitePackages(sitePackagesPath)
.Build();
var discovery = new PythonPackageDiscovery();
var result = await discovery.DiscoverAsync(vfs, cancellationToken);
Assert.True(result.IsSuccessful);
Assert.Equal(2, result.Packages.Count(p => p.Name.Contains("mynamespace")));
// Verify both subpackages are found
Assert.Contains(result.Packages, p => p.Name == "mynamespace-subpkg1");
Assert.Contains(result.Packages, p => p.Name == "mynamespace-subpkg2");
}
/// <summary>
/// Tests that simple virtualenv packages are correctly detected.
/// </summary>
[Fact]
public async Task SimpleVenv_DetectsPackageWithEntrypoints()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = Path.Combine(FixturesPath, "simple-venv");
if (!Directory.Exists(fixturePath))
{
return;
}
var sitePackagesPath = Path.Combine(fixturePath, "lib", "python3.11", "site-packages");
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSitePackages(sitePackagesPath)
.Build();
var discovery = new PythonPackageDiscovery();
var result = await discovery.DiscoverAsync(vfs, cancellationToken);
Assert.True(result.IsSuccessful);
Assert.Contains(result.Packages, p => p.Name == "simple");
var simplePkg = result.Packages.First(p => p.Name == "simple");
Assert.Equal("1.0.0", simplePkg.Version);
Assert.Equal("pip", simplePkg.InstallerTool);
}
/// <summary>
/// Tests that editable (development) installs are correctly detected.
/// </summary>
[Fact]
public async Task LayeredEditable_DetectsEditableInstall()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = Path.Combine(FixturesPath, "layered-editable");
if (!Directory.Exists(fixturePath))
{
return;
}
var layer1Path = Path.Combine(fixturePath, "layer1", "usr", "lib", "python3.11", "site-packages");
var layer2Path = Path.Combine(fixturePath, "layer2", "usr", "lib", "python3.11", "site-packages");
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSitePackages(layer1Path)
.AddSitePackages(layer2Path)
.Build();
var discovery = new PythonPackageDiscovery();
var result = await discovery.DiscoverAsync(vfs, cancellationToken);
Assert.True(result.IsSuccessful);
Assert.Contains(result.Packages, p => p.Name == "layered");
}
/// <summary>
/// Tests that containers with multiple layers are handled correctly.
/// </summary>
[Fact]
public async Task Container_DetectsPackagesAcrossLayers()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = Path.Combine(FixturesPath, "container");
if (!Directory.Exists(fixturePath))
{
return;
}
var layer1Path = Path.Combine(fixturePath, "layer1", "usr", "lib", "python3.11", "site-packages");
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSitePackages(layer1Path)
.Build();
var discovery = new PythonPackageDiscovery();
var result = await discovery.DiscoverAsync(vfs, cancellationToken);
Assert.True(result.IsSuccessful);
Assert.Contains(result.Packages, p => p.Name == "Flask");
var flaskPkg = result.Packages.First(p => p.Name == "Flask");
Assert.Equal("3.0.0", flaskPkg.Version);
}
/// <summary>
/// Tests Lambda handler detection from SAM templates.
/// </summary>
[Fact]
public void LambdaHandler_DetectsHandlerFromSamTemplate()
{
var fixturePath = Path.Combine(FixturesPath, "lambda-handler");
if (!Directory.Exists(fixturePath))
{
return;
}
var handlerPath = Path.Combine(fixturePath, "app", "handler.py");
Assert.True(File.Exists(handlerPath));
var content = File.ReadAllText(handlerPath);
// Verify the handler signature is present
Assert.Contains("def lambda_handler(event, context)", content);
Assert.Contains("def process_event(event, context)", content);
}
/// <summary>
/// Tests framework detection for Flask applications.
/// </summary>
[Fact]
public async Task FrameworkDetection_DetectsFlask()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = Path.Combine(FixturesPath, "container", "layer2", "app");
if (!Directory.Exists(fixturePath))
{
return;
}
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(fixturePath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.Flask);
}
/// <summary>
/// Tests capability detection for network and process execution.
/// </summary>
[Fact]
public async Task CapabilityDetection_DetectsNetworkAccess()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = Path.Combine(FixturesPath, "lambda-handler", "app");
if (!Directory.Exists(fixturePath))
{
return;
}
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(fixturePath)
.Build();
var detector = new PythonCapabilityDetector();
var capabilities = await detector.DetectAsync(vfs, cancellationToken);
// boto3 import indicates AWS SDK usage - check for any detected capability
Assert.NotEmpty(capabilities);
}
/// <summary>
/// Tests observation document generation from fixtures.
/// </summary>
[Fact]
public async Task ObservationBuilder_ProducesValidDocument()
{
var cancellationToken = TestContext.Current.CancellationToken;
var fixturePath = Path.Combine(FixturesPath, "simple-venv");
if (!Directory.Exists(fixturePath))
{
return;
}
var sitePackagesPath = Path.Combine(fixturePath, "lib", "python3.11", "site-packages");
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSitePackages(sitePackagesPath)
.Build();
var discovery = new PythonPackageDiscovery();
var result = await discovery.DiscoverAsync(vfs, cancellationToken);
var builder = new PythonObservationBuilder();
var document = builder
.AddPackages(result.Packages)
.SetEnvironment("3.11.0", [sitePackagesPath])
.Build();
Assert.Equal("python-aoc-v1", document.Schema);
Assert.NotEmpty(document.Packages);
// Verify serialization produces valid JSON
var json = PythonObservationSerializer.Serialize(document);
var parsed = JsonDocument.Parse(json);
Assert.NotNull(parsed);
}
/// <summary>
/// Tests import graph building from module files.
/// </summary>
[Fact]
public void ImportGraph_ExtractsImportsFromSource()
{
var fixturePath = Path.Combine(FixturesPath, "lambda-handler", "app");
if (!Directory.Exists(fixturePath))
{
return;
}
var handlerPath = Path.Combine(fixturePath, "handler.py");
var content = File.ReadAllText(handlerPath);
var extractor = new PythonSourceImportExtractor(handlerPath);
extractor.Extract(content);
Assert.Contains(extractor.Imports, i => i.Module == "json");
Assert.Contains(extractor.Imports, i => i.Module == "os");
Assert.Contains(extractor.Imports, i => i.Module == "boto3");
}
}

View File

@@ -0,0 +1,6 @@
FROM python:3.11-slim
WORKDIR /app
COPY requirements.txt .
RUN pip install --no-cache-dir -r requirements.txt
COPY app/ /app/
CMD ["python", "-m", "app"]

View File

@@ -0,0 +1,55 @@
[
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/flask@3.0.0",
"purl": "pkg:pypi/flask@3.0.0",
"name": "Flask",
"version": "3.0.0",
"type": "pypi",
"metadata": {
"author": "Pallets",
"distInfoPath": "layer1/usr/lib/python3.11/site-packages/flask-3.0.0.dist-info",
"installer": "pip",
"license": "BSD-3-Clause",
"name": "Flask",
"normalizedName": "flask",
"provenance": "dist-info",
"containerLayer": "layer1",
"requiresDist": "Werkzeug>=3.0;Jinja2>=3.1;click>=8.1",
"requiresPython": ">=3.8",
"summary": "A simple framework for building complex web applications",
"version": "3.0.0",
"wheel.generator": "pip 24.0",
"wheel.rootIsPurelib": "true",
"wheel.tags": "py3-none-any",
"wheel.version": "1.0"
},
"evidence": [
{
"kind": "file",
"source": "INSTALLER",
"locator": "layer1/usr/lib/python3.11/site-packages/flask-3.0.0.dist-info/INSTALLER"
},
{
"kind": "file",
"source": "METADATA",
"locator": "layer1/usr/lib/python3.11/site-packages/flask-3.0.0.dist-info/METADATA"
},
{
"kind": "file",
"source": "RECORD",
"locator": "layer1/usr/lib/python3.11/site-packages/flask-3.0.0.dist-info/RECORD"
},
{
"kind": "file",
"source": "WHEEL",
"locator": "layer1/usr/lib/python3.11/site-packages/flask-3.0.0.dist-info/WHEEL"
},
{
"kind": "container",
"source": "Dockerfile",
"locator": "Dockerfile"
}
]
}
]

View File

@@ -0,0 +1,10 @@
Metadata-Version: 2.1
Name: Flask
Version: 3.0.0
Summary: A simple framework for building complex web applications
Author: Pallets
License: BSD-3-Clause
Requires-Python: >=3.8
Requires-Dist: Werkzeug>=3.0
Requires-Dist: Jinja2>=3.1
Requires-Dist: click>=8.1

View File

@@ -0,0 +1,5 @@
flask/__init__.py,sha256=abc123,200
flask-3.0.0.dist-info/METADATA,sha256=def456,500
flask-3.0.0.dist-info/WHEEL,sha256=ghi789,80
flask-3.0.0.dist-info/INSTALLER,sha256=jkl012,4
flask-3.0.0.dist-info/RECORD,,

View File

@@ -0,0 +1,4 @@
Wheel-Version: 1.0
Generator: pip 24.0
Root-Is-Purelib: true
Tag: py3-none-any

View File

@@ -0,0 +1,11 @@
"""Flask web framework stub."""
__version__ = "3.0.0"
class Flask:
def __init__(self, name):
self.name = name
def route(self, path):
def decorator(f):
return f
return decorator

View File

@@ -0,0 +1,2 @@
"""Container application package."""
__version__ = "1.0.0"

View File

@@ -0,0 +1,11 @@
"""Application entry point."""
from flask import Flask
app = Flask(__name__)
@app.route("/")
def index():
return {"status": "healthy"}
if __name__ == "__main__":
app.run(host="0.0.0.0", port=8080)

View File

@@ -0,0 +1,2 @@
"""Lambda application package."""
__version__ = "1.0.0"

View File

@@ -0,0 +1,16 @@
"""AWS Lambda handler module."""
import json
import os
import boto3
def lambda_handler(event, context):
"""Main Lambda handler function."""
return {
"statusCode": 200,
"body": json.dumps({"message": "Hello from Lambda!"})
}
def process_event(event, context):
"""Alternative handler for processing events."""
s3 = boto3.client("s3")
return {"processed": True}

View File

@@ -0,0 +1,7 @@
"""Utility functions for Lambda handler."""
import logging
logger = logging.getLogger(__name__)
def log_event(event):
logger.info("Processing event: %s", event)

View File

@@ -0,0 +1,33 @@
[
{
"analyzerId": "python",
"componentKey": "lambda::app.handler.lambda_handler",
"name": "lambda_handler",
"type": "lambda",
"metadata": {
"handler": "handler.lambda_handler",
"runtime": "python3.11",
"codeUri": "app/",
"framework": "AWSLambda",
"templateFile": "template.yaml",
"timeout": "30"
},
"evidence": [
{
"kind": "file",
"source": "sam-template",
"locator": "template.yaml"
},
{
"kind": "file",
"source": "handler",
"locator": "app/handler.py"
},
{
"kind": "derived",
"source": "handler-signature",
"value": "def lambda_handler(event, context)"
}
]
}
]

View File

@@ -0,0 +1,21 @@
AWSTemplateFormatVersion: '2010-09-09'
Transform: AWS::Serverless-2016-10-31
Description: Sample Lambda Function
Globals:
Function:
Timeout: 30
Runtime: python3.11
Resources:
MyFunction:
Type: AWS::Serverless::Function
Properties:
CodeUri: app/
Handler: handler.lambda_handler
Events:
Api:
Type: Api
Properties:
Path: /hello
Method: get

View File

@@ -0,0 +1,100 @@
[
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/mynamespace-subpkg1@1.0.0",
"purl": "pkg:pypi/mynamespace-subpkg1@1.0.0",
"name": "mynamespace-subpkg1",
"version": "1.0.0",
"type": "pypi",
"metadata": {
"author": "Example Dev",
"authorEmail": "dev@example.com",
"distInfoPath": "lib/python3.11/site-packages/mynamespace_subpkg1-1.0.0.dist-info",
"installer": "pip",
"license": "MIT",
"name": "mynamespace-subpkg1",
"normalizedName": "mynamespace_subpkg1",
"provenance": "dist-info",
"requiresPython": ">=3.9",
"summary": "Namespace package subpkg1",
"topLevelModule": "mynamespace",
"version": "1.0.0",
"wheel.generator": "pip 24.0",
"wheel.rootIsPurelib": "true",
"wheel.tags": "py3-none-any",
"wheel.version": "1.0",
"namespacePackage": "true"
},
"evidence": [
{
"kind": "file",
"source": "INSTALLER",
"locator": "lib/python3.11/site-packages/mynamespace_subpkg1-1.0.0.dist-info/INSTALLER"
},
{
"kind": "file",
"source": "METADATA",
"locator": "lib/python3.11/site-packages/mynamespace_subpkg1-1.0.0.dist-info/METADATA"
},
{
"kind": "file",
"source": "RECORD",
"locator": "lib/python3.11/site-packages/mynamespace_subpkg1-1.0.0.dist-info/RECORD"
},
{
"kind": "file",
"source": "WHEEL",
"locator": "lib/python3.11/site-packages/mynamespace_subpkg1-1.0.0.dist-info/WHEEL"
}
]
},
{
"analyzerId": "python",
"componentKey": "purl::pkg:pypi/mynamespace-subpkg2@1.0.0",
"purl": "pkg:pypi/mynamespace-subpkg2@1.0.0",
"name": "mynamespace-subpkg2",
"version": "1.0.0",
"type": "pypi",
"metadata": {
"author": "Example Dev",
"authorEmail": "dev@example.com",
"distInfoPath": "lib/python3.11/site-packages/mynamespace_subpkg2-1.0.0.dist-info",
"installer": "pip",
"license": "MIT",
"name": "mynamespace-subpkg2",
"normalizedName": "mynamespace_subpkg2",
"provenance": "dist-info",
"requiresPython": ">=3.9",
"summary": "Namespace package subpkg2",
"topLevelModule": "mynamespace",
"version": "1.0.0",
"wheel.generator": "pip 24.0",
"wheel.rootIsPurelib": "true",
"wheel.tags": "py3-none-any",
"wheel.version": "1.0",
"namespacePackage": "true"
},
"evidence": [
{
"kind": "file",
"source": "INSTALLER",
"locator": "lib/python3.11/site-packages/mynamespace_subpkg2-1.0.0.dist-info/INSTALLER"
},
{
"kind": "file",
"source": "METADATA",
"locator": "lib/python3.11/site-packages/mynamespace_subpkg2-1.0.0.dist-info/METADATA"
},
{
"kind": "file",
"source": "RECORD",
"locator": "lib/python3.11/site-packages/mynamespace_subpkg2-1.0.0.dist-info/RECORD"
},
{
"kind": "file",
"source": "WHEEL",
"locator": "lib/python3.11/site-packages/mynamespace_subpkg2-1.0.0.dist-info/WHEEL"
}
]
}
]

View File

@@ -0,0 +1,4 @@
# Namespace subpackage 1
from .core import process
__version__ = "1.0.0"

View File

@@ -0,0 +1,5 @@
"""Core functionality for subpkg1."""
import json
def process(data):
return json.dumps(data)

View File

@@ -0,0 +1,4 @@
# Namespace subpackage 2
from .utils import helper
__version__ = "1.0.0"

View File

@@ -0,0 +1,5 @@
"""Utilities for subpkg2."""
import os
def helper():
return os.getcwd()

View File

@@ -0,0 +1,8 @@
Metadata-Version: 2.1
Name: mynamespace-subpkg1
Version: 1.0.0
Summary: Namespace package subpkg1
Author: Example Dev
Author-email: dev@example.com
License: MIT
Requires-Python: >=3.9

View File

@@ -0,0 +1,6 @@
mynamespace/subpkg1/__init__.py,sha256=abc123,50
mynamespace/subpkg1/core.py,sha256=def456,100
mynamespace_subpkg1-1.0.0.dist-info/METADATA,sha256=ghi789,200
mynamespace_subpkg1-1.0.0.dist-info/WHEEL,sha256=jkl012,80
mynamespace_subpkg1-1.0.0.dist-info/INSTALLER,sha256=mno345,4
mynamespace_subpkg1-1.0.0.dist-info/RECORD,,

View File

@@ -0,0 +1,4 @@
Wheel-Version: 1.0
Generator: pip 24.0
Root-Is-Purelib: true
Tag: py3-none-any

View File

@@ -0,0 +1,8 @@
Metadata-Version: 2.1
Name: mynamespace-subpkg2
Version: 1.0.0
Summary: Namespace package subpkg2
Author: Example Dev
Author-email: dev@example.com
License: MIT
Requires-Python: >=3.9

View File

@@ -0,0 +1,6 @@
mynamespace/subpkg2/__init__.py,sha256=abc123,50
mynamespace/subpkg2/utils.py,sha256=def456,80
mynamespace_subpkg2-1.0.0.dist-info/METADATA,sha256=ghi789,200
mynamespace_subpkg2-1.0.0.dist-info/WHEEL,sha256=jkl012,80
mynamespace_subpkg2-1.0.0.dist-info/INSTALLER,sha256=mno345,4
mynamespace_subpkg2-1.0.0.dist-info/RECORD,,

View File

@@ -0,0 +1,4 @@
Wheel-Version: 1.0
Generator: pip 24.0
Root-Is-Purelib: true
Tag: py3-none-any

View File

@@ -0,0 +1,33 @@
[
{
"analyzerId": "python",
"componentKey": "zipapp::myapp.pyz",
"name": "myapp.pyz",
"version": "2.0.0",
"type": "zipapp",
"metadata": {
"archiveType": "zipapp",
"mainModule": "__main__",
"interpreter": "/usr/bin/env python3",
"version": "2.0.0",
"modules": "myapp,myapp.cli"
},
"evidence": [
{
"kind": "file",
"source": "zipapp",
"locator": "myapp.pyz.contents/__main__.py"
},
{
"kind": "file",
"source": "zipapp",
"locator": "myapp.pyz.contents/myapp/__init__.py"
},
{
"kind": "file",
"source": "zipapp",
"locator": "myapp.pyz.contents/myapp/cli.py"
}
]
}
]

View File

@@ -0,0 +1,7 @@
#!/usr/bin/env python3
"""Main entry point for zipapp."""
import sys
from myapp.cli import main
if __name__ == "__main__":
sys.exit(main())

View File

@@ -0,0 +1,2 @@
"""MyApp zipapp package."""
__version__ = "2.0.0"

View File

@@ -0,0 +1,10 @@
"""CLI entry point."""
import argparse
import json
def main():
parser = argparse.ArgumentParser(description="MyApp CLI")
parser.add_argument("--version", action="version", version="2.0.0")
args = parser.parse_args()
print(json.dumps({"status": "ok"}))
return 0

View File

@@ -0,0 +1,642 @@
using System.Collections.Immutable;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Framework;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests.Framework;
public sealed class PythonFrameworkDetectorTests
{
[Fact]
public async Task DetectAsync_DjangoProject_FindsDjangoHints()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
// Create Django project structure
await File.WriteAllTextAsync(
Path.Combine(tempPath, "manage.py"),
"""
#!/usr/bin/env python
import os
import sys
if __name__ == "__main__":
os.environ.setdefault("DJANGO_SETTINGS_MODULE", "myproject.settings")
from django.core.management import execute_from_command_line
execute_from_command_line(sys.argv)
""",
cancellationToken);
Directory.CreateDirectory(Path.Combine(tempPath, "myproject"));
await File.WriteAllTextAsync(
Path.Combine(tempPath, "myproject", "settings.py"),
"""
INSTALLED_APPS = [
'django.contrib.admin',
'django.contrib.auth',
'myapp',
]
MIDDLEWARE = [
'django.middleware.security.SecurityMiddleware',
]
ROOT_URLCONF = 'myproject.urls'
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.Django);
Assert.Contains(hints, h => h.Evidence.Contains("INSTALLED_APPS"));
Assert.Contains(hints, h => h.Evidence.Contains("DJANGO_SETTINGS_MODULE"));
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_FlaskApp_FindsFlaskHints()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "app.py"),
"""
from flask import Flask, Blueprint
app = Flask(__name__)
api_bp = Blueprint('api', __name__, url_prefix='/api')
@app.route('/')
def index():
return 'Hello, World!'
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.Flask);
// Due to deduplication, we get the highest confidence match per kind/file
var flaskHint = hints.First(h => h.Kind == PythonFrameworkKind.Flask);
Assert.Equal(PythonFrameworkConfidence.Definitive, flaskHint.Confidence);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_FastAPIApp_FindsFastAPIHints()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "main.py"),
"""
from fastapi import FastAPI, APIRouter
app = FastAPI()
router = APIRouter()
@router.get("/items")
async def get_items():
return []
app.include_router(router)
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.FastAPI);
// Due to deduplication, we get the highest confidence match per kind/file
var fastApiHint = hints.First(h => h.Kind == PythonFrameworkKind.FastAPI);
Assert.Equal(PythonFrameworkConfidence.Definitive, fastApiHint.Confidence);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_CeleryApp_FindsCeleryHints()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "celery.py"),
"""
from celery import Celery
app = Celery('tasks', broker='redis://localhost:6379/0')
@app.task
def add(x, y):
return x + y
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.Celery);
// Due to deduplication, we get the highest confidence match per kind/file
var celeryHint = hints.First(h => h.Kind == PythonFrameworkKind.Celery);
Assert.Equal(PythonFrameworkConfidence.Definitive, celeryHint.Confidence);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_AwsLambdaHandler_FindsLambdaHint()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "handler.py"),
"""
import json
def lambda_handler(event, context):
return {
'statusCode': 200,
'body': json.dumps('Hello from Lambda!')
}
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.AwsLambda);
Assert.Contains(hints, h => h.Evidence.Contains("Lambda handler function"));
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_ClickCli_FindsClickHints()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "cli.py"),
"""
import click
@click.group()
def cli():
pass
@click.command()
@click.option('--name', default='World')
def hello(name):
click.echo(f'Hello {name}!')
cli.add_command(hello)
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.Click);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_TyperCli_FindsTyperHints()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "main.py"),
"""
import typer
app = typer.Typer()
@app.command()
def hello(name: str):
print(f"Hello {name}")
if __name__ == "__main__":
app()
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.Typer);
Assert.Contains(hints, h => h.Evidence.Contains("typer.Typer()"));
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_GunicornConfig_FindsGunicornHint()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "gunicorn.conf.py"),
"""
bind = "0.0.0.0:8000"
workers = 4
worker_class = "uvicorn.workers.UvicornWorker"
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.Gunicorn);
Assert.Contains(hints, h => h.Confidence == PythonFrameworkConfidence.Definitive);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_LoggingConfig_FindsLoggingHint()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "config.py"),
"""
import logging.config
LOGGING = {
'version': 1,
'handlers': {
'console': {
'class': 'logging.StreamHandler',
},
},
}
logging.config.dictConfig(LOGGING)
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.LoggingConfig);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_JupyterNotebook_FindsJupyterHint()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
// Create a minimal Jupyter notebook file
await File.WriteAllTextAsync(
Path.Combine(tempPath, "analysis.ipynb"),
"""
{
"cells": [],
"metadata": {},
"nbformat": 4,
"nbformat_minor": 5
}
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.Jupyter);
Assert.Contains(hints, h => h.Confidence == PythonFrameworkConfidence.Definitive);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task DetectAsync_StreamlitApp_FindsStreamlitHint()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "app.py"),
"""
import streamlit as st
st.title('My Streamlit App')
st.write('Hello, World!')
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var detector = new PythonFrameworkDetector();
var hints = await detector.DetectAsync(vfs, cancellationToken);
Assert.Contains(hints, h => h.Kind == PythonFrameworkKind.Streamlit);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public void PythonFrameworkHint_Categories_ReturnCorrectly()
{
var webHint = new PythonFrameworkHint(
Kind: PythonFrameworkKind.Flask,
SourceFile: "app.py",
LineNumber: 1,
Evidence: "Flask()",
Confidence: PythonFrameworkConfidence.Definitive);
Assert.True(webHint.IsWebFramework);
Assert.False(webHint.IsTaskQueue);
Assert.False(webHint.IsServerless);
Assert.False(webHint.IsCliFramework);
var taskHint = new PythonFrameworkHint(
Kind: PythonFrameworkKind.Celery,
SourceFile: "tasks.py",
LineNumber: 1,
Evidence: "Celery()",
Confidence: PythonFrameworkConfidence.Definitive);
Assert.False(taskHint.IsWebFramework);
Assert.True(taskHint.IsTaskQueue);
var serverlessHint = new PythonFrameworkHint(
Kind: PythonFrameworkKind.AwsLambda,
SourceFile: "handler.py",
LineNumber: 1,
Evidence: "lambda_handler",
Confidence: PythonFrameworkConfidence.High);
Assert.True(serverlessHint.IsServerless);
var cliHint = new PythonFrameworkHint(
Kind: PythonFrameworkKind.Click,
SourceFile: "cli.py",
LineNumber: 1,
Evidence: "@click.command",
Confidence: PythonFrameworkConfidence.High);
Assert.True(cliHint.IsCliFramework);
}
[Fact]
public void PythonFrameworkHint_ToMetadata_GeneratesExpectedKeys()
{
var hint = new PythonFrameworkHint(
Kind: PythonFrameworkKind.FastAPI,
SourceFile: "main.py",
LineNumber: 5,
Evidence: "FastAPI()",
Confidence: PythonFrameworkConfidence.Definitive);
var metadata = hint.ToMetadata("fw").ToDictionary(kv => kv.Key, kv => kv.Value);
Assert.Equal("FastAPI", metadata["fw.kind"]);
Assert.Equal("main.py", metadata["fw.file"]);
Assert.Equal("5", metadata["fw.line"]);
Assert.Equal("FastAPI()", metadata["fw.evidence"]);
Assert.Equal("WebFramework", metadata["fw.category"]);
}
private static string CreateTemporaryWorkspace()
{
var path = Path.Combine(Path.GetTempPath(), $"stellaops-framework-{Guid.NewGuid():N}");
Directory.CreateDirectory(path);
return path;
}
}
public sealed class PythonProjectConfigParserTests
{
[Fact]
public async Task ParsePyprojectAsync_WithOptionalDependencies_ExtractsExtras()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "pyproject.toml"),
"""
[project]
name = "mypackage"
version = "1.0.0"
[project.optional-dependencies]
dev = [
"pytest",
"black",
"mypy",
]
docs = ["sphinx", "sphinx-rtd-theme"]
all = ["mypackage[dev,docs]"]
[project.scripts]
myapp = "mypackage.cli:main"
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var parser = new PythonProjectConfigParser();
var config = await parser.ParsePyprojectAsync(vfs, "pyproject.toml", cancellationToken);
Assert.NotNull(config);
Assert.Equal("mypackage", config.ProjectName);
Assert.Equal("1.0.0", config.ProjectVersion);
Assert.Contains("dev", config.Extras);
Assert.Contains("docs", config.Extras);
Assert.Contains("all", config.Extras);
Assert.True(config.OptionalDependencies.ContainsKey("dev"));
Assert.Contains("pytest", config.OptionalDependencies["dev"]);
Assert.True(config.Scripts.ContainsKey("myapp"));
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task ParsePyprojectAsync_PoetryExtras_ExtractsExtras()
{
var cancellationToken = TestContext.Current.CancellationToken;
var tempPath = CreateTemporaryWorkspace();
try
{
await File.WriteAllTextAsync(
Path.Combine(tempPath, "pyproject.toml"),
"""
[tool.poetry]
name = "mypoetryapp"
version = "2.0.0"
[tool.poetry.extras]
ml = ["tensorflow", "numpy"]
web = ["flask", "gunicorn"]
[tool.poetry.group.dev.dependencies]
pytest = "^7.0"
[tool.poetry.scripts]
mypoetryapp = "mypoetryapp.main:run"
""",
cancellationToken);
var vfs = PythonVirtualFileSystem.CreateBuilder()
.AddSourceTree(tempPath)
.Build();
var parser = new PythonProjectConfigParser();
var config = await parser.ParsePyprojectAsync(vfs, "pyproject.toml", cancellationToken);
Assert.NotNull(config);
Assert.Contains("ml", config.Extras);
Assert.Contains("web", config.Extras);
Assert.Contains("dev", config.Extras);
Assert.True(config.OptionalDependencies.ContainsKey("ml"));
Assert.Contains("tensorflow", config.OptionalDependencies["ml"]);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public void PythonProjectConfig_ToMetadata_GeneratesExpectedKeys()
{
var config = new PythonProjectConfig(
FilePath: "pyproject.toml",
ProjectName: "myapp",
ProjectVersion: "1.2.3",
OptionalDependencies: new Dictionary<string, System.Collections.Immutable.ImmutableArray<string>>
{
["dev"] = ["pytest", "black"]
}.ToImmutableDictionary(),
Extras: ["dev", "docs"],
Scripts: new Dictionary<string, string>
{
["myapp"] = "myapp.cli:main"
}.ToImmutableDictionary());
var metadata = config.ToMetadata("proj").ToDictionary(kv => kv.Key, kv => kv.Value);
Assert.Equal("pyproject.toml", metadata["proj.path"]);
Assert.Equal("myapp", metadata["proj.name"]);
Assert.Equal("1.2.3", metadata["proj.version"]);
Assert.Equal("dev,docs", metadata["proj.extras"]);
Assert.Equal("myapp", metadata["proj.scripts"]);
}
private static string CreateTemporaryWorkspace()
{
var path = Path.Combine(Path.GetTempPath(), $"stellaops-config-{Guid.NewGuid():N}");
Directory.CreateDirectory(path);
return path;
}
}

View File

@@ -0,0 +1,400 @@
using System.Collections.Immutable;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Capabilities;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Entrypoints;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Framework;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Imports;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Observations;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Packaging;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.VirtualFileSystem;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests.Observations;
public sealed class PythonObservationBuilderTests
{
[Fact]
public void Build_WithNoData_ReturnsEmptyDocument()
{
var builder = new PythonObservationBuilder();
var document = builder.Build();
Assert.Equal("python-aoc-v1", document.Schema);
Assert.Empty(document.Packages);
Assert.Empty(document.Modules);
Assert.Empty(document.Entrypoints);
Assert.Empty(document.DependencyEdges);
Assert.Empty(document.ImportEdges);
Assert.Empty(document.NativeExtensions);
Assert.Empty(document.Frameworks);
Assert.Empty(document.Warnings);
Assert.False(document.Capabilities.UsesProcessExecution);
Assert.False(document.Capabilities.UsesNetworkAccess);
}
[Fact]
public void AddPackages_AddsPackagesAndDependencyEdges()
{
var packages = new[]
{
new PythonPackageInfo(
Name: "requests",
Version: "2.31.0",
Kind: PythonPackageKind.Wheel,
Location: "/venv/lib/python3.11/site-packages",
MetadataPath: "/venv/lib/python3.11/site-packages/requests-2.31.0.dist-info",
TopLevelModules: ImmutableArray.Create("requests"),
Dependencies: ImmutableArray.Create("urllib3>=1.21.1", "certifi>=2017.4.17"),
Extras: ImmutableArray<string>.Empty,
RecordFiles: ImmutableArray<PythonRecordEntry>.Empty,
InstallerTool: "pip",
EditableTarget: null,
IsDirectDependency: true,
Confidence: PythonPackageConfidence.High)
};
var builder = new PythonObservationBuilder();
var document = builder.AddPackages(packages).Build();
Assert.Single(document.Packages);
var pkg = document.Packages[0];
Assert.Equal("requests", pkg.Name);
Assert.Equal("2.31.0", pkg.Version);
Assert.Equal("Wheel", pkg.Source);
Assert.True(pkg.IsDirect);
Assert.Equal("pip", pkg.InstallerKind);
Assert.Equal(2, document.DependencyEdges.Length);
Assert.Contains(document.DependencyEdges, e => e.FromPackage == "requests" && e.ToPackage == "urllib3");
Assert.Contains(document.DependencyEdges, e => e.FromPackage == "requests" && e.ToPackage == "certifi");
}
[Fact]
public void AddModules_AddsModulesCorrectly()
{
var modules = new[]
{
new PythonModuleNode(
ModulePath: "mypackage",
VirtualPath: "/app/mypackage/__init__.py",
IsPackage: true,
IsNamespacePackage: false,
Source: PythonFileSource.SourceTree),
new PythonModuleNode(
ModulePath: "mypackage.core",
VirtualPath: "/app/mypackage/core.py",
IsPackage: false,
IsNamespacePackage: false,
Source: PythonFileSource.SourceTree)
};
var builder = new PythonObservationBuilder();
var document = builder.AddModules(modules).Build();
Assert.Equal(2, document.Modules.Length);
var pkgModule = document.Modules.First(m => m.Name == "mypackage");
Assert.Equal("package", pkgModule.Type);
Assert.Contains("__init__.py", pkgModule.FilePath);
var coreModule = document.Modules.First(m => m.Name == "mypackage.core");
Assert.Equal("module", coreModule.Type);
Assert.Equal("mypackage", coreModule.ParentPackage);
}
[Fact]
public void AddEntrypoints_AddsEntrypointsCorrectly()
{
var entrypoints = new[]
{
new PythonEntrypoint(
Name: "myapp",
Kind: PythonEntrypointKind.PackageMain,
Target: "myapp.__main__",
VirtualPath: "/app/myapp/__main__.py",
InvocationContext: PythonInvocationContext.AsModule("myapp"),
Confidence: PythonEntrypointConfidence.High,
Source: "__main__.py detection")
};
var builder = new PythonObservationBuilder();
var document = builder.AddEntrypoints(entrypoints).Build();
Assert.Single(document.Entrypoints);
var ep = document.Entrypoints[0];
Assert.Equal("/app/myapp/__main__.py", ep.Path);
Assert.Equal("PackageMain", ep.Type);
Assert.Equal("Module", ep.InvocationContext);
}
[Fact]
public void AddCapabilities_SetsCapabilityFlags()
{
var capabilities = new[]
{
new PythonCapability(
Kind: PythonCapabilityKind.ProcessExecution,
SourceFile: "/app/utils.py",
LineNumber: 10,
Evidence: "subprocess.run()",
Confidence: PythonCapabilityConfidence.Definitive),
new PythonCapability(
Kind: PythonCapabilityKind.NetworkAccess,
SourceFile: "/app/client.py",
LineNumber: 20,
Evidence: "requests.get()",
Confidence: PythonCapabilityConfidence.High),
new PythonCapability(
Kind: PythonCapabilityKind.AsyncAwait,
SourceFile: "/app/async_handler.py",
LineNumber: 5,
Evidence: "async def",
Confidence: PythonCapabilityConfidence.Definitive)
};
var builder = new PythonObservationBuilder();
var document = builder.AddCapabilities(capabilities).Build();
Assert.True(document.Capabilities.UsesProcessExecution);
Assert.True(document.Capabilities.UsesNetworkAccess);
Assert.True(document.Capabilities.UsesAsyncAwait);
Assert.False(document.Capabilities.UsesFileSystem);
Assert.False(document.Capabilities.UsesNativeCode);
// ProcessExecution is security sensitive
Assert.Contains("ProcessExecution", document.Capabilities.SecuritySensitiveCapabilities);
// NetworkAccess is also security sensitive
Assert.Contains("NetworkAccess", document.Capabilities.SecuritySensitiveCapabilities);
}
[Fact]
public void AddNativeExtensions_AddsExtensionsAndSetsFlag()
{
var extensions = new[]
{
new PythonNativeExtension(
ModuleName: "numpy.core._multiarray_umath",
Path: "/venv/lib/python3.11/site-packages/numpy/core/_multiarray_umath.cpython-311-x86_64-linux-gnu.so",
Kind: PythonNativeExtensionKind.CExtension,
Platform: "linux",
Architecture: "x86_64",
Source: PythonFileSource.SitePackages,
PackageName: "numpy",
Dependencies: ImmutableArray<string>.Empty)
};
var builder = new PythonObservationBuilder();
var document = builder.AddNativeExtensions(extensions).Build();
Assert.Single(document.NativeExtensions);
Assert.True(document.Capabilities.UsesNativeCode);
var ext = document.NativeExtensions[0];
Assert.Equal("numpy.core._multiarray_umath", ext.ModuleName);
Assert.Equal("CExtension", ext.Kind);
Assert.Equal("numpy", ext.PackageName);
}
[Fact]
public void AddFrameworkHints_AddsHintsWithCategory()
{
var hints = new[]
{
new PythonFrameworkHint(
Kind: PythonFrameworkKind.Flask,
SourceFile: "/app/main.py",
LineNumber: 5,
Evidence: "Flask(__name__)",
Confidence: PythonFrameworkConfidence.Definitive),
new PythonFrameworkHint(
Kind: PythonFrameworkKind.Celery,
SourceFile: "/app/tasks.py",
LineNumber: 1,
Evidence: "Celery()",
Confidence: PythonFrameworkConfidence.High)
};
var builder = new PythonObservationBuilder();
var document = builder.AddFrameworkHints(hints).Build();
Assert.Equal(2, document.Frameworks.Length);
var flask = document.Frameworks.First(f => f.Kind == "Flask");
Assert.Equal("WebFramework", flask.Category);
Assert.Equal(PythonObservationConfidence.Definitive, flask.Confidence);
var celery = document.Frameworks.First(f => f.Kind == "Celery");
Assert.Equal("TaskQueue", celery.Category);
Assert.Contains("Flask", document.Capabilities.DetectedFrameworks);
Assert.Contains("Celery", document.Capabilities.DetectedFrameworks);
}
[Fact]
public void SetEnvironment_SetsEnvironmentCorrectly()
{
var builder = new PythonObservationBuilder();
var document = builder
.SetEnvironment(
pythonVersion: "3.11.4",
sitePackagesPaths: ["/venv/lib/python3.11/site-packages"],
requirementsFiles: ["/app/requirements.txt"],
pyprojectFiles: ["/app/pyproject.toml"],
virtualenvPath: "/venv",
condaPrefix: null,
isContainer: true)
.Build();
Assert.Equal("3.11.4", document.Environment.PythonVersion);
Assert.Single(document.Environment.SitePackagesPaths);
Assert.Single(document.Environment.RequirementsFiles);
Assert.Single(document.Environment.PyprojectFiles);
Assert.Equal("/venv", document.Environment.VirtualenvPath);
Assert.Null(document.Environment.CondaPrefix);
Assert.True(document.Environment.IsContainer);
}
[Fact]
public void AddWarning_AddsWarningsCorrectly()
{
var builder = new PythonObservationBuilder();
var document = builder
.AddWarning("PY001", "Unresolved import: missing_module", "/app/main.py", 15)
.AddWarning("PY002", "Deprecated package usage", severity: "info")
.Build();
Assert.Equal(2, document.Warnings.Length);
var warning1 = document.Warnings.First(w => w.Code == "PY001");
Assert.Equal("Unresolved import: missing_module", warning1.Message);
Assert.Equal("/app/main.py", warning1.FilePath);
Assert.Equal(15, warning1.Line);
Assert.Equal("warning", warning1.Severity);
var warning2 = document.Warnings.First(w => w.Code == "PY002");
Assert.Null(warning2.FilePath);
Assert.Equal("info", warning2.Severity);
}
[Fact]
public void SetRuntimeEvidence_SetsEvidenceCorrectly()
{
var evidence = new PythonObservationRuntimeEvidence(
HasEvidence: true,
RuntimePythonVersion: "3.11.4",
RuntimePlatform: "linux",
LoadedModulesCount: 50,
LoadedPackages: ImmutableArray.Create("numpy", "pandas"),
LoadedModules: ImmutableArray.Create("myapp", "myapp.core"),
PathHashes: ImmutableDictionary<string, string>.Empty.Add("/app/main.py", "abc123"),
RuntimeCapabilities: ImmutableArray.Create("network", "filesystem"),
Errors: ImmutableArray<PythonObservationRuntimeError>.Empty);
var builder = new PythonObservationBuilder();
var document = builder.SetRuntimeEvidence(evidence).Build();
Assert.NotNull(document.RuntimeEvidence);
Assert.True(document.RuntimeEvidence.HasEvidence);
Assert.Equal("3.11.4", document.RuntimeEvidence.RuntimePythonVersion);
Assert.Equal(50, document.RuntimeEvidence.LoadedModulesCount);
Assert.Contains("numpy", document.RuntimeEvidence.LoadedPackages);
}
[Fact]
public void AddImportEdges_AddsEdgesCorrectly()
{
var import1 = new PythonImport(
Module: "requests",
Names: null,
Alias: null,
Kind: PythonImportKind.Import,
RelativeLevel: 0,
SourceFile: "/app/client.py",
LineNumber: 1,
Confidence: PythonImportConfidence.Definitive);
var import2 = new PythonImport(
Module: "json",
Names: [new PythonImportedName("loads"), new PythonImportedName("dumps")],
Alias: null,
Kind: PythonImportKind.FromImport,
RelativeLevel: 0,
SourceFile: "/app/client.py",
LineNumber: 2,
Confidence: PythonImportConfidence.High);
var edges = new[]
{
new PythonImportEdge("myapp.client", "requests", import1),
new PythonImportEdge("myapp.client", "json", import2)
};
var builder = new PythonObservationBuilder();
var document = builder.AddImportEdges(edges).Build();
Assert.Equal(2, document.ImportEdges.Length);
var requestsEdge = document.ImportEdges.First(e => e.ToModule == "requests");
Assert.Equal("myapp.client", requestsEdge.FromModule);
Assert.Equal(PythonObservationImportKind.Import, requestsEdge.Kind);
Assert.Equal(PythonObservationConfidence.Definitive, requestsEdge.Confidence);
var jsonEdge = document.ImportEdges.First(e => e.ToModule == "json");
Assert.Equal(PythonObservationImportKind.FromImport, jsonEdge.Kind);
Assert.Equal(PythonObservationConfidence.High, jsonEdge.Confidence);
}
[Fact]
public void FluentBuilder_ChainsCorrectly()
{
var packages = new[]
{
new PythonPackageInfo(
Name: "flask",
Version: "3.0.0",
Kind: PythonPackageKind.Wheel,
Location: "/venv/lib/python3.11/site-packages",
MetadataPath: null,
TopLevelModules: ImmutableArray.Create("flask"),
Dependencies: ImmutableArray<string>.Empty,
Extras: ImmutableArray<string>.Empty,
RecordFiles: ImmutableArray<PythonRecordEntry>.Empty,
InstallerTool: "pip",
EditableTarget: null,
IsDirectDependency: true,
Confidence: PythonPackageConfidence.High)
};
var capabilities = new[]
{
new PythonCapability(
Kind: PythonCapabilityKind.NetworkAccess,
SourceFile: "/app/app.py",
LineNumber: 10,
Evidence: "http server",
Confidence: PythonCapabilityConfidence.High)
};
var frameworks = new[]
{
new PythonFrameworkHint(
Kind: PythonFrameworkKind.Flask,
SourceFile: "/app/app.py",
LineNumber: 3,
Evidence: "Flask(__name__)",
Confidence: PythonFrameworkConfidence.Definitive)
};
var document = new PythonObservationBuilder()
.AddPackages(packages)
.AddCapabilities(capabilities)
.AddFrameworkHints(frameworks)
.SetEnvironment("3.11.0", isContainer: false)
.AddWarning("PY100", "Test warning")
.Build();
Assert.Single(document.Packages);
Assert.Single(document.Frameworks);
Assert.Single(document.Warnings);
Assert.True(document.Capabilities.UsesNetworkAccess);
Assert.Equal("3.11.0", document.Environment.PythonVersion);
}
}

View File

@@ -0,0 +1,280 @@
using System.Collections.Immutable;
using System.Text.Json;
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.Observations;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests.Observations;
public sealed class PythonObservationSerializerTests
{
[Fact]
public void Serialize_EmptyDocument_ProducesValidJson()
{
var document = new PythonObservationBuilder().Build();
var json = PythonObservationSerializer.Serialize(document);
Assert.NotEmpty(json);
Assert.Contains("\"schema\":", json);
Assert.Contains("\"python-aoc-v1\"", json);
Assert.Contains("\"packages\":", json);
Assert.Contains("\"modules\":", json);
Assert.Contains("\"capabilities\":", json);
// Validate it's parseable JSON
var parsed = JsonDocument.Parse(json);
Assert.NotNull(parsed);
}
[Fact]
public void Serialize_Compact_ProducesMinifiedJson()
{
var document = new PythonObservationBuilder()
.AddWarning("PY001", "Test warning")
.Build();
var pretty = PythonObservationSerializer.Serialize(document, compact: false);
var compact = PythonObservationSerializer.Serialize(document, compact: true);
Assert.True(compact.Length < pretty.Length);
Assert.DoesNotContain("\n", compact);
}
[Fact]
public void Serialize_UsesCamelCase()
{
var document = new PythonObservationBuilder()
.SetEnvironment("3.11.0", isContainer: true)
.Build();
var json = PythonObservationSerializer.Serialize(document);
Assert.Contains("\"pythonVersion\":", json);
Assert.Contains("\"sitePackagesPaths\":", json);
Assert.Contains("\"isContainer\":", json);
Assert.DoesNotContain("\"PythonVersion\":", json);
Assert.DoesNotContain("\"SitePackagesPaths\":", json);
}
[Fact]
public void Serialize_OmitsNullValues()
{
var document = new PythonObservationBuilder().Build();
var json = PythonObservationSerializer.Serialize(document);
// RuntimeEvidence is null by default
Assert.DoesNotContain("\"runtimeEvidence\":", json);
}
[Fact]
public void Serialize_EnumsAsStrings()
{
var edges = ImmutableArray.Create(
new PythonObservationImportEdge(
FromModule: "app",
ToModule: "requests",
Kind: PythonObservationImportKind.Import,
Confidence: PythonObservationConfidence.High,
ResolvedPath: null,
SourceFile: "/app/main.py",
Line: 1,
ResolverTrace: ImmutableArray<string>.Empty));
var document = new PythonObservationDocument(
Schema: "python-aoc-v1",
Packages: ImmutableArray<PythonObservationPackage>.Empty,
Modules: ImmutableArray<PythonObservationModule>.Empty,
Entrypoints: ImmutableArray<PythonObservationEntrypoint>.Empty,
DependencyEdges: ImmutableArray<PythonObservationDependencyEdge>.Empty,
ImportEdges: edges,
NativeExtensions: ImmutableArray<PythonObservationNativeExtension>.Empty,
Frameworks: ImmutableArray<PythonObservationFrameworkHint>.Empty,
Warnings: ImmutableArray<PythonObservationWarning>.Empty,
Environment: new PythonObservationEnvironment(
PythonVersion: null,
SitePackagesPaths: ImmutableArray<string>.Empty,
VersionSources: ImmutableArray<PythonObservationVersionSource>.Empty,
RequirementsFiles: ImmutableArray<string>.Empty,
PyprojectFiles: ImmutableArray<string>.Empty,
VirtualenvPath: null,
CondaPrefix: null,
IsContainer: false),
Capabilities: new PythonObservationCapabilitySummary(
UsesProcessExecution: false,
UsesNetworkAccess: false,
UsesFileSystem: false,
UsesCodeExecution: false,
UsesDeserialization: false,
UsesNativeCode: false,
UsesAsyncAwait: false,
UsesMultiprocessing: false,
DetectedFrameworks: ImmutableArray<string>.Empty,
SecuritySensitiveCapabilities: ImmutableArray<string>.Empty));
var json = PythonObservationSerializer.Serialize(document);
// Check that enums are serialized as camelCase strings
Assert.Contains("\"import\"", json);
Assert.Contains("\"high\"", json);
}
[Fact]
public void Deserialize_RoundTrips()
{
var original = new PythonObservationBuilder()
.SetEnvironment("3.11.0", ["/venv/lib/python3.11/site-packages"], isContainer: true)
.AddWarning("PY001", "Test warning", "/app/main.py", 10)
.Build();
var json = PythonObservationSerializer.Serialize(original);
var deserialized = PythonObservationSerializer.Deserialize(json);
Assert.NotNull(deserialized);
Assert.Equal(original.Schema, deserialized.Schema);
Assert.Equal(original.Environment.PythonVersion, deserialized.Environment.PythonVersion);
Assert.Equal(original.Environment.IsContainer, deserialized.Environment.IsContainer);
Assert.Equal(original.Warnings.Length, deserialized.Warnings.Length);
Assert.Equal(original.Warnings[0].Code, deserialized.Warnings[0].Code);
}
[Fact]
public async Task SerializeAsync_WritesToStream()
{
var cancellationToken = TestContext.Current.CancellationToken;
var document = new PythonObservationBuilder()
.AddWarning("PY001", "Test")
.Build();
using var stream = new MemoryStream();
await PythonObservationSerializer.SerializeAsync(document, stream, compact: false, cancellationToken);
stream.Position = 0;
using var reader = new StreamReader(stream);
var json = await reader.ReadToEndAsync(cancellationToken);
Assert.Contains("\"python-aoc-v1\"", json);
Assert.Contains("\"PY001\"", json);
}
[Fact]
public async Task DeserializeAsync_ReadsFromStream()
{
var cancellationToken = TestContext.Current.CancellationToken;
var json = """
{
"schema": "python-aoc-v1",
"packages": [],
"modules": [],
"entrypoints": [],
"dependencyEdges": [],
"importEdges": [],
"nativeExtensions": [],
"frameworks": [],
"warnings": [{"code": "PY001", "message": "Test", "severity": "warning"}],
"environment": {
"sitePackagesPaths": [],
"versionSources": [],
"requirementsFiles": [],
"pyprojectFiles": [],
"isContainer": false
},
"capabilities": {
"usesProcessExecution": false,
"usesNetworkAccess": false,
"usesFileSystem": false,
"usesCodeExecution": false,
"usesDeserialization": false,
"usesNativeCode": false,
"usesAsyncAwait": false,
"usesMultiprocessing": false,
"detectedFrameworks": [],
"securitySensitiveCapabilities": []
}
}
""";
using var stream = new MemoryStream(System.Text.Encoding.UTF8.GetBytes(json));
var document = await PythonObservationSerializer.DeserializeAsync(stream, cancellationToken);
Assert.NotNull(document);
Assert.Equal("python-aoc-v1", document.Schema);
Assert.Single(document.Warnings);
Assert.Equal("PY001", document.Warnings[0].Code);
}
[Fact]
public void Serialize_FullDocument_ProducesExpectedStructure()
{
var document = new PythonObservationDocument(
Schema: "python-aoc-v1",
Packages: ImmutableArray.Create(
new PythonObservationPackage(
Name: "requests",
Version: "2.31.0",
Source: "Wheel",
Platform: null,
IsDirect: true,
InstallerKind: "pip",
DistInfoPath: "/site-packages/requests-2.31.0.dist-info",
Groups: ImmutableArray<string>.Empty,
Extras: ImmutableArray<string>.Empty)),
Modules: ImmutableArray.Create(
new PythonObservationModule(
Name: "myapp",
Type: "package",
FilePath: "/app/myapp/__init__.py",
Line: null,
IsNamespacePackage: false,
ParentPackage: null,
Imports: ImmutableArray.Create("requests", "json"))),
Entrypoints: ImmutableArray.Create(
new PythonObservationEntrypoint(
Path: "/app/myapp/__main__.py",
Type: "PackageMain",
Handler: null,
RequiredPackages: ImmutableArray<string>.Empty,
InvocationContext: "Module")),
DependencyEdges: ImmutableArray.Create(
new PythonObservationDependencyEdge(
FromPackage: "myapp",
ToPackage: "requests",
VersionConstraint: ">=2.28.0",
Extra: null,
IsOptional: false)),
ImportEdges: ImmutableArray<PythonObservationImportEdge>.Empty,
NativeExtensions: ImmutableArray<PythonObservationNativeExtension>.Empty,
Frameworks: ImmutableArray<PythonObservationFrameworkHint>.Empty,
Warnings: ImmutableArray<PythonObservationWarning>.Empty,
Environment: new PythonObservationEnvironment(
PythonVersion: "3.11.4",
SitePackagesPaths: ImmutableArray.Create("/app/.venv/lib/python3.11/site-packages"),
VersionSources: ImmutableArray<PythonObservationVersionSource>.Empty,
RequirementsFiles: ImmutableArray.Create("/app/requirements.txt"),
PyprojectFiles: ImmutableArray<string>.Empty,
VirtualenvPath: "/app/.venv",
CondaPrefix: null,
IsContainer: true),
Capabilities: new PythonObservationCapabilitySummary(
UsesProcessExecution: false,
UsesNetworkAccess: true,
UsesFileSystem: false,
UsesCodeExecution: false,
UsesDeserialization: false,
UsesNativeCode: false,
UsesAsyncAwait: true,
UsesMultiprocessing: false,
DetectedFrameworks: ImmutableArray.Create("FastAPI"),
SecuritySensitiveCapabilities: ImmutableArray<string>.Empty));
var json = PythonObservationSerializer.Serialize(document);
var parsed = JsonDocument.Parse(json);
// Verify structure
var root = parsed.RootElement;
Assert.Equal("python-aoc-v1", root.GetProperty("schema").GetString());
Assert.Equal(1, root.GetProperty("packages").GetArrayLength());
Assert.Equal("requests", root.GetProperty("packages")[0].GetProperty("name").GetString());
Assert.True(root.GetProperty("capabilities").GetProperty("usesNetworkAccess").GetBoolean());
Assert.Equal("3.11.4", root.GetProperty("environment").GetProperty("pythonVersion").GetString());
}
}

View File

@@ -0,0 +1,125 @@
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.RuntimeEvidence;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests.RuntimeEvidence;
/// <summary>
/// Tests for PythonPathHasher path scrubbing and hashing functionality.
/// </summary>
public sealed class PythonPathHasherTests
{
[Theory]
[InlineData("/home/user/project/main.py", "[HOME]/project/main.py")]
[InlineData("/Users/developer/code/app.py", "[HOME]/code/app.py")]
[InlineData("C:\\Users\\admin\\Documents\\script.py", "[HOME]/Documents/script.py")]
[InlineData("/root/.local/lib/python3.11/site-packages/flask/__init__.py", "[ROOT]/.local/lib/python3.11/site-packages/flask/__init__.py")]
[InlineData("/tmp/abc123/temp.py", "[TEMP]/temp.py")]
public void ScrubPath_ReplacesSensitiveComponents(string input, string expected)
{
var result = PythonPathHasher.ScrubPath(input);
Assert.Equal(expected, result);
}
[Fact]
public void ScrubPath_NullInput_ReturnsEmpty()
{
var result = PythonPathHasher.ScrubPath(null);
Assert.Equal(string.Empty, result);
}
[Fact]
public void ScrubPath_EmptyInput_ReturnsEmpty()
{
var result = PythonPathHasher.ScrubPath(string.Empty);
Assert.Equal(string.Empty, result);
}
[Fact]
public void ScrubPath_NormalizesPathSeparators()
{
var result = PythonPathHasher.ScrubPath("C:\\Users\\test\\project\\main.py");
Assert.Contains("/", result);
Assert.DoesNotContain("\\", result);
}
[Fact]
public void HashPath_ReturnsDeterministicHash()
{
var path = "/usr/lib/python3.11/site-packages/flask/__init__.py";
var hash1 = PythonPathHasher.HashPath(path);
var hash2 = PythonPathHasher.HashPath(path);
Assert.Equal(hash1, hash2);
Assert.Equal(64, hash1.Length); // SHA-256 produces 64 hex chars
}
[Fact]
public void HashPath_NullInput_ReturnsEmpty()
{
var result = PythonPathHasher.HashPath(null);
Assert.Equal(string.Empty, result);
}
[Fact]
public void HashPath_DifferentPaths_ProduceDifferentHashes()
{
var hash1 = PythonPathHasher.HashPath("/path/to/module1.py");
var hash2 = PythonPathHasher.HashPath("/path/to/module2.py");
Assert.NotEqual(hash1, hash2);
}
[Fact]
public void HashPath_CaseInsensitiveNormalization()
{
// Windows paths with different case should hash the same
var hash1 = PythonPathHasher.HashPath("/Path/To/Module.py");
var hash2 = PythonPathHasher.HashPath("/path/to/module.py");
Assert.Equal(hash1, hash2);
}
[Fact]
public void ScrubAndHash_ReturnsBothValues()
{
var path = "/home/user/project/main.py";
var (scrubbed, hash) = PythonPathHasher.ScrubAndHash(path);
Assert.Equal("[HOME]/project/main.py", scrubbed);
Assert.NotEmpty(hash);
Assert.Equal(64, hash.Length);
}
[Theory]
[InlineData("/usr/lib/python3.11/site-packages/flask/__init__.py", "flask")]
[InlineData("/usr/lib/python3.11/site-packages/requests/api.py", "requests.api")]
[InlineData("/usr/lib/python3.11/site-packages/numpy/core/__init__.py", "numpy.core")]
[InlineData("/usr/lib/python3.11/dist-packages/django/views.py", "django.views")]
public void ExtractModuleName_ExtractsFromSitePackages(string path, string expected)
{
var result = PythonPathHasher.ExtractModuleName(path);
Assert.Equal(expected, result);
}
[Theory]
[InlineData("/usr/lib/python3.11/site-packages/numpy/core/multiarray.cpython-311-x86_64-linux-gnu.so", "numpy.core.multiarray")]
[InlineData("/usr/lib/python3.11/site-packages/_ssl.cpython-311-x86_64-linux-gnu.so", "_ssl")]
public void ExtractModuleName_HandlesNativeExtensions(string path, string expected)
{
var result = PythonPathHasher.ExtractModuleName(path);
Assert.Equal(expected, result);
}
[Fact]
public void ExtractModuleName_NullInput_ReturnsNull()
{
var result = PythonPathHasher.ExtractModuleName(null);
Assert.Null(result);
}
[Fact]
public void ExtractModuleName_FallbackToFilename()
{
var result = PythonPathHasher.ExtractModuleName("/some/other/path/mymodule.py");
Assert.Equal("mymodule", result);
}
}

View File

@@ -0,0 +1,240 @@
using StellaOps.Scanner.Analyzers.Lang.Python.Internal.RuntimeEvidence;
namespace StellaOps.Scanner.Analyzers.Lang.Python.Tests.RuntimeEvidence;
/// <summary>
/// Tests for PythonRuntimeEvidenceCollector.
/// </summary>
public sealed class PythonRuntimeEvidenceCollectorTests
{
[Fact]
public void ParseLine_InterpreterStart_CapturesVersion()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "interpreter_start", "python_version": "3.11.5", "platform": "linux", "pid": 12345}""";
collector.ParseLine(json);
var evidence = collector.Build();
Assert.True(evidence.HasEvidence);
Assert.Equal("3.11.5", evidence.RuntimePythonVersion);
Assert.Equal("linux", evidence.RuntimePlatform);
}
[Fact]
public void ParseLine_ModuleImport_TracksModule()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "module_import", "module": "flask", "path": "/usr/lib/python3.11/site-packages/flask/__init__.py", "pid": 12345}""";
collector.ParseLine(json);
var evidence = collector.Build();
Assert.Contains("flask", evidence.LoadedModules);
Assert.Contains("flask", evidence.LoadedPackages);
Assert.Equal(1, evidence.LoadedModulesCount);
}
[Fact]
public void ParseLine_NestedModule_ExtractsTopLevelPackage()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "module_import", "module": "flask.views", "path": "/usr/lib/python3.11/site-packages/flask/views.py", "pid": 12345}""";
collector.ParseLine(json);
var evidence = collector.Build();
Assert.Contains("flask.views", evidence.LoadedModules);
Assert.Contains("flask", evidence.LoadedPackages);
}
[Fact]
public void ParseLine_NativeLoad_TracksCapability()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "native_load", "module": "numpy.core.multiarray", "path": "/usr/lib/python3.11/site-packages/numpy/core/multiarray.cpython-311-x86_64-linux-gnu.so", "pid": 12345}""";
collector.ParseLine(json);
var evidence = collector.Build();
Assert.Contains("numpy.core.multiarray", evidence.LoadedModules);
Assert.Contains("native_code", evidence.RuntimeCapabilities);
}
[Fact]
public void ParseLine_DynamicImport_TracksCapability()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "dynamic_import", "module": "plugin_module", "pid": 12345}""";
collector.ParseLine(json);
var evidence = collector.Build();
Assert.Contains("plugin_module", evidence.LoadedModules);
Assert.Contains("dynamic_import", evidence.RuntimeCapabilities);
}
[Fact]
public void ParseLine_ProcessSpawn_TracksCapability()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "process_spawn", "spawn_type": "subprocess", "pid": 12345}""";
collector.ParseLine(json);
var evidence = collector.Build();
Assert.Contains("process_spawn", evidence.RuntimeCapabilities);
}
[Fact]
public void ParseLine_MultiprocessingSpawn_TracksMultiprocessingCapability()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "process_spawn", "spawn_type": "multiprocessing", "pid": 12345}""";
collector.ParseLine(json);
var evidence = collector.Build();
Assert.Contains("process_spawn", evidence.RuntimeCapabilities);
Assert.Contains("multiprocessing", evidence.RuntimeCapabilities);
}
[Fact]
public void ParseLine_ModuleError_CapturesError()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "module_error", "module": "missing_module", "error": "ModuleNotFoundError: No module named 'missing_module'", "pid": 12345}""";
collector.ParseLine(json);
var evidence = collector.Build();
Assert.NotEmpty(evidence.Errors);
Assert.Contains(evidence.Errors, e => e.Message.Contains("missing_module"));
}
[Fact]
public void ParseLine_PathModification_AddsHash()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "path_modification", "path": "/usr/lib/python3.11/site-packages", "action": "append", "pid": 12345}""";
collector.ParseLine(json);
var evidence = collector.Build();
Assert.NotEmpty(evidence.PathHashes);
}
[Fact]
public void ParseOutput_MultipleLines_ParsesAll()
{
var collector = new PythonRuntimeEvidenceCollector();
var ndjson = """
{"type": "interpreter_start", "python_version": "3.11.5", "platform": "linux", "pid": 12345}
{"type": "module_import", "module": "os", "path": "/usr/lib/python3.11/os.py", "pid": 12345}
{"type": "module_import", "module": "sys", "path": null, "pid": 12345}
{"type": "module_import", "module": "json", "path": "/usr/lib/python3.11/json/__init__.py", "pid": 12345}
""";
collector.ParseOutput(ndjson);
var evidence = collector.Build();
Assert.True(evidence.HasEvidence);
Assert.Equal("3.11.5", evidence.RuntimePythonVersion);
Assert.Contains("os", evidence.LoadedModules);
Assert.Contains("sys", evidence.LoadedModules);
Assert.Contains("json", evidence.LoadedModules);
Assert.Equal(3, evidence.LoadedModulesCount);
}
[Fact]
public void ParseLine_MalformedJson_Ignored()
{
var collector = new PythonRuntimeEvidenceCollector();
collector.ParseLine("not valid json");
collector.ParseLine("{incomplete");
var evidence = collector.Build();
Assert.False(evidence.HasEvidence);
}
[Fact]
public void ParseLine_EmptyLine_Ignored()
{
var collector = new PythonRuntimeEvidenceCollector();
collector.ParseLine(string.Empty);
collector.ParseLine(" ");
collector.ParseLine(null!);
var evidence = collector.Build();
Assert.False(evidence.HasEvidence);
}
[Fact]
public void Build_NoEvents_ReturnsEmptyEvidence()
{
var collector = new PythonRuntimeEvidenceCollector();
var evidence = collector.Build();
Assert.False(evidence.HasEvidence);
Assert.Null(evidence.RuntimePythonVersion);
Assert.Null(evidence.RuntimePlatform);
Assert.Equal(0, evidence.LoadedModulesCount);
Assert.Empty(evidence.LoadedModules);
Assert.Empty(evidence.LoadedPackages);
}
[Fact]
public void Build_ModulesAreSorted()
{
var collector = new PythonRuntimeEvidenceCollector();
collector.ParseLine("""{"type": "module_import", "module": "zebra", "pid": 1}""");
collector.ParseLine("""{"type": "module_import", "module": "alpha", "pid": 1}""");
collector.ParseLine("""{"type": "module_import", "module": "middle", "pid": 1}""");
var evidence = collector.Build();
Assert.Equal(["alpha", "middle", "zebra"], evidence.LoadedModules);
}
[Fact]
public void Events_ReturnsAllCapturedEvents()
{
var collector = new PythonRuntimeEvidenceCollector();
collector.ParseLine("""{"type": "interpreter_start", "python_version": "3.11.5", "pid": 1}""");
collector.ParseLine("""{"type": "module_import", "module": "os", "pid": 1}""");
Assert.Equal(2, collector.Events.Count);
Assert.Equal(PythonRuntimeEventKind.InterpreterStart, collector.Events[0].Kind);
Assert.Equal(PythonRuntimeEventKind.ModuleImport, collector.Events[1].Kind);
}
[Fact]
public void ParseLine_ScrubbsPathsInEvents()
{
var collector = new PythonRuntimeEvidenceCollector();
var json = """{"type": "module_import", "module": "mymodule", "path": "/home/user/project/mymodule.py", "pid": 12345}""";
collector.ParseLine(json);
var moduleEvent = collector.Events[0];
Assert.Equal("[HOME]/project/mymodule.py", moduleEvent.ModulePath);
}
}

View File

@@ -0,0 +1,22 @@
{
"name": "jq",
"versions": {
"stable": "1.7"
},
"revision": 0,
"tap": "homebrew/core",
"poured_from_bottle": true,
"time": 1700000000,
"installed_as_dependency": false,
"installed_on_request": true,
"runtime_dependencies": [],
"build_dependencies": [],
"source": {
"url": "https://github.com/jqlang/jq/releases/download/jq-1.7/jq-1.7.tar.gz",
"checksum": "sha256:jq17hash"
},
"desc": "Lightweight and flexible command-line JSON processor",
"homepage": "https://jqlang.github.io/jq/",
"license": "MIT",
"arch": "arm64"
}

View File

@@ -0,0 +1,27 @@
{
"name": "openssl@3",
"versions": {
"stable": "3.1.0"
},
"revision": 0,
"tap": "homebrew/core",
"poured_from_bottle": true,
"time": 1699000000,
"installed_as_dependency": false,
"installed_on_request": true,
"runtime_dependencies": [
{
"full_name": "ca-certificates",
"version": "2023-01-10"
}
],
"build_dependencies": [],
"source": {
"url": "https://www.openssl.org/source/openssl-3.1.0.tar.gz",
"checksum": "sha256:aafde89dd0e91c3d0e87c4b4e3f4d4c9f8f5a6e2b3d4c5a6f7e8d9c0a1b2c3d4e5"
},
"desc": "Cryptography and SSL/TLS Toolkit",
"homepage": "https://openssl.org/",
"license": "Apache-2.0",
"arch": "x86_64"
}

View File

@@ -0,0 +1,31 @@
{
"name": "wget",
"versions": {
"stable": "1.21.4"
},
"revision": 1,
"tap": "homebrew/core",
"poured_from_bottle": true,
"time": 1698500000,
"installed_as_dependency": true,
"installed_on_request": false,
"runtime_dependencies": [
{
"full_name": "openssl@3",
"version": "3.1.0"
},
{
"full_name": "gettext",
"version": "0.21.1"
}
],
"build_dependencies": [],
"source": {
"url": "https://ftp.gnu.org/gnu/wget/wget-1.21.4.tar.gz",
"checksum": "sha256:abc123def456"
},
"desc": "Internet file retriever",
"homepage": "https://www.gnu.org/software/wget/",
"license": "GPL-3.0-or-later",
"arch": "x86_64"
}

View File

@@ -0,0 +1,222 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Homebrew;
using Xunit;
namespace StellaOps.Scanner.Analyzers.OS.Homebrew.Tests;
public sealed class HomebrewPackageAnalyzerTests
{
private static readonly string FixturesRoot = Path.Combine(
AppContext.BaseDirectory,
"Fixtures");
private readonly HomebrewPackageAnalyzer _analyzer;
private readonly ILogger _logger;
public HomebrewPackageAnalyzerTests()
{
_logger = NullLoggerFactory.Instance.CreateLogger<HomebrewPackageAnalyzer>();
_analyzer = new HomebrewPackageAnalyzer((ILogger<HomebrewPackageAnalyzer>)_logger);
}
private OSPackageAnalyzerContext CreateContext(string rootPath)
{
return new OSPackageAnalyzerContext(
rootPath,
workspacePath: null,
TimeProvider.System,
_logger);
}
[Fact]
public void AnalyzerId_ReturnsHomebrew()
{
Assert.Equal("homebrew", _analyzer.AnalyzerId);
}
[Fact]
public async Task AnalyzeAsync_WithValidCellar_ReturnsPackages()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.NotNull(result);
Assert.Equal("homebrew", result.AnalyzerId);
Assert.True(result.Packages.Count > 0, "Expected at least one package");
}
[Fact]
public async Task AnalyzeAsync_FindsIntelCellarPackages()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var openssl = result.Packages.FirstOrDefault(p => p.Name == "openssl@3");
Assert.NotNull(openssl);
Assert.Equal("3.1.0", openssl.Version);
Assert.Equal("x86_64", openssl.Architecture);
Assert.Contains("pkg:brew/homebrew%2Fcore/openssl%403@3.1.0", openssl.PackageUrl);
}
[Fact]
public async Task AnalyzeAsync_FindsAppleSiliconCellarPackages()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var jq = result.Packages.FirstOrDefault(p => p.Name == "jq");
Assert.NotNull(jq);
Assert.Equal("1.7", jq.Version);
Assert.Equal("arm64", jq.Architecture);
}
[Fact]
public async Task AnalyzeAsync_PackageWithRevision_IncludesRevisionInPurl()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var wget = result.Packages.FirstOrDefault(p => p.Name == "wget");
Assert.NotNull(wget);
Assert.Contains("?revision=1", wget.PackageUrl);
Assert.Equal("1", wget.Release);
}
[Fact]
public async Task AnalyzeAsync_ExtractsDependencies()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var wget = result.Packages.FirstOrDefault(p => p.Name == "wget");
Assert.NotNull(wget);
Assert.Contains("openssl@3", wget.Depends);
Assert.Contains("gettext", wget.Depends);
}
[Fact]
public async Task AnalyzeAsync_ExtractsVendorMetadata()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var openssl = result.Packages.FirstOrDefault(p => p.Name == "openssl@3");
Assert.NotNull(openssl);
Assert.Equal("homebrew/core", openssl.VendorMetadata["brew:tap"]);
Assert.Equal("true", openssl.VendorMetadata["brew:poured_from_bottle"]);
Assert.Equal("Cryptography and SSL/TLS Toolkit", openssl.VendorMetadata["description"]);
Assert.Equal("https://openssl.org/", openssl.VendorMetadata["homepage"]);
}
[Fact]
public async Task AnalyzeAsync_SetsEvidenceSourceToHomebrewCellar()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
foreach (var package in result.Packages)
{
Assert.Equal(PackageEvidenceSource.HomebrewCellar, package.EvidenceSource);
}
}
[Fact]
public async Task AnalyzeAsync_DiscoversBinFiles()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var wget = result.Packages.FirstOrDefault(p => p.Name == "wget");
Assert.NotNull(wget);
Assert.Contains(wget.Files, f => f.Path.Contains("wget"));
}
[Fact]
public async Task AnalyzeAsync_ResultsAreDeterministicallySorted()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result1 = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
var result2 = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.Equal(result1.Packages.Count, result2.Packages.Count);
for (int i = 0; i < result1.Packages.Count; i++)
{
Assert.Equal(result1.Packages[i].PackageUrl, result2.Packages[i].PackageUrl);
}
}
[Fact]
public async Task AnalyzeAsync_NoCellar_ReturnsEmptyPackages()
{
// Arrange - use temp directory without Cellar structure
var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
Directory.CreateDirectory(tempPath);
try
{
var context = CreateContext(tempPath);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.Empty(result.Packages);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task AnalyzeAsync_PopulatesTelemetry()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.NotNull(result.Telemetry);
Assert.True(result.Telemetry.PackageCount > 0);
Assert.True(result.Telemetry.Duration > TimeSpan.Zero);
}
}

View File

@@ -0,0 +1,269 @@
using System.Text;
using StellaOps.Scanner.Analyzers.OS.Homebrew;
using Xunit;
namespace StellaOps.Scanner.Analyzers.OS.Homebrew.Tests;
public sealed class HomebrewReceiptParserTests
{
private readonly HomebrewReceiptParser _parser = new();
[Fact]
public void Parse_ValidReceipt_ReturnsExpectedValues()
{
// Arrange
var json = """
{
"name": "openssl@3",
"versions": { "stable": "3.1.0" },
"revision": 0,
"tap": "homebrew/core",
"poured_from_bottle": true,
"time": 1699000000,
"installed_as_dependency": false,
"installed_on_request": true,
"runtime_dependencies": [{ "full_name": "ca-certificates", "version": "2023-01-10" }],
"desc": "Cryptography and SSL/TLS Toolkit",
"homepage": "https://openssl.org/",
"license": "Apache-2.0",
"arch": "x86_64"
}
""";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.NotNull(receipt);
Assert.Equal("openssl@3", receipt.Name);
Assert.Equal("3.1.0", receipt.Version);
Assert.Equal(0, receipt.Revision);
Assert.Equal("homebrew/core", receipt.Tap);
Assert.True(receipt.PouredFromBottle);
Assert.False(receipt.InstalledAsDependency);
Assert.True(receipt.InstalledOnRequest);
Assert.Single(receipt.RuntimeDependencies);
Assert.Equal("ca-certificates", receipt.RuntimeDependencies[0]);
Assert.Equal("Cryptography and SSL/TLS Toolkit", receipt.Description);
Assert.Equal("https://openssl.org/", receipt.Homepage);
Assert.Equal("Apache-2.0", receipt.License);
Assert.Equal("x86_64", receipt.Architecture);
}
[Fact]
public void Parse_WithRevision_ReturnsCorrectRevision()
{
// Arrange
var json = """
{
"name": "wget",
"versions": { "stable": "1.21.4" },
"revision": 1,
"tap": "homebrew/core",
"poured_from_bottle": true,
"arch": "x86_64"
}
""";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.NotNull(receipt);
Assert.Equal("wget", receipt.Name);
Assert.Equal("1.21.4", receipt.Version);
Assert.Equal(1, receipt.Revision);
}
[Fact]
public void Parse_AppleSilicon_ReturnsArm64Architecture()
{
// Arrange
var json = """
{
"name": "jq",
"versions": { "stable": "1.7" },
"revision": 0,
"tap": "homebrew/core",
"arch": "arm64"
}
""";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.NotNull(receipt);
Assert.Equal("arm64", receipt.Architecture);
}
[Fact]
public void Parse_WithSourceInfo_ExtractsSourceUrlAndChecksum()
{
// Arrange
var json = """
{
"name": "test",
"versions": { "stable": "1.0.0" },
"tap": "homebrew/core",
"source": {
"url": "https://example.com/test-1.0.0.tar.gz",
"checksum": "sha256:abcdef123456"
}
}
""";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.NotNull(receipt);
Assert.Equal("https://example.com/test-1.0.0.tar.gz", receipt.SourceUrl);
Assert.Equal("sha256:abcdef123456", receipt.SourceChecksum);
}
[Fact]
public void Parse_MultipleDependencies_SortsAlphabetically()
{
// Arrange
var json = """
{
"name": "test",
"versions": { "stable": "1.0.0" },
"tap": "homebrew/core",
"runtime_dependencies": [
{ "full_name": "zlib" },
{ "full_name": "openssl" },
{ "full_name": "libpng" }
]
}
""";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.NotNull(receipt);
Assert.Equal(3, receipt.RuntimeDependencies.Count);
Assert.Equal("libpng", receipt.RuntimeDependencies[0]);
Assert.Equal("openssl", receipt.RuntimeDependencies[1]);
Assert.Equal("zlib", receipt.RuntimeDependencies[2]);
}
[Fact]
public void Parse_InvalidJson_ReturnsNull()
{
// Arrange
var invalidJson = "{ invalid json }";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(invalidJson));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.Null(receipt);
}
[Fact]
public void Parse_EmptyJson_ReturnsNull()
{
// Arrange
var emptyJson = "{}";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(emptyJson));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.Null(receipt);
}
[Fact]
public void Parse_MissingName_ReturnsNull()
{
// Arrange
var json = """
{
"versions": { "stable": "1.0.0" },
"tap": "homebrew/core"
}
""";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.Null(receipt);
}
[Fact]
public void Parse_TappedFrom_UsesTappedFromOverTap()
{
// Arrange
var json = """
{
"name": "test",
"versions": { "stable": "1.0.0" },
"tap": "homebrew/core",
"tapped_from": "custom/tap"
}
""";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.NotNull(receipt);
Assert.Equal("custom/tap", receipt.Tap);
}
[Fact]
public void Parse_FallbackVersion_UsesVersionFieldWhenVersionsStableMissing()
{
// Arrange - older receipt format uses version field directly
var json = """
{
"name": "test",
"version": "2.0.0",
"tap": "homebrew/core"
}
""";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.NotNull(receipt);
Assert.Equal("2.0.0", receipt.Version);
}
[Fact]
public void Parse_NormalizesArchitecture_AArch64ToArm64()
{
// Arrange
var json = """
{
"name": "test",
"versions": { "stable": "1.0.0" },
"tap": "homebrew/core",
"arch": "aarch64"
}
""";
using var stream = new MemoryStream(Encoding.UTF8.GetBytes(json));
// Act
var receipt = _parser.Parse(stream);
// Assert
Assert.NotNull(receipt);
Assert.Equal("arm64", receipt.Architecture);
}
}

View File

@@ -0,0 +1,27 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<UseConcelierTestInfra>false</UseConcelierTestInfra>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageReference Include="coverlet.collector" Version="6.0.4" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.OS.Homebrew/StellaOps.Scanner.Analyzers.OS.Homebrew.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="Fixtures\**\*" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,132 @@
using StellaOps.Scanner.Analyzers.OS.MacOsBundle;
using Xunit;
namespace StellaOps.Scanner.Analyzers.OS.MacOsBundle.Tests;
public sealed class EntitlementsParserTests
{
private static readonly string FixturesRoot = Path.Combine(
AppContext.BaseDirectory,
"Fixtures");
private readonly EntitlementsParser _parser = new();
[Fact]
public void Parse_ValidEntitlements_ReturnsEntitlements()
{
// Arrange
var entPath = Path.Combine(FixturesRoot, "Applications", "SandboxedApp.app", "Contents", "_CodeSignature", "test.xcent");
// Act
var result = _parser.Parse(entPath);
// Assert
Assert.NotNull(result);
Assert.True(result.IsSandboxed);
}
[Fact]
public void Parse_DetectsHighRiskEntitlements()
{
// Arrange
var entPath = Path.Combine(FixturesRoot, "Applications", "SandboxedApp.app", "Contents", "_CodeSignature", "test.xcent");
// Act
var result = _parser.Parse(entPath);
// Assert
Assert.NotEmpty(result.HighRiskEntitlements);
Assert.Contains("com.apple.security.device.camera", result.HighRiskEntitlements);
Assert.Contains("com.apple.security.device.microphone", result.HighRiskEntitlements);
}
[Fact]
public void Parse_CategorizeEntitlements()
{
// Arrange
var entPath = Path.Combine(FixturesRoot, "Applications", "SandboxedApp.app", "Contents", "_CodeSignature", "test.xcent");
// Act
var result = _parser.Parse(entPath);
// Assert
Assert.Contains("network", result.Categories);
Assert.Contains("camera", result.Categories);
Assert.Contains("microphone", result.Categories);
Assert.Contains("filesystem", result.Categories);
Assert.Contains("sandbox", result.Categories);
}
[Fact]
public void Parse_NonExistentFile_ReturnsEmpty()
{
// Arrange
var entPath = Path.Combine(FixturesRoot, "nonexistent.xcent");
// Act
var result = _parser.Parse(entPath);
// Assert
Assert.Same(BundleEntitlements.Empty, result);
}
[Fact]
public void FindEntitlementsFile_FindsXcentFile()
{
// Arrange
var bundlePath = Path.Combine(FixturesRoot, "Applications", "SandboxedApp.app");
// Act
var result = _parser.FindEntitlementsFile(bundlePath);
// Assert
Assert.NotNull(result);
Assert.EndsWith(".xcent", result);
}
[Fact]
public void FindEntitlementsFile_NoBundlePath_ReturnsNull()
{
// Act
var result = _parser.FindEntitlementsFile("");
// Assert
Assert.Null(result);
}
[Fact]
public void FindEntitlementsFile_NoEntitlements_ReturnsNull()
{
// Arrange - bundle without entitlements
var bundlePath = Path.Combine(FixturesRoot, "Applications", "TestApp.app");
// Act
var result = _parser.FindEntitlementsFile(bundlePath);
// Assert
Assert.Null(result);
}
[Fact]
public void HasEntitlement_ReturnsTrueForExistingEntitlement()
{
// Arrange
var entPath = Path.Combine(FixturesRoot, "Applications", "SandboxedApp.app", "Contents", "_CodeSignature", "test.xcent");
var result = _parser.Parse(entPath);
// Act & Assert
Assert.True(result.HasEntitlement("com.apple.security.app-sandbox"));
Assert.True(result.HasEntitlement("com.apple.security.device.camera"));
}
[Fact]
public void HasEntitlement_ReturnsFalseForMissingEntitlement()
{
// Arrange
var entPath = Path.Combine(FixturesRoot, "Applications", "SandboxedApp.app", "Contents", "_CodeSignature", "test.xcent");
var result = _parser.Parse(entPath);
// Act & Assert
Assert.False(result.HasEntitlement("com.apple.security.nonexistent"));
}
}

View File

@@ -0,0 +1,24 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleIdentifier</key>
<string>com.stellaops.sandboxed</string>
<key>CFBundleName</key>
<string>SandboxedApp</string>
<key>CFBundleVersion</key>
<string>100</string>
<key>CFBundleShortVersionString</key>
<string>2.0.0</string>
<key>LSMinimumSystemVersion</key>
<string>13.0</string>
<key>CFBundleExecutable</key>
<string>SandboxedApp</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleSupportedPlatforms</key>
<array>
<string>MacOSX</string>
</array>
</dict>
</plist>

View File

@@ -0,0 +1,3 @@
#!/bin/bash
# Placeholder executable
echo "SandboxedApp"

View File

@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>files</key>
<dict>
<key>Contents/Info.plist</key>
<data>aGFzaA==</data>
</dict>
<key>rules</key>
<dict>
<key>^.*</key>
<true/>
</dict>
</dict>
</plist>

View File

@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>com.apple.security.app-sandbox</key>
<true/>
<key>com.apple.security.network.client</key>
<true/>
<key>com.apple.security.device.camera</key>
<true/>
<key>com.apple.security.device.microphone</key>
<true/>
<key>com.apple.security.files.user-selected.read-write</key>
<true/>
</dict>
</plist>

View File

@@ -0,0 +1,26 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleIdentifier</key>
<string>com.stellaops.testapp</string>
<key>CFBundleName</key>
<string>TestApp</string>
<key>CFBundleDisplayName</key>
<string>Test Application</string>
<key>CFBundleVersion</key>
<string>123</string>
<key>CFBundleShortVersionString</key>
<string>1.2.3</string>
<key>LSMinimumSystemVersion</key>
<string>12.0</string>
<key>CFBundleExecutable</key>
<string>TestApp</string>
<key>CFBundlePackageType</key>
<string>APPL</string>
<key>CFBundleSupportedPlatforms</key>
<array>
<string>MacOSX</string>
</array>
</dict>
</plist>

View File

@@ -0,0 +1,3 @@
#!/bin/bash
# Placeholder executable
echo "TestApp"

View File

@@ -0,0 +1,16 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>files</key>
<dict>
<key>Contents/Info.plist</key>
<data>aGFzaA==</data>
</dict>
<key>rules</key>
<dict>
<key>^.*</key>
<true/>
</dict>
</dict>
</plist>

View File

@@ -0,0 +1,115 @@
using StellaOps.Scanner.Analyzers.OS.MacOsBundle;
using Xunit;
namespace StellaOps.Scanner.Analyzers.OS.MacOsBundle.Tests;
public sealed class InfoPlistParserTests
{
private static readonly string FixturesRoot = Path.Combine(
AppContext.BaseDirectory,
"Fixtures");
private readonly InfoPlistParser _parser = new();
[Fact]
public void Parse_ValidInfoPlist_ReturnsBundleInfo()
{
// Arrange
var plistPath = Path.Combine(FixturesRoot, "Applications", "TestApp.app", "Contents", "Info.plist");
// Act
var result = _parser.Parse(plistPath);
// Assert
Assert.NotNull(result);
Assert.Equal("com.stellaops.testapp", result.BundleIdentifier);
Assert.Equal("TestApp", result.BundleName);
Assert.Equal("Test Application", result.BundleDisplayName);
Assert.Equal("123", result.Version);
Assert.Equal("1.2.3", result.ShortVersion);
}
[Fact]
public void Parse_ExtractsMinimumSystemVersion()
{
// Arrange
var plistPath = Path.Combine(FixturesRoot, "Applications", "TestApp.app", "Contents", "Info.plist");
// Act
var result = _parser.Parse(plistPath);
// Assert
Assert.NotNull(result);
Assert.Equal("12.0", result.MinimumSystemVersion);
}
[Fact]
public void Parse_ExtractsExecutable()
{
// Arrange
var plistPath = Path.Combine(FixturesRoot, "Applications", "TestApp.app", "Contents", "Info.plist");
// Act
var result = _parser.Parse(plistPath);
// Assert
Assert.NotNull(result);
Assert.Equal("TestApp", result.Executable);
}
[Fact]
public void Parse_ExtractsSupportedPlatforms()
{
// Arrange
var plistPath = Path.Combine(FixturesRoot, "Applications", "TestApp.app", "Contents", "Info.plist");
// Act
var result = _parser.Parse(plistPath);
// Assert
Assert.NotNull(result);
Assert.Single(result.SupportedPlatforms);
Assert.Contains("MacOSX", result.SupportedPlatforms);
}
[Fact]
public void Parse_NonExistentFile_ReturnsNull()
{
// Arrange
var plistPath = Path.Combine(FixturesRoot, "nonexistent.plist");
// Act
var result = _parser.Parse(plistPath);
// Assert
Assert.Null(result);
}
[Fact]
public void Parse_MissingBundleIdentifier_ReturnsNull()
{
// Arrange - Create a temp file without CFBundleIdentifier
var tempPath = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.plist");
File.WriteAllText(tempPath, @"<?xml version=""1.0"" encoding=""UTF-8""?>
<!DOCTYPE plist PUBLIC ""-//Apple//DTD PLIST 1.0//EN"" ""http://www.apple.com/DTDs/PropertyList-1.0.dtd"">
<plist version=""1.0"">
<dict>
<key>CFBundleName</key>
<string>TestApp</string>
</dict>
</plist>");
try
{
// Act
var result = _parser.Parse(tempPath);
// Assert
Assert.Null(result);
}
finally
{
File.Delete(tempPath);
}
}
}

View File

@@ -0,0 +1,322 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scanner.Analyzers.OS.MacOsBundle;
using Xunit;
namespace StellaOps.Scanner.Analyzers.OS.MacOsBundle.Tests;
public sealed class MacOsBundleAnalyzerTests
{
private static readonly string FixturesRoot = Path.Combine(
AppContext.BaseDirectory,
"Fixtures");
private readonly MacOsBundleAnalyzer _analyzer;
private readonly ILogger _logger;
public MacOsBundleAnalyzerTests()
{
_logger = NullLoggerFactory.Instance.CreateLogger<MacOsBundleAnalyzer>();
_analyzer = new MacOsBundleAnalyzer((ILogger<MacOsBundleAnalyzer>)_logger);
}
private OSPackageAnalyzerContext CreateContext(string rootPath)
{
return new OSPackageAnalyzerContext(
rootPath,
workspacePath: null,
TimeProvider.System,
_logger);
}
[Fact]
public void AnalyzerId_ReturnsMacosBundleIdentifier()
{
Assert.Equal("macos-bundle", _analyzer.AnalyzerId);
}
[Fact]
public async Task AnalyzeAsync_WithValidBundles_ReturnsPackages()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.NotNull(result);
Assert.Equal("macos-bundle", result.AnalyzerId);
Assert.True(result.Packages.Count > 0, "Expected at least one bundle");
}
[Fact]
public async Task AnalyzeAsync_FindsTestApp()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var testApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.testapp");
Assert.NotNull(testApp);
Assert.Equal("1.2.3", testApp.Version);
Assert.Equal("Test Application", testApp.Name);
}
[Fact]
public async Task AnalyzeAsync_ExtractsVersionCorrectly()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var testApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.testapp");
Assert.NotNull(testApp);
// ShortVersion takes precedence
Assert.Equal("1.2.3", testApp.Version);
// Build number goes to release
Assert.Equal("123", testApp.Release);
}
[Fact]
public async Task AnalyzeAsync_BuildsCorrectPurl()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var testApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.testapp");
Assert.NotNull(testApp);
Assert.Contains("pkg:generic/macos-app/com.stellaops.testapp@1.2.3", testApp.PackageUrl);
}
[Fact]
public async Task AnalyzeAsync_ExtractsVendorFromBundleId()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var testApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.testapp");
Assert.NotNull(testApp);
Assert.Equal("stellaops", testApp.SourcePackage);
}
[Fact]
public async Task AnalyzeAsync_SetsEvidenceSourceToMacOsBundle()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
foreach (var package in result.Packages)
{
Assert.Equal(PackageEvidenceSource.MacOsBundle, package.EvidenceSource);
}
}
[Fact]
public async Task AnalyzeAsync_ExtractsVendorMetadata()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var testApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.testapp");
Assert.NotNull(testApp);
Assert.Equal("com.stellaops.testapp", testApp.VendorMetadata["macos:bundle_id"]);
Assert.Equal("APPL", testApp.VendorMetadata["macos:bundle_type"]);
Assert.Equal("12.0", testApp.VendorMetadata["macos:min_os_version"]);
Assert.Equal("TestApp", testApp.VendorMetadata["macos:executable"]);
Assert.Equal("MacOSX", testApp.VendorMetadata["macos:platforms"]);
}
[Fact]
public async Task AnalyzeAsync_IncludesCodeResourcesHash()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var testApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.testapp");
Assert.NotNull(testApp);
Assert.True(testApp.VendorMetadata.ContainsKey("macos:code_resources_hash"));
var hash = testApp.VendorMetadata["macos:code_resources_hash"];
Assert.StartsWith("sha256:", hash);
}
[Fact]
public async Task AnalyzeAsync_DetectsSandboxedApp()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var sandboxedApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.sandboxed");
Assert.NotNull(sandboxedApp);
Assert.Equal("true", sandboxedApp.VendorMetadata["macos:sandboxed"]);
}
[Fact]
public async Task AnalyzeAsync_DetectsHighRiskEntitlements()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var sandboxedApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.sandboxed");
Assert.NotNull(sandboxedApp);
Assert.True(sandboxedApp.VendorMetadata.ContainsKey("macos:high_risk_entitlements"));
var highRisk = sandboxedApp.VendorMetadata["macos:high_risk_entitlements"];
// Full entitlement keys are stored
Assert.Contains("com.apple.security.device.camera", highRisk);
Assert.Contains("com.apple.security.device.microphone", highRisk);
}
[Fact]
public async Task AnalyzeAsync_DetectsCapabilityCategories()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var sandboxedApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.sandboxed");
Assert.NotNull(sandboxedApp);
Assert.True(sandboxedApp.VendorMetadata.ContainsKey("macos:capability_categories"));
var categories = sandboxedApp.VendorMetadata["macos:capability_categories"];
Assert.Contains("network", categories);
Assert.Contains("camera", categories);
Assert.Contains("microphone", categories);
Assert.Contains("filesystem", categories);
Assert.Contains("sandbox", categories);
}
[Fact]
public async Task AnalyzeAsync_IncludesFileEvidence()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var testApp = result.Packages.FirstOrDefault(p =>
p.VendorMetadata.TryGetValue("macos:bundle_id", out var id) &&
id == "com.stellaops.testapp");
Assert.NotNull(testApp);
Assert.True(testApp.Files.Count > 0);
var executable = testApp.Files.FirstOrDefault(f => f.Path.Contains("MacOS/TestApp"));
Assert.NotNull(executable);
Assert.False(executable.IsConfigFile);
var infoPlist = testApp.Files.FirstOrDefault(f => f.Path.Contains("Info.plist"));
Assert.NotNull(infoPlist);
Assert.True(infoPlist.IsConfigFile);
}
[Fact]
public async Task AnalyzeAsync_ResultsAreDeterministicallySorted()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result1 = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
var result2 = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.Equal(result1.Packages.Count, result2.Packages.Count);
for (int i = 0; i < result1.Packages.Count; i++)
{
Assert.Equal(result1.Packages[i].PackageUrl, result2.Packages[i].PackageUrl);
}
}
[Fact]
public async Task AnalyzeAsync_NoApplicationsDirectory_ReturnsEmptyPackages()
{
// Arrange - use temp directory without Applications
var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
Directory.CreateDirectory(tempPath);
try
{
var context = CreateContext(tempPath);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.Empty(result.Packages);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task AnalyzeAsync_PopulatesTelemetry()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.NotNull(result.Telemetry);
Assert.True(result.Telemetry.PackageCount > 0);
Assert.True(result.Telemetry.Duration > TimeSpan.Zero);
}
}

View File

@@ -0,0 +1,27 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<UseConcelierTestInfra>false</UseConcelierTestInfra>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageReference Include="coverlet.collector" Version="6.0.4" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.OS.MacOsBundle/StellaOps.Scanner.Analyzers.OS.MacOsBundle.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="Fixtures\**\*" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
</Project>

View File

@@ -0,0 +1,18 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>PackageIdentifier</key>
<string>com.apple.pkg.Safari</string>
<key>PackageVersion</key>
<string>17.1</string>
<key>InstallDate</key>
<date>2024-01-15T12:00:00Z</date>
<key>InstallPrefixPath</key>
<string>/</string>
<key>VolumePath</key>
<string>/</string>
<key>InstallProcessName</key>
<string>installer</string>
</dict>
</plist>

View File

@@ -0,0 +1,14 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>PackageIdentifier</key>
<string>com.example.app</string>
<key>PackageVersion</key>
<string>2.5.0</string>
<key>VolumePath</key>
<string>/</string>
<key>InstallProcessName</key>
<string>installer</string>
</dict>
</plist>

View File

@@ -0,0 +1,171 @@
using Microsoft.Extensions.Logging;
using Microsoft.Extensions.Logging.Abstractions;
using StellaOps.Scanner.Analyzers.OS.Pkgutil;
using Xunit;
namespace StellaOps.Scanner.Analyzers.OS.Pkgutil.Tests;
public sealed class PkgutilPackageAnalyzerTests
{
private static readonly string FixturesRoot = Path.Combine(
AppContext.BaseDirectory,
"Fixtures");
private readonly PkgutilPackageAnalyzer _analyzer;
private readonly ILogger _logger;
public PkgutilPackageAnalyzerTests()
{
_logger = NullLoggerFactory.Instance.CreateLogger<PkgutilPackageAnalyzer>();
_analyzer = new PkgutilPackageAnalyzer((ILogger<PkgutilPackageAnalyzer>)_logger);
}
private OSPackageAnalyzerContext CreateContext(string rootPath)
{
return new OSPackageAnalyzerContext(
rootPath,
workspacePath: null,
TimeProvider.System,
_logger);
}
[Fact]
public void AnalyzerId_ReturnsPkgutil()
{
Assert.Equal("pkgutil", _analyzer.AnalyzerId);
}
[Fact]
public async Task AnalyzeAsync_WithValidReceipts_ReturnsPackages()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.NotNull(result);
Assert.Equal("pkgutil", result.AnalyzerId);
Assert.True(result.Packages.Count > 0, "Expected at least one package");
}
[Fact]
public async Task AnalyzeAsync_FindsSafariPackage()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var safari = result.Packages.FirstOrDefault(p => p.Name == "Safari");
Assert.NotNull(safari);
Assert.Equal("17.1", safari.Version);
Assert.Contains("pkg:generic/apple/com.apple.pkg.Safari@17.1", safari.PackageUrl);
}
[Fact]
public async Task AnalyzeAsync_ExtractsVendorFromIdentifier()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var safari = result.Packages.FirstOrDefault(p => p.Name == "Safari");
Assert.NotNull(safari);
Assert.Equal("apple", safari.SourcePackage);
}
[Fact]
public async Task AnalyzeAsync_SetsEvidenceSourceToPkgutilReceipt()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
foreach (var package in result.Packages)
{
Assert.Equal(PackageEvidenceSource.PkgutilReceipt, package.EvidenceSource);
}
}
[Fact]
public async Task AnalyzeAsync_ExtractsVendorMetadata()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
var safari = result.Packages.FirstOrDefault(p => p.Name == "Safari");
Assert.NotNull(safari);
Assert.Equal("com.apple.pkg.Safari", safari.VendorMetadata["pkgutil:identifier"]);
Assert.Equal("/", safari.VendorMetadata["pkgutil:volume"]);
}
[Fact]
public async Task AnalyzeAsync_ResultsAreDeterministicallySorted()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result1 = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
var result2 = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.Equal(result1.Packages.Count, result2.Packages.Count);
for (int i = 0; i < result1.Packages.Count; i++)
{
Assert.Equal(result1.Packages[i].PackageUrl, result2.Packages[i].PackageUrl);
}
}
[Fact]
public async Task AnalyzeAsync_NoReceiptsDirectory_ReturnsEmptyPackages()
{
// Arrange - use temp directory without receipts
var tempPath = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
Directory.CreateDirectory(tempPath);
try
{
var context = CreateContext(tempPath);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.Empty(result.Packages);
}
finally
{
Directory.Delete(tempPath, recursive: true);
}
}
[Fact]
public async Task AnalyzeAsync_PopulatesTelemetry()
{
// Arrange
var context = CreateContext(FixturesRoot);
// Act
var result = await _analyzer.AnalyzeAsync(context, CancellationToken.None);
// Assert
Assert.NotNull(result.Telemetry);
Assert.True(result.Telemetry.PackageCount > 0);
Assert.True(result.Telemetry.Duration > TimeSpan.Zero);
}
}

View File

@@ -0,0 +1,27 @@
<?xml version='1.0' encoding='utf-8'?>
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net10.0</TargetFramework>
<LangVersion>preview</LangVersion>
<Nullable>enable</Nullable>
<ImplicitUsings>enable</ImplicitUsings>
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
<UseConcelierTestInfra>false</UseConcelierTestInfra>
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
</ItemGroup>
<ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.0" />
<PackageReference Include="xunit" Version="2.9.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.8.2" />
<PackageReference Include="coverlet.collector" Version="6.0.4" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.OS/StellaOps.Scanner.Analyzers.OS.csproj" />
<ProjectReference Include="../../__Libraries/StellaOps.Scanner.Analyzers.OS.Pkgutil/StellaOps.Scanner.Analyzers.OS.Pkgutil.csproj" />
</ItemGroup>
<ItemGroup>
<None Include="Fixtures\**\*" CopyToOutputDirectory="PreserveNewest" />
</ItemGroup>
</Project>