feat(docs): Add comprehensive documentation for Vexer, Vulnerability Explorer, and Zastava modules
- Introduced AGENTS.md, README.md, TASKS.md, and implementation_plan.md for Vexer, detailing mission, responsibilities, key components, and operational notes. - Established similar documentation structure for Vulnerability Explorer and Zastava modules, including their respective workflows, integrations, and observability notes. - Created risk scoring profiles documentation outlining the core workflow, factor model, governance, and deliverables. - Ensured all modules adhere to the Aggregation-Only Contract and maintain determinism and provenance in outputs.
This commit is contained in:
master
@@ -1,20 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="../../src/Concelier/StellaOps.Concelier.PluginBinaries/StellaOps.Concelier.Connector.Osv/StellaOps.Concelier.Connector.Osv.csproj" />
|
||||
<ProjectReference Include="../../src/Concelier/StellaOps.Concelier.PluginBinaries/StellaOps.Concelier.Connector.Ghsa/StellaOps.Concelier.Connector.Ghsa.csproj" />
|
||||
<ProjectReference Include="../../src/Concelier/StellaOps.Concelier.PluginBinaries/StellaOps.Concelier.Connector.Nvd/StellaOps.Concelier.Connector.Nvd.csproj" />
|
||||
<ProjectReference Include="../../src/Concelier/StellaOps.Concelier.PluginBinaries/StellaOps.Concelier.Connector.Common/StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="../../src/Concelier/__Libraries/StellaOps.Concelier.Storage.Mongo/StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
<ProjectReference Include="../../src/Concelier/__Libraries/StellaOps.Concelier.Models/StellaOps.Concelier.Models.csproj" />
|
||||
<ProjectReference Include="../../src/Concelier/__Libraries/StellaOps.Concelier.Testing/StellaOps.Concelier.Testing.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -1,378 +0,0 @@
|
||||
using System.Linq;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using MongoDB.Bson;
|
||||
using StellaOps.Concelier.Models;
|
||||
using StellaOps.Concelier.Connector.Ghsa;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Ghsa.Internal;
|
||||
using StellaOps.Concelier.Connector.Osv.Internal;
|
||||
using StellaOps.Concelier.Connector.Osv;
|
||||
using StellaOps.Concelier.Connector.Nvd;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
using StellaOps.Concelier.Storage.Mongo.Dtos;
|
||||
|
||||
var serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
};
|
||||
|
||||
var projectRoot = Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "..", ".."));
|
||||
|
||||
var osvFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Concelier.Connector.Osv.Tests", "Fixtures");
|
||||
var ghsaFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Concelier.Connector.Ghsa.Tests", "Fixtures");
|
||||
var nvdFixturesPath = Path.Combine(projectRoot, "src", "StellaOps.Concelier.Connector.Nvd.Tests", "Nvd", "Fixtures");
|
||||
|
||||
RewriteOsvFixtures(osvFixturesPath);
|
||||
RewriteSnapshotFixtures(osvFixturesPath);
|
||||
RewriteGhsaFixtures(osvFixturesPath);
|
||||
RewriteCreditParityFixtures(ghsaFixturesPath, nvdFixturesPath);
|
||||
return;
|
||||
|
||||
void RewriteOsvFixtures(string fixturesPath)
|
||||
{
|
||||
var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-osv.json");
|
||||
if (!File.Exists(rawPath))
|
||||
{
|
||||
Console.WriteLine($"[FixtureUpdater] OSV raw fixture missing: {rawPath}");
|
||||
return;
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(File.ReadAllText(rawPath));
|
||||
var advisories = new List<Advisory>();
|
||||
foreach (var element in document.RootElement.EnumerateArray())
|
||||
{
|
||||
var dto = JsonSerializer.Deserialize<OsvVulnerabilityDto>(element.GetRawText(), serializerOptions);
|
||||
if (dto is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var ecosystem = dto.Affected?.FirstOrDefault()?.Package?.Ecosystem ?? "unknown";
|
||||
var uri = new Uri($"https://osv.dev/vulnerability/{dto.Id}");
|
||||
var documentRecord = new DocumentRecord(
|
||||
Guid.NewGuid(),
|
||||
OsvConnectorPlugin.SourceName,
|
||||
uri.ToString(),
|
||||
DateTimeOffset.UtcNow,
|
||||
"fixture-sha",
|
||||
DocumentStatuses.PendingMap,
|
||||
"application/json",
|
||||
null,
|
||||
new Dictionary<string, string>(StringComparer.Ordinal)
|
||||
{
|
||||
["osv.ecosystem"] = ecosystem,
|
||||
},
|
||||
null,
|
||||
DateTimeOffset.UtcNow,
|
||||
null,
|
||||
null);
|
||||
|
||||
var payload = BsonDocument.Parse(element.GetRawText());
|
||||
var dtoRecord = new DtoRecord(
|
||||
Guid.NewGuid(),
|
||||
documentRecord.Id,
|
||||
OsvConnectorPlugin.SourceName,
|
||||
"osv.v1",
|
||||
payload,
|
||||
DateTimeOffset.UtcNow);
|
||||
|
||||
var advisory = OsvMapper.Map(dto, documentRecord, dtoRecord, ecosystem);
|
||||
advisories.Add(advisory);
|
||||
}
|
||||
|
||||
advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal));
|
||||
var snapshot = SnapshotSerializer.ToSnapshot(advisories);
|
||||
File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.osv.json"), snapshot);
|
||||
Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.osv.json")}");
|
||||
}
|
||||
|
||||
void RewriteSnapshotFixtures(string fixturesPath)
|
||||
{
|
||||
var baselinePublished = new DateTimeOffset(2025, 1, 5, 12, 0, 0, TimeSpan.Zero);
|
||||
var baselineModified = new DateTimeOffset(2025, 1, 8, 6, 30, 0, TimeSpan.Zero);
|
||||
var baselineFetched = new DateTimeOffset(2025, 1, 8, 7, 0, 0, TimeSpan.Zero);
|
||||
|
||||
var cases = new (string Ecosystem, string Purl, string PackageName, string SnapshotFile)[]
|
||||
{
|
||||
("npm", "pkg:npm/%40scope%2Fleft-pad", "@scope/left-pad", "osv-npm.snapshot.json"),
|
||||
("PyPI", "pkg:pypi/requests", "requests", "osv-pypi.snapshot.json"),
|
||||
};
|
||||
|
||||
foreach (var (ecosystem, purl, packageName, snapshotFile) in cases)
|
||||
{
|
||||
var dto = new OsvVulnerabilityDto
|
||||
{
|
||||
Id = $"OSV-2025-{ecosystem}-0001",
|
||||
Summary = $"{ecosystem} package vulnerability",
|
||||
Details = $"Detailed description for {ecosystem} package {packageName}.",
|
||||
Published = baselinePublished,
|
||||
Modified = baselineModified,
|
||||
Aliases = new[] { $"CVE-2025-11{ecosystem.Length}", $"GHSA-{ecosystem.Length}abc-{ecosystem.Length}def-{ecosystem.Length}ghi" },
|
||||
Related = new[] { $"OSV-RELATED-{ecosystem}-42" },
|
||||
References = new[]
|
||||
{
|
||||
new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/advisory", Type = "ADVISORY" },
|
||||
new OsvReferenceDto { Url = $"https://example.com/{ecosystem}/fix", Type = "FIX" },
|
||||
},
|
||||
Severity = new[]
|
||||
{
|
||||
new OsvSeverityDto { Type = "CVSS_V3", Score = "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H" },
|
||||
},
|
||||
Affected = new[]
|
||||
{
|
||||
new OsvAffectedPackageDto
|
||||
{
|
||||
Package = new OsvPackageDto
|
||||
{
|
||||
Ecosystem = ecosystem,
|
||||
Name = packageName,
|
||||
Purl = purl,
|
||||
},
|
||||
Ranges = new[]
|
||||
{
|
||||
new OsvRangeDto
|
||||
{
|
||||
Type = "SEMVER",
|
||||
Events = new[]
|
||||
{
|
||||
new OsvEventDto { Introduced = "0" },
|
||||
new OsvEventDto { Fixed = "2.0.0" },
|
||||
},
|
||||
},
|
||||
},
|
||||
Versions = new[] { "1.0.0", "1.5.0" },
|
||||
EcosystemSpecific = JsonDocument.Parse("{\"severity\":\"high\"}").RootElement.Clone(),
|
||||
},
|
||||
},
|
||||
DatabaseSpecific = JsonDocument.Parse("{\"source\":\"osv.dev\"}").RootElement.Clone(),
|
||||
};
|
||||
|
||||
var document = new DocumentRecord(
|
||||
Guid.NewGuid(),
|
||||
OsvConnectorPlugin.SourceName,
|
||||
$"https://osv.dev/vulnerability/{dto.Id}",
|
||||
baselineFetched,
|
||||
"fixture-sha",
|
||||
DocumentStatuses.PendingParse,
|
||||
"application/json",
|
||||
null,
|
||||
new Dictionary<string, string>(StringComparer.Ordinal) { ["osv.ecosystem"] = ecosystem },
|
||||
null,
|
||||
baselineModified,
|
||||
null);
|
||||
|
||||
var payload = BsonDocument.Parse(JsonSerializer.Serialize(dto, serializerOptions));
|
||||
var dtoRecord = new DtoRecord(Guid.NewGuid(), document.Id, OsvConnectorPlugin.SourceName, "osv.v1", payload, baselineModified);
|
||||
|
||||
var advisory = OsvMapper.Map(dto, document, dtoRecord, ecosystem);
|
||||
var snapshot = SnapshotSerializer.ToSnapshot(advisory);
|
||||
File.WriteAllText(Path.Combine(fixturesPath, snapshotFile), snapshot);
|
||||
Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, snapshotFile)}");
|
||||
}
|
||||
}
|
||||
|
||||
void RewriteGhsaFixtures(string fixturesPath)
|
||||
{
|
||||
var rawPath = Path.Combine(fixturesPath, "osv-ghsa.raw-ghsa.json");
|
||||
if (!File.Exists(rawPath))
|
||||
{
|
||||
Console.WriteLine($"[FixtureUpdater] GHSA raw fixture missing: {rawPath}");
|
||||
return;
|
||||
}
|
||||
|
||||
JsonDocument document;
|
||||
try
|
||||
{
|
||||
document = JsonDocument.Parse(File.ReadAllText(rawPath));
|
||||
}
|
||||
catch (JsonException ex)
|
||||
{
|
||||
Console.WriteLine($"[FixtureUpdater] Failed to parse GHSA raw fixture '{rawPath}': {ex.Message}");
|
||||
return;
|
||||
}
|
||||
using (document)
|
||||
{
|
||||
var advisories = new List<Advisory>();
|
||||
foreach (var element in document.RootElement.EnumerateArray())
|
||||
{
|
||||
GhsaRecordDto dto;
|
||||
try
|
||||
{
|
||||
dto = GhsaRecordParser.Parse(Encoding.UTF8.GetBytes(element.GetRawText()));
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
var uri = new Uri($"https://github.com/advisories/{dto.GhsaId}");
|
||||
var documentRecord = new DocumentRecord(
|
||||
Guid.NewGuid(),
|
||||
GhsaConnectorPlugin.SourceName,
|
||||
uri.ToString(),
|
||||
DateTimeOffset.UtcNow,
|
||||
"fixture-sha",
|
||||
DocumentStatuses.PendingMap,
|
||||
"application/json",
|
||||
null,
|
||||
new Dictionary<string, string>(StringComparer.Ordinal),
|
||||
null,
|
||||
DateTimeOffset.UtcNow,
|
||||
null,
|
||||
null);
|
||||
|
||||
var advisory = GhsaMapper.Map(dto, documentRecord, DateTimeOffset.UtcNow);
|
||||
advisories.Add(advisory);
|
||||
}
|
||||
|
||||
advisories.Sort((left, right) => string.Compare(left.AdvisoryKey, right.AdvisoryKey, StringComparison.Ordinal));
|
||||
var snapshot = SnapshotSerializer.ToSnapshot(advisories);
|
||||
File.WriteAllText(Path.Combine(fixturesPath, "osv-ghsa.ghsa.json"), snapshot);
|
||||
Console.WriteLine($"[FixtureUpdater] Updated {Path.Combine(fixturesPath, "osv-ghsa.ghsa.json")}");
|
||||
}
|
||||
}
|
||||
|
||||
void RewriteCreditParityFixtures(string ghsaFixturesPath, string nvdFixturesPath)
|
||||
{
|
||||
Directory.CreateDirectory(ghsaFixturesPath);
|
||||
Directory.CreateDirectory(nvdFixturesPath);
|
||||
|
||||
var advisoryKeyGhsa = "GHSA-credit-parity";
|
||||
var advisoryKeyNvd = "CVE-2025-5555";
|
||||
var recordedAt = new DateTimeOffset(2025, 10, 10, 15, 0, 0, TimeSpan.Zero);
|
||||
var published = new DateTimeOffset(2025, 10, 9, 18, 30, 0, TimeSpan.Zero);
|
||||
var modified = new DateTimeOffset(2025, 10, 10, 12, 0, 0, TimeSpan.Zero);
|
||||
|
||||
AdvisoryCredit[] CreateCredits(string source) =>
|
||||
[
|
||||
CreateCredit("Alice Researcher", "reporter", new[] { "mailto:alice.researcher@example.com" }, source),
|
||||
CreateCredit("Bob Maintainer", "remediation_developer", new[] { "https://github.com/acme/bob-maintainer" }, source)
|
||||
];
|
||||
|
||||
AdvisoryCredit CreateCredit(string displayName, string role, IReadOnlyList<string> contacts, string source)
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
source,
|
||||
"credit",
|
||||
$"{source}:{displayName.ToLowerInvariant().Replace(' ', '-')}",
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Credits });
|
||||
|
||||
return new AdvisoryCredit(displayName, role, contacts, provenance);
|
||||
}
|
||||
|
||||
AdvisoryReference[] CreateReferences(string sourceName, params (string Url, string Kind)[] entries)
|
||||
{
|
||||
if (entries is null || entries.Length == 0)
|
||||
{
|
||||
return Array.Empty<AdvisoryReference>();
|
||||
}
|
||||
|
||||
var references = new List<AdvisoryReference>(entries.Length);
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
var provenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"reference",
|
||||
entry.Url,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.References });
|
||||
|
||||
references.Add(new AdvisoryReference(
|
||||
entry.Url,
|
||||
entry.Kind,
|
||||
sourceTag: null,
|
||||
summary: null,
|
||||
provenance));
|
||||
}
|
||||
|
||||
return references.ToArray();
|
||||
}
|
||||
|
||||
Advisory CreateAdvisory(
|
||||
string sourceName,
|
||||
string advisoryKey,
|
||||
IEnumerable<string> aliases,
|
||||
AdvisoryCredit[] credits,
|
||||
AdvisoryReference[] references,
|
||||
string documentValue)
|
||||
{
|
||||
var documentProvenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"document",
|
||||
documentValue,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory });
|
||||
var mappingProvenance = new AdvisoryProvenance(
|
||||
sourceName,
|
||||
"mapping",
|
||||
advisoryKey,
|
||||
recordedAt,
|
||||
new[] { ProvenanceFieldMasks.Advisory });
|
||||
|
||||
return new Advisory(
|
||||
advisoryKey,
|
||||
"Credit parity regression fixture",
|
||||
"Credit parity regression fixture",
|
||||
"en",
|
||||
published,
|
||||
modified,
|
||||
"moderate",
|
||||
exploitKnown: false,
|
||||
aliases,
|
||||
credits,
|
||||
references,
|
||||
Array.Empty<AffectedPackage>(),
|
||||
Array.Empty<CvssMetric>(),
|
||||
new[] { documentProvenance, mappingProvenance });
|
||||
}
|
||||
|
||||
var ghsa = CreateAdvisory(
|
||||
"ghsa",
|
||||
advisoryKeyGhsa,
|
||||
new[] { advisoryKeyGhsa, advisoryKeyNvd },
|
||||
CreateCredits("ghsa"),
|
||||
CreateReferences(
|
||||
"ghsa",
|
||||
( $"https://github.com/advisories/{advisoryKeyGhsa}", "advisory"),
|
||||
( "https://example.com/ghsa/patch", "patch")),
|
||||
$"security/advisories/{advisoryKeyGhsa}");
|
||||
|
||||
var osv = CreateAdvisory(
|
||||
OsvConnectorPlugin.SourceName,
|
||||
advisoryKeyGhsa,
|
||||
new[] { advisoryKeyGhsa, advisoryKeyNvd },
|
||||
CreateCredits(OsvConnectorPlugin.SourceName),
|
||||
CreateReferences(
|
||||
OsvConnectorPlugin.SourceName,
|
||||
( $"https://github.com/advisories/{advisoryKeyGhsa}", "advisory"),
|
||||
( $"https://osv.dev/vulnerability/{advisoryKeyGhsa}", "advisory")),
|
||||
$"https://osv.dev/vulnerability/{advisoryKeyGhsa}");
|
||||
|
||||
var nvd = CreateAdvisory(
|
||||
NvdConnectorPlugin.SourceName,
|
||||
advisoryKeyNvd,
|
||||
new[] { advisoryKeyNvd, advisoryKeyGhsa },
|
||||
CreateCredits(NvdConnectorPlugin.SourceName),
|
||||
CreateReferences(
|
||||
NvdConnectorPlugin.SourceName,
|
||||
( $"https://services.nvd.nist.gov/vuln/detail/{advisoryKeyNvd}", "advisory"),
|
||||
( "https://example.com/nvd/reference", "report")),
|
||||
$"https://services.nvd.nist.gov/vuln/detail/{advisoryKeyNvd}");
|
||||
|
||||
var ghsaSnapshot = SnapshotSerializer.ToSnapshot(ghsa);
|
||||
var osvSnapshot = SnapshotSerializer.ToSnapshot(osv);
|
||||
var nvdSnapshot = SnapshotSerializer.ToSnapshot(nvd);
|
||||
|
||||
File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.ghsa.json"), ghsaSnapshot);
|
||||
File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.osv.json"), osvSnapshot);
|
||||
File.WriteAllText(Path.Combine(ghsaFixturesPath, "credit-parity.nvd.json"), nvdSnapshot);
|
||||
|
||||
File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.ghsa.json"), ghsaSnapshot);
|
||||
File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.osv.json"), osvSnapshot);
|
||||
File.WriteAllText(Path.Combine(nvdFixturesPath, "credit-parity.nvd.json"), nvdSnapshot);
|
||||
|
||||
Console.WriteLine($"[FixtureUpdater] Updated credit parity fixtures under {ghsaFixturesPath} and {nvdFixturesPath}");
|
||||
}
|
||||
@@ -1,18 +0,0 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="10.0.0-rc.2.25502.107" />
|
||||
<PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="10.0.0-rc.2.25502.107" />
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\StellaOps.Scanner.Analyzers.Lang\StellaOps.Scanner.Analyzers.Lang.csproj" />
|
||||
<ProjectReference Include="..\..\src\StellaOps.Scanner.Core\StellaOps.Scanner.Core.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,348 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Diagnostics;
|
||||
using System.Reflection;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Scanner.Analyzers.Lang;
|
||||
using StellaOps.Scanner.Analyzers.Lang.Plugin;
|
||||
using StellaOps.Scanner.Core.Security;
|
||||
|
||||
internal sealed record SmokeScenario(string Name, string[] UsageHintRelatives)
|
||||
{
|
||||
public IReadOnlyList<string> ResolveUsageHints(string scenarioRoot)
|
||||
=> UsageHintRelatives.Select(relative => Path.GetFullPath(Path.Combine(scenarioRoot, relative))).ToArray();
|
||||
}
|
||||
|
||||
internal sealed class SmokeOptions
|
||||
{
|
||||
public string RepoRoot { get; set; } = Directory.GetCurrentDirectory();
|
||||
public string PluginDirectoryName { get; set; } = "StellaOps.Scanner.Analyzers.Lang.Python";
|
||||
public string FixtureRelativePath { get; set; } = Path.Combine("src", "StellaOps.Scanner.Analyzers.Lang.Python.Tests", "Fixtures", "lang", "python");
|
||||
|
||||
public static SmokeOptions Parse(string[] args)
|
||||
{
|
||||
var options = new SmokeOptions();
|
||||
|
||||
for (var index = 0; index < args.Length; index++)
|
||||
{
|
||||
var current = args[index];
|
||||
switch (current)
|
||||
{
|
||||
case "--repo-root":
|
||||
case "-r":
|
||||
options.RepoRoot = RequireValue(args, ref index, current);
|
||||
break;
|
||||
case "--plugin-directory":
|
||||
case "-p":
|
||||
options.PluginDirectoryName = RequireValue(args, ref index, current);
|
||||
break;
|
||||
case "--fixture-path":
|
||||
case "-f":
|
||||
options.FixtureRelativePath = RequireValue(args, ref index, current);
|
||||
break;
|
||||
case "--help":
|
||||
case "-h":
|
||||
PrintUsage();
|
||||
Environment.Exit(0);
|
||||
break;
|
||||
default:
|
||||
throw new ArgumentException($"Unknown argument '{current}'. Use --help for usage.");
|
||||
}
|
||||
}
|
||||
|
||||
options.RepoRoot = Path.GetFullPath(options.RepoRoot);
|
||||
return options;
|
||||
}
|
||||
|
||||
private static string RequireValue(string[] args, ref int index, string switchName)
|
||||
{
|
||||
if (index + 1 >= args.Length)
|
||||
{
|
||||
throw new ArgumentException($"Missing value for '{switchName}'.");
|
||||
}
|
||||
|
||||
index++;
|
||||
var value = args[index];
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
throw new ArgumentException($"Value for '{switchName}' cannot be empty.");
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
private static void PrintUsage()
|
||||
{
|
||||
Console.WriteLine("Language Analyzer Smoke Harness");
|
||||
Console.WriteLine("Usage: dotnet run --project tools/LanguageAnalyzerSmoke -- [options]");
|
||||
Console.WriteLine();
|
||||
Console.WriteLine("Options:");
|
||||
Console.WriteLine(" -r, --repo-root <path> Repository root (defaults to current working directory)");
|
||||
Console.WriteLine(" -p, --plugin-directory <name> Analyzer plug-in directory under plugins/scanner/analyzers/lang (defaults to StellaOps.Scanner.Analyzers.Lang.Python)");
|
||||
Console.WriteLine(" -f, --fixture-path <path> Relative path to fixtures root (defaults to src/Scanner/__Tests/StellaOps.Scanner.Analyzers.Lang.Python.Tests/Fixtures/lang/python)");
|
||||
Console.WriteLine(" -h, --help Show usage information");
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record PluginManifest
|
||||
{
|
||||
[JsonPropertyName("schemaVersion")]
|
||||
public string SchemaVersion { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("id")]
|
||||
public string Id { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("displayName")]
|
||||
public string DisplayName { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("version")]
|
||||
public string Version { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("requiresRestart")]
|
||||
public bool RequiresRestart { get; init; }
|
||||
|
||||
[JsonPropertyName("entryPoint")]
|
||||
public PluginEntryPoint EntryPoint { get; init; } = new();
|
||||
|
||||
[JsonPropertyName("capabilities")]
|
||||
public IReadOnlyList<string> Capabilities { get; init; } = Array.Empty<string>();
|
||||
|
||||
[JsonPropertyName("metadata")]
|
||||
public IReadOnlyDictionary<string, string> Metadata { get; init; } = ImmutableDictionary<string, string>.Empty;
|
||||
}
|
||||
|
||||
internal sealed record PluginEntryPoint
|
||||
{
|
||||
[JsonPropertyName("type")]
|
||||
public string Type { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("assembly")]
|
||||
public string Assembly { get; init; } = string.Empty;
|
||||
|
||||
[JsonPropertyName("typeName")]
|
||||
public string TypeName { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
file static class Program
|
||||
{
|
||||
private static readonly SmokeScenario[] PythonScenarios =
|
||||
{
|
||||
new("simple-venv", new[] { Path.Combine("bin", "simple-tool") }),
|
||||
new("pip-cache", new[] { Path.Combine("lib", "python3.11", "site-packages", "cache_pkg-1.2.3.data", "scripts", "cache-tool") }),
|
||||
new("layered-editable", new[] { Path.Combine("layer1", "usr", "bin", "layered-cli") })
|
||||
};
|
||||
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
try
|
||||
{
|
||||
var options = SmokeOptions.Parse(args);
|
||||
await RunAsync(options).ConfigureAwait(false);
|
||||
Console.WriteLine("✅ Python analyzer smoke checks passed");
|
||||
return 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"❌ {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task RunAsync(SmokeOptions options)
|
||||
{
|
||||
ValidateOptions(options);
|
||||
|
||||
var pluginRoot = Path.Combine(options.RepoRoot, "plugins", "scanner", "analyzers", "lang", options.PluginDirectoryName);
|
||||
var manifestPath = Path.Combine(pluginRoot, "manifest.json");
|
||||
if (!File.Exists(manifestPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Plug-in manifest not found at '{manifestPath}'.", manifestPath);
|
||||
}
|
||||
|
||||
using var manifestStream = File.OpenRead(manifestPath);
|
||||
var manifest = JsonSerializer.Deserialize<PluginManifest>(manifestStream, new JsonSerializerOptions
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip
|
||||
}) ?? throw new InvalidOperationException($"Unable to parse manifest '{manifestPath}'.");
|
||||
|
||||
ValidateManifest(manifest, options.PluginDirectoryName);
|
||||
|
||||
var pluginAssemblyPath = Path.Combine(pluginRoot, manifest.EntryPoint.Assembly);
|
||||
if (!File.Exists(pluginAssemblyPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Plug-in assembly '{manifest.EntryPoint.Assembly}' not found under '{pluginRoot}'.", pluginAssemblyPath);
|
||||
}
|
||||
|
||||
var sha256 = ComputeSha256(pluginAssemblyPath);
|
||||
Console.WriteLine($"→ Plug-in assembly SHA-256: {sha256}");
|
||||
|
||||
using var serviceProvider = BuildServiceProvider();
|
||||
var catalog = new LanguageAnalyzerPluginCatalog(new RestartOnlyPluginGuard(), NullLogger<LanguageAnalyzerPluginCatalog>.Instance);
|
||||
catalog.LoadFromDirectory(pluginRoot, seal: true);
|
||||
|
||||
if (catalog.Plugins.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"No analyzer plug-ins were loaded from '{pluginRoot}'.");
|
||||
}
|
||||
|
||||
var analyzerSet = catalog.CreateAnalyzers(serviceProvider);
|
||||
if (analyzerSet.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("Language analyzer plug-ins reported no analyzers.");
|
||||
}
|
||||
|
||||
var analyzerIds = analyzerSet.Select(analyzer => analyzer.Id).ToArray();
|
||||
Console.WriteLine($"→ Loaded analyzers: {string.Join(", ", analyzerIds)}");
|
||||
|
||||
if (!analyzerIds.Contains("python", StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Python analyzer was not created by the plug-in.");
|
||||
}
|
||||
|
||||
var fixtureRoot = Path.GetFullPath(Path.Combine(options.RepoRoot, options.FixtureRelativePath));
|
||||
if (!Directory.Exists(fixtureRoot))
|
||||
{
|
||||
throw new DirectoryNotFoundException($"Fixture directory '{fixtureRoot}' does not exist.");
|
||||
}
|
||||
|
||||
foreach (var scenario in PythonScenarios)
|
||||
{
|
||||
await RunScenarioAsync(scenario, fixtureRoot, catalog, serviceProvider).ConfigureAwait(false);
|
||||
}
|
||||
}
|
||||
|
||||
private static ServiceProvider BuildServiceProvider()
|
||||
{
|
||||
var services = new ServiceCollection();
|
||||
services.AddLogging();
|
||||
return services.BuildServiceProvider();
|
||||
}
|
||||
|
||||
private static async Task RunScenarioAsync(SmokeScenario scenario, string fixtureRoot, ILanguageAnalyzerPluginCatalog catalog, IServiceProvider services)
|
||||
{
|
||||
var scenarioRoot = Path.Combine(fixtureRoot, scenario.Name);
|
||||
if (!Directory.Exists(scenarioRoot))
|
||||
{
|
||||
throw new DirectoryNotFoundException($"Scenario '{scenario.Name}' directory missing at '{scenarioRoot}'.");
|
||||
}
|
||||
|
||||
var goldenPath = Path.Combine(scenarioRoot, "expected.json");
|
||||
string? goldenNormalized = null;
|
||||
if (File.Exists(goldenPath))
|
||||
{
|
||||
goldenNormalized = NormalizeJson(await File.ReadAllTextAsync(goldenPath).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
var usageHints = new LanguageUsageHints(scenario.ResolveUsageHints(scenarioRoot));
|
||||
var context = new LanguageAnalyzerContext(scenarioRoot, TimeProvider.System, usageHints, services);
|
||||
|
||||
var coldEngine = new LanguageAnalyzerEngine(catalog.CreateAnalyzers(services));
|
||||
var coldStopwatch = Stopwatch.StartNew();
|
||||
var coldResult = await coldEngine.AnalyzeAsync(context, CancellationToken.None).ConfigureAwait(false);
|
||||
coldStopwatch.Stop();
|
||||
|
||||
if (coldResult.Components.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Scenario '{scenario.Name}' produced no components during cold run.");
|
||||
}
|
||||
|
||||
var coldJson = NormalizeJson(coldResult.ToJson(indent: true));
|
||||
if (goldenNormalized is string expected && !string.Equals(coldJson, expected, StringComparison.Ordinal))
|
||||
{
|
||||
Console.WriteLine($"⚠️ Scenario '{scenario.Name}' output deviates from repository golden snapshot.");
|
||||
}
|
||||
|
||||
var warmEngine = new LanguageAnalyzerEngine(catalog.CreateAnalyzers(services));
|
||||
var warmStopwatch = Stopwatch.StartNew();
|
||||
var warmResult = await warmEngine.AnalyzeAsync(context, CancellationToken.None).ConfigureAwait(false);
|
||||
warmStopwatch.Stop();
|
||||
|
||||
var warmJson = NormalizeJson(warmResult.ToJson(indent: true));
|
||||
if (!string.Equals(coldJson, warmJson, StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"Scenario '{scenario.Name}' produced different outputs between cold and warm runs.");
|
||||
}
|
||||
|
||||
EnsureDurationWithinBudget(scenario.Name, coldStopwatch.Elapsed, warmStopwatch.Elapsed);
|
||||
|
||||
Console.WriteLine($"✓ Scenario '{scenario.Name}' — components {coldResult.Components.Count}, cold {coldStopwatch.Elapsed.TotalMilliseconds:F1} ms, warm {warmStopwatch.Elapsed.TotalMilliseconds:F1} ms");
|
||||
}
|
||||
|
||||
private static void EnsureDurationWithinBudget(string scenarioName, TimeSpan coldDuration, TimeSpan warmDuration)
|
||||
{
|
||||
var coldBudget = TimeSpan.FromSeconds(30);
|
||||
var warmBudget = TimeSpan.FromSeconds(5);
|
||||
|
||||
if (coldDuration > coldBudget)
|
||||
{
|
||||
throw new InvalidOperationException($"Scenario '{scenarioName}' cold run exceeded budget ({coldDuration.TotalSeconds:F2}s > {coldBudget.TotalSeconds:F2}s).");
|
||||
}
|
||||
|
||||
if (warmDuration > warmBudget)
|
||||
{
|
||||
throw new InvalidOperationException($"Scenario '{scenarioName}' warm run exceeded budget ({warmDuration.TotalSeconds:F2}s > {warmBudget.TotalSeconds:F2}s).");
|
||||
}
|
||||
}
|
||||
|
||||
private static string NormalizeJson(string json)
|
||||
=> json.Replace("\r\n", "\n", StringComparison.Ordinal).TrimEnd();
|
||||
|
||||
private static void ValidateOptions(SmokeOptions options)
|
||||
{
|
||||
if (!Directory.Exists(options.RepoRoot))
|
||||
{
|
||||
throw new DirectoryNotFoundException($"Repository root '{options.RepoRoot}' does not exist.");
|
||||
}
|
||||
}
|
||||
|
||||
private static void ValidateManifest(PluginManifest manifest, string expectedDirectory)
|
||||
{
|
||||
if (!string.Equals(manifest.SchemaVersion, "1.0", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"Unexpected manifest schema version '{manifest.SchemaVersion}'.");
|
||||
}
|
||||
|
||||
if (!manifest.RequiresRestart)
|
||||
{
|
||||
throw new InvalidOperationException("Language analyzer plug-in must be marked as restart-only.");
|
||||
}
|
||||
|
||||
if (!string.Equals(manifest.EntryPoint.Type, "dotnet", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Unsupported entry point type '{manifest.EntryPoint.Type}'.");
|
||||
}
|
||||
|
||||
if (!manifest.Capabilities.Contains("python", StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException("Manifest capabilities do not include 'python'.");
|
||||
}
|
||||
|
||||
if (!string.Equals(manifest.EntryPoint.TypeName, "StellaOps.Scanner.Analyzers.Lang.Python.PythonAnalyzerPlugin", StringComparison.Ordinal))
|
||||
{
|
||||
throw new InvalidOperationException($"Unexpected entry point type name '{manifest.EntryPoint.TypeName}'.");
|
||||
}
|
||||
|
||||
if (!string.Equals(manifest.Id, "stellaops.analyzer.lang.python", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
throw new InvalidOperationException($"Manifest id '{manifest.Id}' does not match expected plug-in id for directory '{expectedDirectory}'.");
|
||||
}
|
||||
}
|
||||
|
||||
private static string ComputeSha256(string path)
|
||||
{
|
||||
using var hash = SHA256.Create();
|
||||
using var stream = File.OpenRead(path);
|
||||
var digest = hash.ComputeHash(stream);
|
||||
var builder = new StringBuilder(digest.Length * 2);
|
||||
foreach (var b in digest)
|
||||
{
|
||||
builder.Append(b.ToString("x2"));
|
||||
}
|
||||
return builder.ToString();
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="StackExchange.Redis" Version="2.8.24" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,198 +0,0 @@
|
||||
using System.Globalization;
|
||||
using System.Net.Http.Headers;
|
||||
using System.Linq;
|
||||
using System.Text.Json;
|
||||
using StackExchange.Redis;
|
||||
|
||||
static string RequireEnv(string name)
|
||||
{
|
||||
var value = Environment.GetEnvironmentVariable(name);
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
throw new InvalidOperationException($"Environment variable '{name}' is required for Notify smoke validation.");
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
static string? GetField(StreamEntry entry, string fieldName)
|
||||
{
|
||||
foreach (var pair in entry.Values)
|
||||
{
|
||||
if (string.Equals(pair.Name, fieldName, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
return pair.Value.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
static void Ensure(bool condition, string message)
|
||||
{
|
||||
if (!condition)
|
||||
{
|
||||
throw new InvalidOperationException(message);
|
||||
}
|
||||
}
|
||||
|
||||
var redisDsn = RequireEnv("NOTIFY_SMOKE_REDIS_DSN");
|
||||
var redisStream = Environment.GetEnvironmentVariable("NOTIFY_SMOKE_STREAM");
|
||||
if (string.IsNullOrWhiteSpace(redisStream))
|
||||
{
|
||||
redisStream = "stella.events";
|
||||
}
|
||||
|
||||
var expectedKindsEnv = RequireEnv("NOTIFY_SMOKE_EXPECT_KINDS");
|
||||
|
||||
var expectedKinds = expectedKindsEnv
|
||||
.Split(',', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
|
||||
.Select(kind => kind.ToLowerInvariant())
|
||||
.Distinct()
|
||||
.ToArray();
|
||||
Ensure(expectedKinds.Length > 0, "Expected at least one event kind in NOTIFY_SMOKE_EXPECT_KINDS.");
|
||||
|
||||
var lookbackMinutesEnv = RequireEnv("NOTIFY_SMOKE_LOOKBACK_MINUTES");
|
||||
if (!double.TryParse(lookbackMinutesEnv, NumberStyles.Any, CultureInfo.InvariantCulture, out var lookbackMinutes))
|
||||
{
|
||||
throw new InvalidOperationException("NOTIFY_SMOKE_LOOKBACK_MINUTES must be numeric.");
|
||||
}
|
||||
Ensure(lookbackMinutes > 0, "NOTIFY_SMOKE_LOOKBACK_MINUTES must be greater than zero.");
|
||||
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var sinceThreshold = now - TimeSpan.FromMinutes(Math.Max(1, lookbackMinutes));
|
||||
|
||||
Console.WriteLine($"ℹ️ Checking Redis stream '{redisStream}' for kinds [{string.Join(", ", expectedKinds)}] within the last {lookbackMinutes:F1} minutes.");
|
||||
|
||||
var redisConfig = ConfigurationOptions.Parse(redisDsn);
|
||||
redisConfig.AbortOnConnectFail = false;
|
||||
|
||||
await using var redisConnection = await ConnectionMultiplexer.ConnectAsync(redisConfig);
|
||||
var database = redisConnection.GetDatabase();
|
||||
|
||||
var streamEntries = await database.StreamRangeAsync(redisStream, "-", "+", count: 200);
|
||||
if (streamEntries.Length > 1)
|
||||
{
|
||||
Array.Reverse(streamEntries);
|
||||
}
|
||||
Ensure(streamEntries.Length > 0, $"Redis stream '{redisStream}' is empty.");
|
||||
|
||||
var recentEntries = new List<StreamEntry>();
|
||||
foreach (var entry in streamEntries)
|
||||
{
|
||||
var timestampText = GetField(entry, "ts");
|
||||
if (timestampText is null)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!DateTimeOffset.TryParse(timestampText, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal, out var entryTimestamp))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (entryTimestamp >= sinceThreshold)
|
||||
{
|
||||
recentEntries.Add(entry);
|
||||
}
|
||||
}
|
||||
|
||||
Ensure(recentEntries.Count > 0, $"No Redis events newer than {sinceThreshold:u} located in stream '{redisStream}'.");
|
||||
|
||||
var missingKinds = new List<string>();
|
||||
foreach (var kind in expectedKinds)
|
||||
{
|
||||
var match = recentEntries.FirstOrDefault(entry =>
|
||||
{
|
||||
var entryKind = GetField(entry, "kind")?.ToLowerInvariant();
|
||||
return entryKind == kind;
|
||||
});
|
||||
|
||||
if (match.Equals(default(StreamEntry)))
|
||||
{
|
||||
missingKinds.Add(kind);
|
||||
}
|
||||
}
|
||||
|
||||
Ensure(missingKinds.Count == 0, $"Missing expected Redis events for kinds: {string.Join(", ", missingKinds)}");
|
||||
|
||||
Console.WriteLine("✅ Redis event stream contains the expected scanner events.");
|
||||
|
||||
var notifyBaseUrl = RequireEnv("NOTIFY_SMOKE_NOTIFY_BASEURL").TrimEnd('/');
|
||||
var notifyToken = RequireEnv("NOTIFY_SMOKE_NOTIFY_TOKEN");
|
||||
var notifyTenant = RequireEnv("NOTIFY_SMOKE_NOTIFY_TENANT");
|
||||
var notifyTenantHeader = Environment.GetEnvironmentVariable("NOTIFY_SMOKE_NOTIFY_TENANT_HEADER");
|
||||
if (string.IsNullOrWhiteSpace(notifyTenantHeader))
|
||||
{
|
||||
notifyTenantHeader = "X-StellaOps-Tenant";
|
||||
}
|
||||
|
||||
var notifyTimeoutSeconds = 30;
|
||||
var notifyTimeoutEnv = Environment.GetEnvironmentVariable("NOTIFY_SMOKE_NOTIFY_TIMEOUT_SECONDS");
|
||||
if (!string.IsNullOrWhiteSpace(notifyTimeoutEnv) && int.TryParse(notifyTimeoutEnv, NumberStyles.Integer, CultureInfo.InvariantCulture, out var parsedTimeout))
|
||||
{
|
||||
notifyTimeoutSeconds = Math.Max(5, parsedTimeout);
|
||||
}
|
||||
|
||||
using var httpClient = new HttpClient
|
||||
{
|
||||
Timeout = TimeSpan.FromSeconds(notifyTimeoutSeconds),
|
||||
};
|
||||
|
||||
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", notifyToken);
|
||||
httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
|
||||
httpClient.DefaultRequestHeaders.Add(notifyTenantHeader, notifyTenant);
|
||||
|
||||
var sinceQuery = Uri.EscapeDataString(sinceThreshold.ToString("O", CultureInfo.InvariantCulture));
|
||||
var deliveriesUrl = $"{notifyBaseUrl}/api/v1/deliveries?since={sinceQuery}&limit=200";
|
||||
|
||||
Console.WriteLine($"ℹ️ Querying Notify deliveries via {deliveriesUrl}.");
|
||||
|
||||
using var response = await httpClient.GetAsync(deliveriesUrl);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
var body = await response.Content.ReadAsStringAsync();
|
||||
throw new InvalidOperationException($"Notify deliveries request failed with {(int)response.StatusCode} {response.ReasonPhrase}: {body}");
|
||||
}
|
||||
|
||||
var json = await response.Content.ReadAsStringAsync();
|
||||
if (string.IsNullOrWhiteSpace(json))
|
||||
{
|
||||
throw new InvalidOperationException("Notify deliveries response body was empty.");
|
||||
}
|
||||
|
||||
using var document = JsonDocument.Parse(json);
|
||||
var root = document.RootElement;
|
||||
|
||||
IEnumerable<JsonElement> EnumerateDeliveries(JsonElement element)
|
||||
{
|
||||
return element.ValueKind switch
|
||||
{
|
||||
JsonValueKind.Array => element.EnumerateArray(),
|
||||
JsonValueKind.Object when element.TryGetProperty("items", out var items) && items.ValueKind == JsonValueKind.Array => items.EnumerateArray(),
|
||||
_ => throw new InvalidOperationException("Notify deliveries response was not an array or did not contain an 'items' collection.")
|
||||
};
|
||||
}
|
||||
|
||||
var deliveries = EnumerateDeliveries(root).ToArray();
|
||||
Ensure(deliveries.Length > 0, "Notify deliveries response did not return any records.");
|
||||
|
||||
var missingDeliveryKinds = new List<string>();
|
||||
foreach (var kind in expectedKinds)
|
||||
{
|
||||
var found = deliveries.Any(delivery =>
|
||||
delivery.TryGetProperty("kind", out var kindProperty) &&
|
||||
kindProperty.GetString()?.Equals(kind, StringComparison.OrdinalIgnoreCase) == true &&
|
||||
delivery.TryGetProperty("status", out var statusProperty) &&
|
||||
!string.Equals(statusProperty.GetString(), "failed", StringComparison.OrdinalIgnoreCase));
|
||||
|
||||
if (!found)
|
||||
{
|
||||
missingDeliveryKinds.Add(kind);
|
||||
}
|
||||
}
|
||||
|
||||
Ensure(missingDeliveryKinds.Count == 0, $"Notify deliveries missing successful records for kinds: {string.Join(", ", missingDeliveryKinds)}");
|
||||
|
||||
Console.WriteLine("✅ Notify deliveries include the expected scanner events.");
|
||||
Console.WriteLine("🎉 Notify smoke validation completed successfully.");
|
||||
@@ -1,14 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<Nullable>enable</Nullable>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\StellaOps.Policy\StellaOps.Policy.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -1,56 +0,0 @@
|
||||
using StellaOps.Policy;
|
||||
|
||||
if (args.Length == 0)
|
||||
{
|
||||
Console.Error.WriteLine("Usage: policy-dsl-validator [--strict] [--json] <path-or-glob> [<path-or-glob> ...]");
|
||||
Console.Error.WriteLine("Example: policy-dsl-validator --strict docs/examples/policies");
|
||||
return 64; // EX_USAGE
|
||||
}
|
||||
|
||||
var inputs = new List<string>();
|
||||
var strict = false;
|
||||
var outputJson = false;
|
||||
|
||||
foreach (var arg in args)
|
||||
{
|
||||
switch (arg)
|
||||
{
|
||||
case "--strict":
|
||||
case "-s":
|
||||
strict = true;
|
||||
break;
|
||||
|
||||
case "--json":
|
||||
case "-j":
|
||||
outputJson = true;
|
||||
break;
|
||||
|
||||
case "--help":
|
||||
case "-h":
|
||||
case "-?":
|
||||
Console.WriteLine("Usage: policy-dsl-validator [--strict] [--json] <path-or-glob> [<path-or-glob> ...]");
|
||||
Console.WriteLine("Example: policy-dsl-validator --strict docs/examples/policies");
|
||||
return 0;
|
||||
|
||||
default:
|
||||
inputs.Add(arg);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (inputs.Count == 0)
|
||||
{
|
||||
Console.Error.WriteLine("No input files or directories provided.");
|
||||
return 64; // EX_USAGE
|
||||
}
|
||||
|
||||
var options = new PolicyValidationCliOptions
|
||||
{
|
||||
Inputs = inputs,
|
||||
Strict = strict,
|
||||
OutputJson = outputJson,
|
||||
};
|
||||
|
||||
var cli = new PolicyValidationCli();
|
||||
var exitCode = await cli.RunAsync(options, CancellationToken.None);
|
||||
return exitCode;
|
||||
@@ -1,21 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
<TreatWarningsAsErrors>true</TreatWarningsAsErrors>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<PackageReference Include="NJsonSchema" Version="11.5.1" />
|
||||
<PackageReference Include="NJsonSchema.SystemTextJson" Version="11.5.1" />
|
||||
<PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\StellaOps.Scheduler.Models\StellaOps.Scheduler.Models.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -1,48 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using NJsonSchema;
|
||||
using NJsonSchema.Generation;
|
||||
using NJsonSchema.Generation.SystemTextJson;
|
||||
using Newtonsoft.Json;
|
||||
using StellaOps.Scheduler.Models;
|
||||
|
||||
var output = args.Length switch
|
||||
{
|
||||
0 => Path.GetFullPath(Path.Combine(AppContext.BaseDirectory, "..", "..", "..", "docs", "schemas")),
|
||||
1 => Path.GetFullPath(args[0]),
|
||||
_ => throw new ArgumentException("Usage: dotnet run --project tools/PolicySchemaExporter -- [outputDirectory]")
|
||||
};
|
||||
|
||||
Directory.CreateDirectory(output);
|
||||
|
||||
var generatorSettings = new SystemTextJsonSchemaGeneratorSettings
|
||||
{
|
||||
SchemaType = SchemaType.JsonSchema,
|
||||
DefaultReferenceTypeNullHandling = ReferenceTypeNullHandling.NotNull,
|
||||
SerializerOptions = new JsonSerializerOptions
|
||||
{
|
||||
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
|
||||
DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull,
|
||||
},
|
||||
};
|
||||
|
||||
var generator = new JsonSchemaGenerator(generatorSettings);
|
||||
|
||||
var exports = ImmutableArray.Create(
|
||||
(FileName: "policy-run-request.schema.json", Type: typeof(PolicyRunRequest)),
|
||||
(FileName: "policy-run-status.schema.json", Type: typeof(PolicyRunStatus)),
|
||||
(FileName: "policy-diff-summary.schema.json", Type: typeof(PolicyDiffSummary)),
|
||||
(FileName: "policy-explain-trace.schema.json", Type: typeof(PolicyExplainTrace))
|
||||
);
|
||||
|
||||
foreach (var export in exports)
|
||||
{
|
||||
var schema = generator.Generate(export.Type);
|
||||
schema.Title = export.Type.Name;
|
||||
schema.AllowAdditionalProperties = false;
|
||||
|
||||
var outputPath = Path.Combine(output, export.FileName);
|
||||
await File.WriteAllTextAsync(outputPath, schema.ToJson(Formatting.Indented) + Environment.NewLine);
|
||||
Console.WriteLine($"Wrote {outputPath}");
|
||||
}
|
||||
@@ -1,14 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\StellaOps.Policy\StellaOps.Policy.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
</Project>
|
||||
@@ -1,291 +0,0 @@
|
||||
using System.Collections.Immutable;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using StellaOps.Policy;
|
||||
|
||||
var scenarioRoot = "samples/policy/simulations";
|
||||
string? outputDir = null;
|
||||
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
{
|
||||
var arg = args[i];
|
||||
switch (arg)
|
||||
{
|
||||
case "--scenario-root":
|
||||
case "-r":
|
||||
if (i + 1 >= args.Length)
|
||||
{
|
||||
Console.Error.WriteLine("Missing value for --scenario-root.");
|
||||
return 64;
|
||||
}
|
||||
scenarioRoot = args[++i];
|
||||
break;
|
||||
case "--output":
|
||||
case "-o":
|
||||
if (i + 1 >= args.Length)
|
||||
{
|
||||
Console.Error.WriteLine("Missing value for --output.");
|
||||
return 64;
|
||||
}
|
||||
outputDir = args[++i];
|
||||
break;
|
||||
case "--help":
|
||||
case "-h":
|
||||
case "-?":
|
||||
PrintUsage();
|
||||
return 0;
|
||||
default:
|
||||
Console.Error.WriteLine($"Unknown argument '{arg}'.");
|
||||
PrintUsage();
|
||||
return 64;
|
||||
}
|
||||
}
|
||||
|
||||
if (!Directory.Exists(scenarioRoot))
|
||||
{
|
||||
Console.Error.WriteLine($"Scenario root '{scenarioRoot}' does not exist.");
|
||||
return 66;
|
||||
}
|
||||
|
||||
var scenarioFiles = Directory.GetFiles(scenarioRoot, "scenario.json", SearchOption.AllDirectories);
|
||||
if (scenarioFiles.Length == 0)
|
||||
{
|
||||
Console.Error.WriteLine($"No scenario.json files found under '{scenarioRoot}'.");
|
||||
return 0;
|
||||
}
|
||||
|
||||
var loggerFactory = NullLoggerFactory.Instance;
|
||||
var snapshotStore = new PolicySnapshotStore(
|
||||
new NullPolicySnapshotRepository(),
|
||||
new NullPolicyAuditRepository(),
|
||||
TimeProvider.System,
|
||||
loggerFactory.CreateLogger<PolicySnapshotStore>());
|
||||
var previewService = new PolicyPreviewService(snapshotStore, loggerFactory.CreateLogger<PolicyPreviewService>());
|
||||
|
||||
var serializerOptions = new JsonSerializerOptions(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
};
|
||||
|
||||
var summary = new List<ScenarioResult>();
|
||||
var success = true;
|
||||
|
||||
foreach (var scenarioFile in scenarioFiles.OrderBy(static f => f, StringComparer.OrdinalIgnoreCase))
|
||||
{
|
||||
var scenarioText = await File.ReadAllTextAsync(scenarioFile);
|
||||
var scenario = JsonSerializer.Deserialize<PolicySimulationScenario>(scenarioText, serializerOptions);
|
||||
if (scenario is null)
|
||||
{
|
||||
Console.Error.WriteLine($"Failed to deserialize scenario '{scenarioFile}'.");
|
||||
success = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
var repoRoot = Directory.GetCurrentDirectory();
|
||||
var policyPath = Path.Combine(repoRoot, scenario.PolicyPath);
|
||||
if (!File.Exists(policyPath))
|
||||
{
|
||||
Console.Error.WriteLine($"Policy file '{scenario.PolicyPath}' referenced by scenario '{scenario.Name}' does not exist.");
|
||||
success = false;
|
||||
continue;
|
||||
}
|
||||
|
||||
var policyContent = await File.ReadAllTextAsync(policyPath);
|
||||
var policyFormat = PolicySchema.DetectFormat(policyPath);
|
||||
var findings = scenario.Findings.Select(ToPolicyFinding).ToImmutableArray();
|
||||
var baseline = scenario.Baseline?.Select(ToPolicyVerdict).ToImmutableArray() ?? ImmutableArray<PolicyVerdict>.Empty;
|
||||
|
||||
var request = new PolicyPreviewRequest(
|
||||
ImageDigest: $"sha256:simulation-{scenario.Name}",
|
||||
Findings: findings,
|
||||
BaselineVerdicts: baseline,
|
||||
SnapshotOverride: null,
|
||||
ProposedPolicy: new PolicySnapshotContent(
|
||||
Content: policyContent,
|
||||
Format: policyFormat,
|
||||
Actor: "ci",
|
||||
Source: "ci/simulation-smoke",
|
||||
Description: $"CI simulation for scenario '{scenario.Name}'"));
|
||||
|
||||
var response = await previewService.PreviewAsync(request, CancellationToken.None);
|
||||
var scenarioResult = EvaluateScenario(scenario, response);
|
||||
summary.Add(scenarioResult);
|
||||
|
||||
if (!scenarioResult.Success)
|
||||
{
|
||||
success = false;
|
||||
}
|
||||
}
|
||||
|
||||
if (outputDir is not null)
|
||||
{
|
||||
Directory.CreateDirectory(outputDir);
|
||||
var summaryPath = Path.Combine(outputDir, "policy-simulation-summary.json");
|
||||
await File.WriteAllTextAsync(summaryPath, JsonSerializer.Serialize(summary, new JsonSerializerOptions { WriteIndented = true }));
|
||||
}
|
||||
|
||||
return success ? 0 : 1;
|
||||
|
||||
static void PrintUsage()
|
||||
{
|
||||
Console.WriteLine("Usage: policy-simulation-smoke [--scenario-root <path>] [--output <dir>]");
|
||||
Console.WriteLine("Example: policy-simulation-smoke --scenario-root samples/policy/simulations --output artifacts/policy-simulations");
|
||||
}
|
||||
|
||||
static PolicyFinding ToPolicyFinding(ScenarioFinding finding)
|
||||
{
|
||||
var tags = finding.Tags is null ? ImmutableArray<string>.Empty : ImmutableArray.CreateRange(finding.Tags);
|
||||
var severity = Enum.Parse<PolicySeverity>(finding.Severity, ignoreCase: true);
|
||||
return new PolicyFinding(
|
||||
finding.FindingId,
|
||||
severity,
|
||||
finding.Environment,
|
||||
finding.Source,
|
||||
finding.Vendor,
|
||||
finding.License,
|
||||
finding.Image,
|
||||
finding.Repository,
|
||||
finding.Package,
|
||||
finding.Purl,
|
||||
finding.Cve,
|
||||
finding.Path,
|
||||
finding.LayerDigest,
|
||||
tags);
|
||||
}
|
||||
|
||||
static PolicyVerdict ToPolicyVerdict(ScenarioBaseline baseline)
|
||||
{
|
||||
var status = Enum.Parse<PolicyVerdictStatus>(baseline.Status, ignoreCase: true);
|
||||
var inputs = baseline.Inputs?.ToImmutableDictionary(StringComparer.OrdinalIgnoreCase) ?? ImmutableDictionary<string, double>.Empty;
|
||||
return new PolicyVerdict(
|
||||
baseline.FindingId,
|
||||
status,
|
||||
RuleName: baseline.RuleName,
|
||||
RuleAction: baseline.RuleAction,
|
||||
Notes: baseline.Notes,
|
||||
Score: baseline.Score,
|
||||
ConfigVersion: baseline.ConfigVersion ?? PolicyScoringConfig.Default.Version,
|
||||
Inputs: inputs,
|
||||
QuietedBy: null,
|
||||
Quiet: false,
|
||||
UnknownConfidence: null,
|
||||
ConfidenceBand: null,
|
||||
UnknownAgeDays: null,
|
||||
SourceTrust: null,
|
||||
Reachability: null);
|
||||
}
|
||||
|
||||
static ScenarioResult EvaluateScenario(PolicySimulationScenario scenario, PolicyPreviewResponse response)
|
||||
{
|
||||
var result = new ScenarioResult(scenario.Name);
|
||||
if (!response.Success)
|
||||
{
|
||||
result.Failures.Add("Preview failed.");
|
||||
return result with { Success = false, ChangedCount = response.ChangedCount };
|
||||
}
|
||||
|
||||
var diffs = response.Diffs.ToDictionary(diff => diff.Projected.FindingId, StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var expected in scenario.ExpectedDiffs)
|
||||
{
|
||||
if (!diffs.TryGetValue(expected.FindingId, out var diff))
|
||||
{
|
||||
result.Failures.Add($"Expected finding '{expected.FindingId}' missing from diff.");
|
||||
continue;
|
||||
}
|
||||
|
||||
var projectedStatus = diff.Projected.Status.ToString();
|
||||
result.ActualStatuses[expected.FindingId] = projectedStatus;
|
||||
if (!string.Equals(projectedStatus, expected.Status, StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
result.Failures.Add($"Finding '{expected.FindingId}' expected status '{expected.Status}' but was '{projectedStatus}'.");
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var diff in diffs.Values)
|
||||
{
|
||||
if (!result.ActualStatuses.ContainsKey(diff.Projected.FindingId))
|
||||
{
|
||||
result.ActualStatuses[diff.Projected.FindingId] = diff.Projected.Status.ToString();
|
||||
}
|
||||
}
|
||||
|
||||
var success = result.Failures.Count == 0;
|
||||
return result with
|
||||
{
|
||||
Success = success,
|
||||
ChangedCount = response.ChangedCount
|
||||
};
|
||||
}
|
||||
|
||||
internal sealed record PolicySimulationScenario
|
||||
{
|
||||
public string Name { get; init; } = "scenario";
|
||||
public string PolicyPath { get; init; } = string.Empty;
|
||||
public List<ScenarioFinding> Findings { get; init; } = new();
|
||||
public List<ScenarioExpectedDiff> ExpectedDiffs { get; init; } = new();
|
||||
public List<ScenarioBaseline>? Baseline { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record ScenarioFinding
|
||||
{
|
||||
public string FindingId { get; init; } = string.Empty;
|
||||
public string Severity { get; init; } = "Low";
|
||||
public string? Environment { get; init; }
|
||||
public string? Source { get; init; }
|
||||
public string? Vendor { get; init; }
|
||||
public string? License { get; init; }
|
||||
public string? Image { get; init; }
|
||||
public string? Repository { get; init; }
|
||||
public string? Package { get; init; }
|
||||
public string? Purl { get; init; }
|
||||
public string? Cve { get; init; }
|
||||
public string? Path { get; init; }
|
||||
public string? LayerDigest { get; init; }
|
||||
public string[]? Tags { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record ScenarioExpectedDiff
|
||||
{
|
||||
public string FindingId { get; init; } = string.Empty;
|
||||
public string Status { get; init; } = "Pass";
|
||||
}
|
||||
|
||||
internal sealed record ScenarioBaseline
|
||||
{
|
||||
public string FindingId { get; init; } = string.Empty;
|
||||
public string Status { get; init; } = "Pass";
|
||||
public string? RuleName { get; init; }
|
||||
public string? RuleAction { get; init; }
|
||||
public string? Notes { get; init; }
|
||||
public double Score { get; init; }
|
||||
public string? ConfigVersion { get; init; }
|
||||
public Dictionary<string, double>? Inputs { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record ScenarioResult(string ScenarioName)
|
||||
{
|
||||
public bool Success { get; init; } = true;
|
||||
public int ChangedCount { get; init; }
|
||||
public List<string> Failures { get; } = new();
|
||||
public Dictionary<string, string> ActualStatuses { get; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
internal sealed class NullPolicySnapshotRepository : IPolicySnapshotRepository
|
||||
{
|
||||
public Task AddAsync(PolicySnapshot snapshot, CancellationToken cancellationToken = default) => Task.CompletedTask;
|
||||
|
||||
public Task<PolicySnapshot?> GetLatestAsync(CancellationToken cancellationToken = default) => Task.FromResult<PolicySnapshot?>(null);
|
||||
|
||||
public Task<IReadOnlyList<PolicySnapshot>> ListAsync(int limit, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult<IReadOnlyList<PolicySnapshot>>(Array.Empty<PolicySnapshot>());
|
||||
}
|
||||
|
||||
internal sealed class NullPolicyAuditRepository : IPolicyAuditRepository
|
||||
{
|
||||
public Task AddAsync(PolicyAuditEntry entry, CancellationToken cancellationToken = default) => Task.CompletedTask;
|
||||
|
||||
public Task<IReadOnlyList<PolicyAuditEntry>> ListAsync(int limit, CancellationToken cancellationToken = default)
|
||||
=> Task.FromResult<IReadOnlyList<PolicyAuditEntry>>(Array.Empty<PolicyAuditEntry>());
|
||||
}
|
||||
@@ -1,286 +0,0 @@
|
||||
using Amazon;
|
||||
using Amazon.Runtime;
|
||||
using Amazon.S3;
|
||||
using Amazon.S3.Model;
|
||||
using System.Net.Http.Headers;
|
||||
|
||||
var options = MigrationOptions.Parse(args);
|
||||
if (options is null)
|
||||
{
|
||||
MigrationOptions.PrintUsage();
|
||||
return 1;
|
||||
}
|
||||
|
||||
Console.WriteLine($"RustFS migrator starting (prefix: '{options.Prefix ?? "<all>"}')");
|
||||
if (options.DryRun)
|
||||
{
|
||||
Console.WriteLine("Dry-run enabled. No objects will be written to RustFS.");
|
||||
}
|
||||
|
||||
var s3Config = new AmazonS3Config
|
||||
{
|
||||
ForcePathStyle = true,
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.S3ServiceUrl))
|
||||
{
|
||||
s3Config.ServiceURL = options.S3ServiceUrl;
|
||||
s3Config.UseHttp = options.S3ServiceUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.S3Region))
|
||||
{
|
||||
s3Config.RegionEndpoint = RegionEndpoint.GetBySystemName(options.S3Region);
|
||||
}
|
||||
|
||||
using var s3Client = CreateS3Client(options, s3Config);
|
||||
using var httpClient = CreateRustFsClient(options);
|
||||
|
||||
var listRequest = new ListObjectsV2Request
|
||||
{
|
||||
BucketName = options.S3Bucket,
|
||||
Prefix = options.Prefix,
|
||||
MaxKeys = 1000,
|
||||
};
|
||||
|
||||
var migrated = 0;
|
||||
var skipped = 0;
|
||||
|
||||
do
|
||||
{
|
||||
var response = await s3Client.ListObjectsV2Async(listRequest).ConfigureAwait(false);
|
||||
foreach (var entry in response.S3Objects)
|
||||
{
|
||||
if (entry.Size == 0 && entry.Key.EndsWith('/'))
|
||||
{
|
||||
skipped++;
|
||||
continue;
|
||||
}
|
||||
|
||||
Console.WriteLine($"Migrating {entry.Key} ({entry.Size} bytes)...");
|
||||
|
||||
if (options.DryRun)
|
||||
{
|
||||
migrated++;
|
||||
continue;
|
||||
}
|
||||
|
||||
using var getResponse = await s3Client.GetObjectAsync(new GetObjectRequest
|
||||
{
|
||||
BucketName = options.S3Bucket,
|
||||
Key = entry.Key,
|
||||
}).ConfigureAwait(false);
|
||||
|
||||
await using var memory = new MemoryStream();
|
||||
await getResponse.ResponseStream.CopyToAsync(memory).ConfigureAwait(false);
|
||||
memory.Position = 0;
|
||||
|
||||
using var request = new HttpRequestMessage(HttpMethod.Put, BuildRustFsUri(options, entry.Key))
|
||||
{
|
||||
Content = new ByteArrayContent(memory.ToArray()),
|
||||
};
|
||||
request.Content.Headers.ContentType = MediaTypeHeaderValue.Parse("application/octet-stream");
|
||||
|
||||
if (options.Immutable)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation("X-RustFS-Immutable", "true");
|
||||
}
|
||||
|
||||
if (options.RetentionSeconds is { } retainSeconds)
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation("X-RustFS-Retain-Seconds", retainSeconds.ToString());
|
||||
}
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
|
||||
{
|
||||
request.Headers.TryAddWithoutValidation(options.RustFsApiKeyHeader!, options.RustFsApiKey!);
|
||||
}
|
||||
|
||||
using var responseMessage = await httpClient.SendAsync(request).ConfigureAwait(false);
|
||||
if (!responseMessage.IsSuccessStatusCode)
|
||||
{
|
||||
var error = await responseMessage.Content.ReadAsStringAsync().ConfigureAwait(false);
|
||||
Console.Error.WriteLine($"Failed to upload {entry.Key}: {(int)responseMessage.StatusCode} {responseMessage.ReasonPhrase}\n{error}");
|
||||
return 2;
|
||||
}
|
||||
|
||||
migrated++;
|
||||
}
|
||||
|
||||
listRequest.ContinuationToken = response.NextContinuationToken;
|
||||
} while (!string.IsNullOrEmpty(listRequest.ContinuationToken));
|
||||
|
||||
Console.WriteLine($"Migration complete. Migrated {migrated} objects. Skipped {skipped} directory markers.");
|
||||
return 0;
|
||||
|
||||
static AmazonS3Client CreateS3Client(MigrationOptions options, AmazonS3Config config)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(options.S3AccessKey) && !string.IsNullOrWhiteSpace(options.S3SecretKey))
|
||||
{
|
||||
var credentials = new BasicAWSCredentials(options.S3AccessKey, options.S3SecretKey);
|
||||
return new AmazonS3Client(credentials, config);
|
||||
}
|
||||
|
||||
return new AmazonS3Client(config);
|
||||
}
|
||||
|
||||
static HttpClient CreateRustFsClient(MigrationOptions options)
|
||||
{
|
||||
var client = new HttpClient
|
||||
{
|
||||
BaseAddress = new Uri(options.RustFsEndpoint, UriKind.Absolute),
|
||||
Timeout = TimeSpan.FromMinutes(5),
|
||||
};
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(options.RustFsApiKeyHeader) && !string.IsNullOrWhiteSpace(options.RustFsApiKey))
|
||||
{
|
||||
client.DefaultRequestHeaders.TryAddWithoutValidation(options.RustFsApiKeyHeader, options.RustFsApiKey);
|
||||
}
|
||||
|
||||
return client;
|
||||
}
|
||||
|
||||
static Uri BuildRustFsUri(MigrationOptions options, string key)
|
||||
{
|
||||
var normalized = string.Join('/', key
|
||||
.Split('/', StringSplitOptions.RemoveEmptyEntries)
|
||||
.Select(Uri.EscapeDataString));
|
||||
|
||||
var builder = new UriBuilder(options.RustFsEndpoint)
|
||||
{
|
||||
Path = $"/api/v1/buckets/{Uri.EscapeDataString(options.RustFsBucket)}/objects/{normalized}",
|
||||
};
|
||||
|
||||
return builder.Uri;
|
||||
}
|
||||
|
||||
internal sealed record MigrationOptions
|
||||
{
|
||||
public string S3Bucket { get; init; } = string.Empty;
|
||||
|
||||
public string? S3ServiceUrl { get; init; }
|
||||
= null;
|
||||
|
||||
public string? S3Region { get; init; }
|
||||
= null;
|
||||
|
||||
public string? S3AccessKey { get; init; }
|
||||
= null;
|
||||
|
||||
public string? S3SecretKey { get; init; }
|
||||
= null;
|
||||
|
||||
public string RustFsEndpoint { get; init; } = string.Empty;
|
||||
|
||||
public string RustFsBucket { get; init; } = string.Empty;
|
||||
|
||||
public string? RustFsApiKeyHeader { get; init; }
|
||||
= null;
|
||||
|
||||
public string? RustFsApiKey { get; init; }
|
||||
= null;
|
||||
|
||||
public string? Prefix { get; init; }
|
||||
= null;
|
||||
|
||||
public bool Immutable { get; init; }
|
||||
= false;
|
||||
|
||||
public int? RetentionSeconds { get; init; }
|
||||
= null;
|
||||
|
||||
public bool DryRun { get; init; }
|
||||
= false;
|
||||
|
||||
public static MigrationOptions? Parse(string[] args)
|
||||
{
|
||||
var builder = new Dictionary<string, string?>(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
{
|
||||
var key = args[i];
|
||||
if (key.StartsWith("--", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
var normalized = key[2..];
|
||||
if (string.Equals(normalized, "immutable", StringComparison.OrdinalIgnoreCase) || string.Equals(normalized, "dry-run", StringComparison.OrdinalIgnoreCase))
|
||||
{
|
||||
builder[normalized] = "true";
|
||||
continue;
|
||||
}
|
||||
|
||||
if (i + 1 >= args.Length)
|
||||
{
|
||||
Console.Error.WriteLine($"Missing value for argument '{key}'.");
|
||||
return null;
|
||||
}
|
||||
|
||||
builder[normalized] = args[++i];
|
||||
}
|
||||
}
|
||||
|
||||
if (!builder.TryGetValue("s3-bucket", out var bucket) || string.IsNullOrWhiteSpace(bucket))
|
||||
{
|
||||
Console.Error.WriteLine("--s3-bucket is required.");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!builder.TryGetValue("rustfs-endpoint", out var rustFsEndpoint) || string.IsNullOrWhiteSpace(rustFsEndpoint))
|
||||
{
|
||||
Console.Error.WriteLine("--rustfs-endpoint is required.");
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!builder.TryGetValue("rustfs-bucket", out var rustFsBucket) || string.IsNullOrWhiteSpace(rustFsBucket))
|
||||
{
|
||||
Console.Error.WriteLine("--rustfs-bucket is required.");
|
||||
return null;
|
||||
}
|
||||
|
||||
int? retentionSeconds = null;
|
||||
if (builder.TryGetValue("retain-days", out var retainStr) && !string.IsNullOrWhiteSpace(retainStr))
|
||||
{
|
||||
if (double.TryParse(retainStr, out var days) && days > 0)
|
||||
{
|
||||
retentionSeconds = (int)Math.Ceiling(days * 24 * 60 * 60);
|
||||
}
|
||||
else
|
||||
{
|
||||
Console.Error.WriteLine("--retain-days must be a positive number.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
return new MigrationOptions
|
||||
{
|
||||
S3Bucket = bucket,
|
||||
S3ServiceUrl = builder.TryGetValue("s3-endpoint", out var s3Endpoint) ? s3Endpoint : null,
|
||||
S3Region = builder.TryGetValue("s3-region", out var s3Region) ? s3Region : null,
|
||||
S3AccessKey = builder.TryGetValue("s3-access-key", out var s3AccessKey) ? s3AccessKey : null,
|
||||
S3SecretKey = builder.TryGetValue("s3-secret-key", out var s3SecretKey) ? s3SecretKey : null,
|
||||
RustFsEndpoint = rustFsEndpoint!,
|
||||
RustFsBucket = rustFsBucket!,
|
||||
RustFsApiKeyHeader = builder.TryGetValue("rustfs-api-key-header", out var apiKeyHeader) ? apiKeyHeader : null,
|
||||
RustFsApiKey = builder.TryGetValue("rustfs-api-key", out var apiKey) ? apiKey : null,
|
||||
Prefix = builder.TryGetValue("prefix", out var prefix) ? prefix : null,
|
||||
Immutable = builder.ContainsKey("immutable"),
|
||||
RetentionSeconds = retentionSeconds,
|
||||
DryRun = builder.ContainsKey("dry-run"),
|
||||
};
|
||||
}
|
||||
|
||||
public static void PrintUsage()
|
||||
{
|
||||
Console.WriteLine(@"Usage: dotnet run --project tools/RustFsMigrator -- \
|
||||
--s3-bucket <name> \
|
||||
[--s3-endpoint http://minio:9000] \
|
||||
[--s3-region us-east-1] \
|
||||
[--s3-access-key key --s3-secret-key secret] \
|
||||
--rustfs-endpoint http://rustfs:8080 \
|
||||
--rustfs-bucket scanner-artifacts \
|
||||
[--rustfs-api-key-header X-API-Key --rustfs-api-key token] \
|
||||
[--prefix scanner/] \
|
||||
[--immutable] \
|
||||
[--retain-days 365] \
|
||||
[--dry-run]");
|
||||
}
|
||||
}
|
||||
@@ -1,11 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<PackageReference Include="AWSSDK.S3" Version="3.7.305.6" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,346 +0,0 @@
|
||||
using System.Globalization;
|
||||
using System.Text.Json;
|
||||
using Microsoft.Extensions.Logging;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
using MongoDB.Driver;
|
||||
using StellaOps.Concelier.Connector.Common;
|
||||
using StellaOps.Concelier.Connector.Common.Fetch;
|
||||
using StellaOps.Concelier.Connector.Common.State;
|
||||
using StellaOps.Concelier.Storage.Mongo;
|
||||
using StellaOps.Concelier.Storage.Mongo.Documents;
|
||||
|
||||
namespace SourceStateSeeder;
|
||||
|
||||
internal static class Program
|
||||
{
|
||||
private static readonly JsonSerializerOptions JsonOptions = new(JsonSerializerDefaults.Web)
|
||||
{
|
||||
PropertyNameCaseInsensitive = true,
|
||||
ReadCommentHandling = JsonCommentHandling.Skip,
|
||||
AllowTrailingCommas = true,
|
||||
};
|
||||
|
||||
public static async Task<int> Main(string[] args)
|
||||
{
|
||||
try
|
||||
{
|
||||
var options = SeedOptions.Parse(args);
|
||||
if (options is null)
|
||||
{
|
||||
SeedOptions.PrintUsage();
|
||||
return 1;
|
||||
}
|
||||
|
||||
var seed = await LoadSpecificationAsync(options.InputPath).ConfigureAwait(false);
|
||||
var sourceName = seed.Source ?? options.SourceName;
|
||||
if (string.IsNullOrWhiteSpace(sourceName))
|
||||
{
|
||||
Console.Error.WriteLine("Source name must be supplied via --source or the seed file.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
var specification = await BuildSpecificationAsync(seed, sourceName, options.InputPath, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
var client = new MongoClient(options.ConnectionString);
|
||||
var database = client.GetDatabase(options.DatabaseName);
|
||||
var loggerFactory = NullLoggerFactory.Instance;
|
||||
|
||||
var documentStore = new DocumentStore(database, loggerFactory.CreateLogger<DocumentStore>());
|
||||
var rawStorage = new RawDocumentStorage(database);
|
||||
var stateRepository = new MongoSourceStateRepository(database, loggerFactory.CreateLogger<MongoSourceStateRepository>());
|
||||
|
||||
var processor = new SourceStateSeedProcessor(
|
||||
documentStore,
|
||||
rawStorage,
|
||||
stateRepository,
|
||||
TimeProvider.System,
|
||||
loggerFactory.CreateLogger<SourceStateSeedProcessor>());
|
||||
|
||||
var result = await processor.ProcessAsync(specification, CancellationToken.None).ConfigureAwait(false);
|
||||
|
||||
Console.WriteLine(
|
||||
$"Seeded {result.DocumentsProcessed} document(s) for {sourceName} " +
|
||||
$"(pendingDocuments+= {result.PendingDocumentsAdded}, pendingMappings+= {result.PendingMappingsAdded}, knownAdvisories+= {result.KnownAdvisoriesAdded.Count}).");
|
||||
return 0;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.Error.WriteLine($"Error: {ex.Message}");
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<StateSeed> LoadSpecificationAsync(string inputPath)
|
||||
{
|
||||
await using var stream = File.OpenRead(inputPath);
|
||||
var seed = await JsonSerializer.DeserializeAsync<StateSeed>(stream, JsonOptions).ConfigureAwait(false)
|
||||
?? throw new InvalidOperationException("Input file deserialized to null.");
|
||||
return seed;
|
||||
}
|
||||
|
||||
private static async Task<SourceStateSeedSpecification> BuildSpecificationAsync(
|
||||
StateSeed seed,
|
||||
string sourceName,
|
||||
string inputPath,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var baseDirectory = Path.GetDirectoryName(Path.GetFullPath(inputPath)) ?? Directory.GetCurrentDirectory();
|
||||
var documents = new List<SourceStateSeedDocument>(seed.Documents.Count);
|
||||
|
||||
foreach (var documentSeed in seed.Documents)
|
||||
{
|
||||
documents.Add(await BuildDocumentAsync(documentSeed, baseDirectory, cancellationToken).ConfigureAwait(false));
|
||||
}
|
||||
|
||||
return new SourceStateSeedSpecification
|
||||
{
|
||||
Source = sourceName,
|
||||
Documents = documents.AsReadOnly(),
|
||||
Cursor = BuildCursor(seed.Cursor),
|
||||
KnownAdvisories = NormalizeStrings(seed.KnownAdvisories),
|
||||
CompletedAt = seed.CompletedAt,
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<SourceStateSeedDocument> BuildDocumentAsync(
|
||||
DocumentSeed seed,
|
||||
string baseDirectory,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(seed.Uri))
|
||||
{
|
||||
throw new InvalidOperationException("Seed entry missing 'uri'.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(seed.ContentFile))
|
||||
{
|
||||
throw new InvalidOperationException($"Seed entry for '{seed.Uri}' missing 'contentFile'.");
|
||||
}
|
||||
|
||||
var contentPath = ResolvePath(seed.ContentFile, baseDirectory);
|
||||
if (!File.Exists(contentPath))
|
||||
{
|
||||
throw new FileNotFoundException($"Content file not found for '{seed.Uri}'.", contentPath);
|
||||
}
|
||||
|
||||
var contentBytes = await File.ReadAllBytesAsync(contentPath, cancellationToken).ConfigureAwait(false);
|
||||
|
||||
var metadata = seed.Metadata is null
|
||||
? null
|
||||
: new Dictionary<string, string>(seed.Metadata, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var headers = seed.Headers is null
|
||||
? null
|
||||
: new Dictionary<string, string>(seed.Headers, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(seed.ContentType))
|
||||
{
|
||||
headers ??= new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase);
|
||||
if (!headers.ContainsKey("content-type"))
|
||||
{
|
||||
headers["content-type"] = seed.ContentType!;
|
||||
}
|
||||
}
|
||||
|
||||
return new SourceStateSeedDocument
|
||||
{
|
||||
Uri = seed.Uri,
|
||||
DocumentId = seed.DocumentId,
|
||||
Content = contentBytes,
|
||||
ContentType = seed.ContentType,
|
||||
Status = string.IsNullOrWhiteSpace(seed.Status) ? DocumentStatuses.PendingParse : seed.Status,
|
||||
Headers = headers,
|
||||
Metadata = metadata,
|
||||
Etag = seed.Etag,
|
||||
LastModified = ParseOptionalDate(seed.LastModified),
|
||||
ExpiresAt = seed.ExpiresAt,
|
||||
FetchedAt = ParseOptionalDate(seed.FetchedAt),
|
||||
AddToPendingDocuments = seed.AddToPendingDocuments,
|
||||
AddToPendingMappings = seed.AddToPendingMappings,
|
||||
KnownIdentifiers = NormalizeStrings(seed.KnownIdentifiers),
|
||||
};
|
||||
}
|
||||
|
||||
private static SourceStateSeedCursor? BuildCursor(CursorSeed? cursorSeed)
|
||||
{
|
||||
if (cursorSeed is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new SourceStateSeedCursor
|
||||
{
|
||||
PendingDocuments = NormalizeGuids(cursorSeed.PendingDocuments),
|
||||
PendingMappings = NormalizeGuids(cursorSeed.PendingMappings),
|
||||
KnownAdvisories = NormalizeStrings(cursorSeed.KnownAdvisories),
|
||||
LastModifiedCursor = cursorSeed.LastModifiedCursor,
|
||||
LastFetchAt = cursorSeed.LastFetchAt,
|
||||
Additional = cursorSeed.Additional is null
|
||||
? null
|
||||
: new Dictionary<string, string>(cursorSeed.Additional, StringComparer.OrdinalIgnoreCase),
|
||||
};
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<Guid>? NormalizeGuids(IEnumerable<Guid>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var set = new HashSet<Guid>();
|
||||
foreach (var guid in values)
|
||||
{
|
||||
if (guid != Guid.Empty)
|
||||
{
|
||||
set.Add(guid);
|
||||
}
|
||||
}
|
||||
|
||||
return set.Count == 0 ? null : set.ToList();
|
||||
}
|
||||
|
||||
private static IReadOnlyCollection<string>? NormalizeStrings(IEnumerable<string>? values)
|
||||
{
|
||||
if (values is null)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var set = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var value in values)
|
||||
{
|
||||
if (!string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
set.Add(value.Trim());
|
||||
}
|
||||
}
|
||||
|
||||
return set.Count == 0 ? null : set.ToList();
|
||||
}
|
||||
|
||||
private static DateTimeOffset? ParseOptionalDate(string? value)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(value))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.AssumeUniversal | DateTimeStyles.AdjustToUniversal);
|
||||
}
|
||||
|
||||
private static string ResolvePath(string path, string baseDirectory)
|
||||
=> Path.IsPathRooted(path) ? path : Path.GetFullPath(Path.Combine(baseDirectory, path));
|
||||
}
|
||||
|
||||
internal sealed record SeedOptions
|
||||
{
|
||||
public required string ConnectionString { get; init; }
|
||||
public required string DatabaseName { get; init; }
|
||||
public required string InputPath { get; init; }
|
||||
public string? SourceName { get; init; }
|
||||
|
||||
public static SeedOptions? Parse(string[] args)
|
||||
{
|
||||
string? connectionString = null;
|
||||
string? database = null;
|
||||
string? input = null;
|
||||
string? source = null;
|
||||
|
||||
for (var i = 0; i < args.Length; i++)
|
||||
{
|
||||
var arg = args[i];
|
||||
switch (arg)
|
||||
{
|
||||
case "--connection-string":
|
||||
case "-c":
|
||||
connectionString = TakeValue(args, ref i, arg);
|
||||
break;
|
||||
case "--database":
|
||||
case "-d":
|
||||
database = TakeValue(args, ref i, arg);
|
||||
break;
|
||||
case "--input":
|
||||
case "-i":
|
||||
input = TakeValue(args, ref i, arg);
|
||||
break;
|
||||
case "--source":
|
||||
case "-s":
|
||||
source = TakeValue(args, ref i, arg);
|
||||
break;
|
||||
case "--help":
|
||||
case "-h":
|
||||
return null;
|
||||
default:
|
||||
Console.Error.WriteLine($"Unrecognized argument '{arg}'.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(connectionString) || string.IsNullOrWhiteSpace(database) || string.IsNullOrWhiteSpace(input))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
return new SeedOptions
|
||||
{
|
||||
ConnectionString = connectionString,
|
||||
DatabaseName = database,
|
||||
InputPath = input,
|
||||
SourceName = source,
|
||||
};
|
||||
}
|
||||
|
||||
public static void PrintUsage()
|
||||
{
|
||||
Console.WriteLine("Usage: dotnet run --project tools/SourceStateSeeder -- --connection-string <connection> --database <name> --input <seed.json> [--source <source>]");
|
||||
}
|
||||
|
||||
private static string TakeValue(string[] args, ref int index, string arg)
|
||||
{
|
||||
if (index + 1 >= args.Length)
|
||||
{
|
||||
throw new ArgumentException($"Missing value for {arg}.");
|
||||
}
|
||||
|
||||
index++;
|
||||
return args[index];
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed record StateSeed
|
||||
{
|
||||
public string? Source { get; init; }
|
||||
public List<DocumentSeed> Documents { get; init; } = new();
|
||||
public CursorSeed? Cursor { get; init; }
|
||||
public List<string>? KnownAdvisories { get; init; }
|
||||
public DateTimeOffset? CompletedAt { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record DocumentSeed
|
||||
{
|
||||
public string Uri { get; init; } = string.Empty;
|
||||
public string ContentFile { get; init; } = string.Empty;
|
||||
public Guid? DocumentId { get; init; }
|
||||
public string? ContentType { get; init; }
|
||||
public Dictionary<string, string>? Metadata { get; init; }
|
||||
public Dictionary<string, string>? Headers { get; init; }
|
||||
public string Status { get; init; } = DocumentStatuses.PendingParse;
|
||||
public bool AddToPendingDocuments { get; init; } = true;
|
||||
public bool AddToPendingMappings { get; init; }
|
||||
public string? LastModified { get; init; }
|
||||
public string? FetchedAt { get; init; }
|
||||
public string? Etag { get; init; }
|
||||
public DateTimeOffset? ExpiresAt { get; init; }
|
||||
public List<string>? KnownIdentifiers { get; init; }
|
||||
}
|
||||
|
||||
internal sealed record CursorSeed
|
||||
{
|
||||
public List<Guid>? PendingDocuments { get; init; }
|
||||
public List<Guid>? PendingMappings { get; init; }
|
||||
public List<string>? KnownAdvisories { get; init; }
|
||||
public DateTimeOffset? LastModifiedCursor { get; init; }
|
||||
public DateTimeOffset? LastFetchAt { get; init; }
|
||||
public Dictionary<string, string>? Additional { get; init; }
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\..\src\StellaOps.Concelier.Connector.Common\StellaOps.Concelier.Connector.Common.csproj" />
|
||||
<ProjectReference Include="..\..\src\StellaOps.Concelier.Storage.Mongo\StellaOps.Concelier.Storage.Mongo.csproj" />
|
||||
</ItemGroup>
|
||||
</Project>
|
||||
@@ -1,444 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Capture CERT-Bund search/export JSON snapshots and generate Offline Kit manifests.
|
||||
|
||||
The script can bootstrap a session against https://wid.cert-bund.de, fetch
|
||||
paginated search results plus per-year export payloads, and emit a manifest
|
||||
that records source, date range, SHA-256, and capture timestamps for each artefact.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import datetime as dt
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path, PurePosixPath
|
||||
import sys
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from http.cookiejar import MozillaCookieJar
|
||||
from typing import Any, Dict, Iterable, List, Optional
|
||||
|
||||
|
||||
PORTAL_ROOT = "https://wid.cert-bund.de/portal/"
|
||||
SEARCH_ENDPOINT = "https://wid.cert-bund.de/portal/api/securityadvisory/search"
|
||||
EXPORT_ENDPOINT = "https://wid.cert-bund.de/portal/api/securityadvisory/export"
|
||||
CSRF_ENDPOINT = "https://wid.cert-bund.de/portal/api/security/csrf"
|
||||
USER_AGENT = "StellaOps.CertBundOffline/0.1"
|
||||
|
||||
UTC = dt.timezone.utc
|
||||
|
||||
|
||||
class CertBundClient:
|
||||
def __init__(
|
||||
self,
|
||||
cookie_file: Optional[Path] = None,
|
||||
xsrf_token: Optional[str] = None,
|
||||
auto_bootstrap: bool = True,
|
||||
) -> None:
|
||||
self.cookie_path = cookie_file
|
||||
self.cookie_jar = MozillaCookieJar()
|
||||
|
||||
if self.cookie_path and self.cookie_path.exists():
|
||||
self.cookie_jar.load(self.cookie_path, ignore_discard=True, ignore_expires=True)
|
||||
|
||||
self.opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(self.cookie_jar))
|
||||
self.opener.addheaders = [("User-Agent", USER_AGENT)]
|
||||
|
||||
self._xsrf_token = xsrf_token
|
||||
self.auto_bootstrap = auto_bootstrap
|
||||
|
||||
if self.auto_bootstrap and not self._xsrf_token:
|
||||
self._bootstrap()
|
||||
|
||||
@property
|
||||
def xsrf_token(self) -> str:
|
||||
if self._xsrf_token:
|
||||
return self._xsrf_token
|
||||
|
||||
token = _extract_cookie_value(self.cookie_jar, "XSRF-TOKEN")
|
||||
if token:
|
||||
self._xsrf_token = token
|
||||
return token
|
||||
|
||||
raise RuntimeError(
|
||||
"CERT-Bund XSRF token not available. Provide --xsrf-token or a cookie file "
|
||||
"containing XSRF-TOKEN (see docs/ops/concelier-certbund-operations.md)."
|
||||
)
|
||||
|
||||
def fetch_search_pages(
|
||||
self,
|
||||
destination: Path,
|
||||
page_size: int,
|
||||
max_pages: int,
|
||||
) -> None:
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for page in range(max_pages):
|
||||
payload = {
|
||||
"page": page,
|
||||
"size": page_size,
|
||||
"sort": ["published,desc"],
|
||||
}
|
||||
try:
|
||||
document = self._post_json(SEARCH_ENDPOINT, payload)
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise RuntimeError(
|
||||
f"Failed to fetch CERT-Bund search page {page}: HTTP {exc.code}. "
|
||||
"Double-check the XSRF token or portal cookies."
|
||||
) from exc
|
||||
|
||||
content = document.get("content") or []
|
||||
if not content and page > 0:
|
||||
break
|
||||
|
||||
file_path = destination / f"certbund-search-page-{page:02d}.json"
|
||||
_write_pretty_json(file_path, document)
|
||||
print(f"[certbund] wrote search page {page:02d} → {file_path}")
|
||||
|
||||
if not content:
|
||||
break
|
||||
|
||||
self._persist_cookies()
|
||||
|
||||
def fetch_exports(self, destination: Path, start_year: int, end_year: int) -> None:
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
for year in range(start_year, end_year + 1):
|
||||
from_value = f"{year}-01-01"
|
||||
to_value = f"{year}-12-31"
|
||||
query = urllib.parse.urlencode({"format": "json", "from": from_value, "to": to_value})
|
||||
url = f"{EXPORT_ENDPOINT}?{query}"
|
||||
try:
|
||||
document = self._get_json(url)
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise RuntimeError(
|
||||
f"Failed to fetch CERT-Bund export for {year}: HTTP {exc.code}. "
|
||||
"Ensure the XSRF token and cookies are valid."
|
||||
) from exc
|
||||
|
||||
file_path = destination / f"certbund-export-{year}.json"
|
||||
_write_pretty_json(file_path, document)
|
||||
print(f"[certbund] wrote export {year} → {file_path}")
|
||||
|
||||
self._persist_cookies()
|
||||
|
||||
def _bootstrap(self) -> None:
|
||||
try:
|
||||
self._request("GET", PORTAL_ROOT, headers={"Accept": "text/html,application/xhtml+xml"})
|
||||
except urllib.error.HTTPError as exc:
|
||||
raise RuntimeError(f"Failed to bootstrap CERT-Bund session: HTTP {exc.code}") from exc
|
||||
|
||||
# First attempt to obtain CSRF token directly.
|
||||
self._attempt_csrf_fetch()
|
||||
|
||||
if _extract_cookie_value(self.cookie_jar, "XSRF-TOKEN"):
|
||||
return
|
||||
|
||||
# If the token is still missing, trigger the search endpoint once (likely 403)
|
||||
# to make the portal materialise JSESSIONID, then retry token acquisition.
|
||||
try:
|
||||
payload = {"page": 0, "size": 1, "sort": ["published,desc"]}
|
||||
self._post_json(SEARCH_ENDPOINT, payload, include_token=False)
|
||||
except urllib.error.HTTPError:
|
||||
pass
|
||||
|
||||
self._attempt_csrf_fetch()
|
||||
|
||||
token = _extract_cookie_value(self.cookie_jar, "XSRF-TOKEN")
|
||||
if token:
|
||||
self._xsrf_token = token
|
||||
else:
|
||||
print(
|
||||
"[certbund] warning: automatic XSRF token retrieval failed. "
|
||||
"Supply --xsrf-token or reuse a browser-exported cookies file.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
def _attempt_csrf_fetch(self) -> None:
|
||||
headers = {
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Origin": "https://wid.cert-bund.de",
|
||||
"Referer": PORTAL_ROOT,
|
||||
}
|
||||
try:
|
||||
self._request("GET", CSRF_ENDPOINT, headers=headers)
|
||||
except urllib.error.HTTPError:
|
||||
pass
|
||||
|
||||
def _request(self, method: str, url: str, data: Optional[bytes] = None, headers: Optional[Dict[str, str]] = None) -> bytes:
|
||||
request = urllib.request.Request(url, data=data, method=method)
|
||||
default_headers = {
|
||||
"User-Agent": USER_AGENT,
|
||||
"Accept": "application/json",
|
||||
}
|
||||
for key, value in default_headers.items():
|
||||
request.add_header(key, value)
|
||||
|
||||
if headers:
|
||||
for key, value in headers.items():
|
||||
request.add_header(key, value)
|
||||
|
||||
return self.opener.open(request, timeout=60).read()
|
||||
|
||||
def _post_json(self, url: str, payload: Dict[str, Any], include_token: bool = True) -> Dict[str, Any]:
|
||||
data = json.dumps(payload).encode("utf-8")
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"Accept": "application/json",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Origin": "https://wid.cert-bund.de",
|
||||
"Referer": PORTAL_ROOT,
|
||||
}
|
||||
if include_token:
|
||||
headers["X-XSRF-TOKEN"] = self.xsrf_token
|
||||
|
||||
raw = self._request("POST", url, data=data, headers=headers)
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
|
||||
def _get_json(self, url: str) -> Any:
|
||||
headers = {
|
||||
"Accept": "application/json",
|
||||
"X-Requested-With": "XMLHttpRequest",
|
||||
"Referer": PORTAL_ROOT,
|
||||
}
|
||||
headers["X-XSRF-TOKEN"] = self.xsrf_token
|
||||
|
||||
raw = self._request("GET", url, headers=headers)
|
||||
return json.loads(raw.decode("utf-8"))
|
||||
|
||||
def _persist_cookies(self) -> None:
|
||||
if not self.cookie_path:
|
||||
return
|
||||
|
||||
self.cookie_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self.cookie_jar.save(self.cookie_path, ignore_discard=True, ignore_expires=True)
|
||||
|
||||
|
||||
def _extract_cookie_value(jar: MozillaCookieJar, name: str) -> Optional[str]:
|
||||
for cookie in jar:
|
||||
if cookie.name == name:
|
||||
return cookie.value
|
||||
return None
|
||||
|
||||
|
||||
def _write_pretty_json(path: Path, document: Any) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
json.dump(document, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
|
||||
|
||||
def scan_artifacts(root: Path) -> List[Dict[str, Any]]:
|
||||
records: List[Dict[str, Any]] = []
|
||||
search_dir = root / "search"
|
||||
export_dir = root / "export"
|
||||
|
||||
if search_dir.exists():
|
||||
for file_path in sorted(search_dir.glob("certbund-search-page-*.json")):
|
||||
record = _build_search_record(file_path)
|
||||
records.append(record)
|
||||
|
||||
if export_dir.exists():
|
||||
for file_path in sorted(export_dir.glob("certbund-export-*.json")):
|
||||
record = _build_export_record(file_path)
|
||||
records.append(record)
|
||||
|
||||
return records
|
||||
|
||||
|
||||
def _build_search_record(path: Path) -> Dict[str, Any]:
|
||||
with path.open("r", encoding="utf-8") as handle:
|
||||
data = json.load(handle)
|
||||
|
||||
content = data.get("content") or []
|
||||
published_values: List[str] = []
|
||||
for item in content:
|
||||
published = (
|
||||
item.get("published")
|
||||
or item.get("publishedAt")
|
||||
or item.get("datePublished")
|
||||
or item.get("published_date")
|
||||
)
|
||||
if isinstance(published, str):
|
||||
published_values.append(published)
|
||||
|
||||
if published_values:
|
||||
try:
|
||||
ordered = sorted(_parse_iso_timestamp(value) for value in published_values if value)
|
||||
range_from = ordered[0].isoformat()
|
||||
range_to = ordered[-1].isoformat()
|
||||
except ValueError:
|
||||
range_from = range_to = None
|
||||
else:
|
||||
range_from = range_to = None
|
||||
|
||||
return {
|
||||
"type": "search",
|
||||
"path": path,
|
||||
"source": "concelier.cert-bund.search",
|
||||
"itemCount": len(content),
|
||||
"from": range_from,
|
||||
"to": range_to,
|
||||
"capturedAt": _timestamp_from_stat(path),
|
||||
}
|
||||
|
||||
|
||||
def _build_export_record(path: Path) -> Dict[str, Any]:
|
||||
year = _extract_year_from_filename(path.name)
|
||||
if year is not None:
|
||||
from_value = f"{year}-01-01"
|
||||
to_value = f"{year}-12-31"
|
||||
else:
|
||||
from_value = None
|
||||
to_value = None
|
||||
|
||||
return {
|
||||
"type": "export",
|
||||
"path": path,
|
||||
"source": "concelier.cert-bund.export",
|
||||
"itemCount": None,
|
||||
"from": from_value,
|
||||
"to": to_value,
|
||||
"capturedAt": _timestamp_from_stat(path),
|
||||
}
|
||||
|
||||
|
||||
def _timestamp_from_stat(path: Path) -> str:
|
||||
stat = path.stat()
|
||||
return dt.datetime.fromtimestamp(stat.st_mtime, tz=UTC).isoformat()
|
||||
|
||||
|
||||
def _extract_year_from_filename(name: str) -> Optional[int]:
|
||||
stem = Path(name).stem
|
||||
parts = stem.split("-")
|
||||
if parts and parts[-1].isdigit() and len(parts[-1]) == 4:
|
||||
return int(parts[-1])
|
||||
return None
|
||||
|
||||
|
||||
def _parse_iso_timestamp(value: str) -> dt.datetime:
|
||||
try:
|
||||
return dt.datetime.fromisoformat(value.replace("Z", "+00:00"))
|
||||
except ValueError:
|
||||
# Fallback for formats like 2025-10-14T06:24:49
|
||||
return dt.datetime.strptime(value, "%Y-%m-%dT%H:%M:%S").replace(tzinfo=UTC)
|
||||
|
||||
|
||||
def build_manifest(root: Path, records: Iterable[Dict[str, Any]], manifest_path: Path) -> None:
|
||||
manifest_entries = []
|
||||
for record in records:
|
||||
path = record["path"]
|
||||
rel_path = PurePosixPath(path.relative_to(root).as_posix())
|
||||
sha256 = hashlib.sha256(path.read_bytes()).hexdigest()
|
||||
size = path.stat().st_size
|
||||
|
||||
entry = {
|
||||
"source": record["source"],
|
||||
"type": record["type"],
|
||||
"path": str(rel_path),
|
||||
"sha256": sha256,
|
||||
"sizeBytes": size,
|
||||
"capturedAt": record["capturedAt"],
|
||||
"from": record.get("from"),
|
||||
"to": record.get("to"),
|
||||
"itemCount": record.get("itemCount"),
|
||||
}
|
||||
manifest_entries.append(entry)
|
||||
|
||||
sha_file = path.with_suffix(path.suffix + ".sha256")
|
||||
_write_sha_file(sha_file, sha256, path.name)
|
||||
|
||||
manifest_entries.sort(key=lambda item: item["path"])
|
||||
|
||||
manifest_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
manifest_document = {
|
||||
"source": "concelier.cert-bund",
|
||||
"generatedAt": dt.datetime.now(tz=UTC).isoformat(),
|
||||
"artifacts": manifest_entries,
|
||||
}
|
||||
|
||||
with manifest_path.open("w", encoding="utf-8") as handle:
|
||||
json.dump(manifest_document, handle, ensure_ascii=False, indent=2, sort_keys=True)
|
||||
handle.write("\n")
|
||||
|
||||
manifest_sha = hashlib.sha256(manifest_path.read_bytes()).hexdigest()
|
||||
_write_sha_file(manifest_path.with_suffix(".sha256"), manifest_sha, manifest_path.name)
|
||||
|
||||
print(f"[certbund] manifest generated → {manifest_path}")
|
||||
|
||||
|
||||
def _write_sha_file(path: Path, digest: str, filename: str) -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with path.open("w", encoding="utf-8") as handle:
|
||||
handle.write(f"{digest} {filename}\n")
|
||||
|
||||
|
||||
def parse_args(argv: Optional[List[str]] = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Capture CERT-Bund search/export snapshots for Offline Kit packaging.",
|
||||
)
|
||||
parser.add_argument("--output", default="seed-data/cert-bund", help="Destination directory for artefacts.")
|
||||
parser.add_argument("--start-year", type=int, default=2014, help="First year (inclusive) for export snapshots.")
|
||||
parser.add_argument(
|
||||
"--end-year",
|
||||
type=int,
|
||||
default=dt.datetime.now(tz=UTC).year,
|
||||
help="Last year (inclusive) for export snapshots.",
|
||||
)
|
||||
parser.add_argument("--page-size", type=int, default=100, help="Search page size.")
|
||||
parser.add_argument("--max-pages", type=int, default=12, help="Maximum number of search result pages to capture.")
|
||||
parser.add_argument("--cookie-file", type=Path, help="Path to a Netscape cookie file to reuse/persist session cookies.")
|
||||
parser.add_argument("--xsrf-token", help="Optional explicit XSRF token value (overrides cookie discovery).")
|
||||
parser.add_argument(
|
||||
"--skip-fetch",
|
||||
action="store_true",
|
||||
help="Skip HTTP fetches and only regenerate manifest from existing files.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-bootstrap",
|
||||
action="store_true",
|
||||
help="Do not attempt automatic session bootstrap (use with --skip-fetch or pre-populated cookies).",
|
||||
)
|
||||
return parser.parse_args(argv)
|
||||
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> int:
|
||||
args = parse_args(argv)
|
||||
output_dir = Path(args.output).expanduser().resolve()
|
||||
|
||||
if not args.skip_fetch:
|
||||
client = CertBundClient(
|
||||
cookie_file=args.cookie_file,
|
||||
xsrf_token=args.xsrf_token,
|
||||
auto_bootstrap=not args.no_bootstrap,
|
||||
)
|
||||
|
||||
start_year = args.start_year
|
||||
end_year = args.end_year
|
||||
if start_year > end_year:
|
||||
raise SystemExit("start-year cannot be greater than end-year.")
|
||||
|
||||
client.fetch_search_pages(output_dir / "search", args.page_size, args.max_pages)
|
||||
client.fetch_exports(output_dir / "export", start_year, end_year)
|
||||
|
||||
records = scan_artifacts(output_dir)
|
||||
if not records:
|
||||
print(
|
||||
"[certbund] no artefacts discovered. Fetch data first or point --output to the dataset directory.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
manifest_path = output_dir / "manifest" / "certbund-offline-manifest.json"
|
||||
build_manifest(output_dir, records, manifest_path)
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Binary file not shown.
Binary file not shown.
Reference in New Issue
Block a user