devops folders consolidate

2026-01-25 23:27:41 +02:00
parent 6e687b523a
commit a50bbb38ef
334 changed files with 35079 additions and 5569 deletions
--- a/deploy/tools/ci/determinism/compare-platform-hashes.py
+++ b/deploy/tools/ci/determinism/compare-platform-hashes.py
@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""
+Cross-platform hash comparison for determinism verification.
+Sprint: SPRINT_20251226_007_BE_determinism_gaps
+Task: DET-GAP-13 - Cross-platform hash comparison report generation
+"""
+
+import argparse
+import json
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+
+def load_hashes(path: str) -> dict[str, str]:
+    """Load hash file from path."""
+    with open(path) as f:
+        data = json.load(f)
+    return data.get("hashes", data)
+
+
+def compare_hashes(
+    linux: dict[str, str],
+    windows: dict[str, str],
+    macos: dict[str, str]
+) -> tuple[list[dict], list[str]]:
+    """
+    Compare hashes across platforms.
+    Returns (divergences, matched_keys).
+    """
+    all_keys = set(linux.keys()) | set(windows.keys()) | set(macos.keys())
+    divergences = []
+    matched = []
+
+    for key in sorted(all_keys):
+        linux_hash = linux.get(key, "MISSING")
+        windows_hash = windows.get(key, "MISSING")
+        macos_hash = macos.get(key, "MISSING")
+
+        if linux_hash == windows_hash == macos_hash:
+            matched.append(key)
+        else:
+            divergences.append({
+                "key": key,
+                "linux": linux_hash,
+                "windows": windows_hash,
+                "macos": macos_hash
+            })
+
+    return divergences, matched
+
+
+def generate_markdown_report(
+    divergences: list[dict],
+    matched: list[str],
+    linux_path: str,
+    windows_path: str,
+    macos_path: str
+) -> str:
+    """Generate Markdown report."""
+    lines = [
+        f"**Generated:** {datetime.now(timezone.utc).isoformat()}",
+        "",
+        "### Summary",
+        "",
+        f"- ✅ **Matched:** {len(matched)} hashes",
+        f"- {'❌' if divergences else '✅'} **Divergences:** {len(divergences)} hashes",
+        "",
+    ]
+
+    if divergences:
+        lines.extend([
+            "### Divergences",
+            "",
+            "| Key | Linux | Windows | macOS |",
+            "|-----|-------|---------|-------|",
+        ])
+        for d in divergences:
+            linux_short = d["linux"][:16] + "..." if len(d["linux"]) > 16 else d["linux"]
+            windows_short = d["windows"][:16] + "..." if len(d["windows"]) > 16 else d["windows"]
+            macos_short = d["macos"][:16] + "..." if len(d["macos"]) > 16 else d["macos"]
+            lines.append(f"| `{d['key']}` | `{linux_short}` | `{windows_short}` | `{macos_short}` |")
+        lines.append("")
+
+    lines.extend([
+        "### Matched Hashes",
+        "",
+        f"<details><summary>Show {len(matched)} matched hashes</summary>",
+        "",
+    ])
+    for key in matched[:50]:  # Limit display
+        lines.append(f"- `{key}`")
+    if len(matched) > 50:
+        lines.append(f"- ... and {len(matched) - 50} more")
+    lines.extend(["", "</details>", ""])
+
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Compare determinism hashes across platforms")
+    parser.add_argument("--linux", required=True, help="Path to Linux hashes JSON")
+    parser.add_argument("--windows", required=True, help="Path to Windows hashes JSON")
+    parser.add_argument("--macos", required=True, help="Path to macOS hashes JSON")
+    parser.add_argument("--output", required=True, help="Output JSON report path")
+    parser.add_argument("--markdown", required=True, help="Output Markdown report path")
+    args = parser.parse_args()
+
+    # Load hashes
+    linux_hashes = load_hashes(args.linux)
+    windows_hashes = load_hashes(args.windows)
+    macos_hashes = load_hashes(args.macos)
+
+    # Compare
+    divergences, matched = compare_hashes(linux_hashes, windows_hashes, macos_hashes)
+
+    # Generate reports
+    report = {
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+        "sources": {
+            "linux": args.linux,
+            "windows": args.windows,
+            "macos": args.macos
+        },
+        "summary": {
+            "matched": len(matched),
+            "divergences": len(divergences),
+            "total": len(matched) + len(divergences)
+        },
+        "divergences": divergences,
+        "matched": matched
+    }
+
+    # Write JSON report
+    Path(args.output).parent.mkdir(parents=True, exist_ok=True)
+    with open(args.output, "w") as f:
+        json.dump(report, f, indent=2)
+
+    # Write Markdown report
+    markdown = generate_markdown_report(
+        divergences, matched,
+        args.linux, args.windows, args.macos
+    )
+    with open(args.markdown, "w") as f:
+        f.write(markdown)
+
+    # Print summary
+    print(f"Comparison complete:")
+    print(f"  Matched: {len(matched)}")
+    print(f"  Divergences: {len(divergences)}")
+
+    # Exit with error if divergences found
+    if divergences:
+        print("\nERROR: Hash divergences detected!")
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()
--- a/deploy/tools/ci/nuget-prime/__Tests/NugetPrime.Tests/GlobalUsings.cs
+++ b/deploy/tools/ci/nuget-prime/__Tests/NugetPrime.Tests/GlobalUsings.cs
@@ -0,0 +1 @@
+global using Xunit;
--- a/deploy/tools/ci/nuget-prime/__Tests/NugetPrime.Tests/NugetPrime.Tests.csproj
+++ b/deploy/tools/ci/nuget-prime/__Tests/NugetPrime.Tests/NugetPrime.Tests.csproj
@@ -0,0 +1,16 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <IsTestProject>true</IsTestProject>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+    <LangVersion>preview</LangVersion>
+    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="FluentAssertions" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" />
+    <PackageReference Include="xunit" />
+    <PackageReference Include="xunit.runner.visualstudio" />
+  </ItemGroup>
+</Project>
--- a/deploy/tools/ci/nuget-prime/__Tests/NugetPrime.Tests/NugetPrimeTests.cs
+++ b/deploy/tools/ci/nuget-prime/__Tests/NugetPrime.Tests/NugetPrimeTests.cs
@@ -0,0 +1,48 @@
+using System.Xml.Linq;
+using FluentAssertions;
+
+namespace NugetPrime.Tests;
+
+public sealed class NugetPrimeTests
+{
+    [Theory]
+    [InlineData("nuget-prime.csproj")]
+    [InlineData("nuget-prime-v9.csproj")]
+    public void PackageDownloads_ArePinned(string projectFile)
+    {
+        var repoRoot = FindRepoRoot();
+        var path = Path.Combine(repoRoot, "devops", "tools", "nuget-prime", projectFile);
+        File.Exists(path).Should().BeTrue($"expected {projectFile} under devops/tools/nuget-prime");
+
+        var doc = XDocument.Load(path);
+        var packages = doc.Descendants().Where(element => element.Name.LocalName == "PackageDownload").ToList();
+        packages.Should().NotBeEmpty();
+
+        foreach (var package in packages)
+        {
+            var include = package.Attribute("Include")?.Value;
+            include.Should().NotBeNullOrWhiteSpace();
+
+            var version = package.Attribute("Version")?.Value;
+            version.Should().NotBeNullOrWhiteSpace();
+            version.Should().NotContain("*");
+        }
+    }
+
+    private static string FindRepoRoot()
+    {
+        var current = new DirectoryInfo(AppContext.BaseDirectory);
+        for (var i = 0; i < 12 && current is not null; i++)
+        {
+            var candidate = Path.Combine(current.FullName, "devops", "tools", "nuget-prime", "nuget-prime.csproj");
+            if (File.Exists(candidate))
+            {
+                return current.FullName;
+            }
+
+            current = current.Parent;
+        }
+
+        throw new DirectoryNotFoundException("Repo root not found for devops/tools/nuget-prime");
+    }
+}
--- a/deploy/tools/ci/nuget-prime/mirror-packages.txt
+++ b/deploy/tools/ci/nuget-prime/mirror-packages.txt
@@ -0,0 +1,30 @@
+AWSSDK.S3|3.7.305.6
+CycloneDX.Core|10.0.1
+Google.Protobuf|3.27.2
+Grpc.Net.Client|2.65.0
+Grpc.Tools|2.65.0
+Microsoft.Data.Sqlite|9.0.0-rc.1.24451.1
+Microsoft.Extensions.Configuration.Abstractions|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Configuration.Abstractions|9.0.0
+Microsoft.Extensions.Configuration.Binder|10.0.0-rc.2.25502.107
+Microsoft.Extensions.DependencyInjection.Abstractions|10.0.0-rc.2.25502.107
+Microsoft.Extensions.DependencyInjection.Abstractions|9.0.0
+Microsoft.Extensions.Diagnostics.Abstractions|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Diagnostics.HealthChecks|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Hosting.Abstractions|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Http.Polly|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Http|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Logging.Abstractions|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Logging.Abstractions|9.0.0
+Microsoft.Extensions.Options.ConfigurationExtensions|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Options|10.0.0-rc.2.25502.107
+Microsoft.Extensions.Options|9.0.0
+Npgsql|9.0.3
+Npgsql.EntityFrameworkCore.PostgreSQL|9.0.3
+RoaringBitmap|0.0.9
+Serilog.AspNetCore|8.0.1
+Serilog.Extensions.Hosting|8.0.0
+Serilog.Sinks.Console|5.0.1
+StackExchange.Redis|2.7.33
+System.Text.Json|10.0.0-preview.7.25380.108
--- a/deploy/tools/ci/nuget-prime/nuget-prime-v9.csproj
+++ b/deploy/tools/ci/nuget-prime/nuget-prime-v9.csproj
@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <RestorePackagesPath>../../.nuget/packages</RestorePackagesPath>
+    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
+    <EnableDefaultItems>false</EnableDefaultItems>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageDownload Include="Microsoft.Extensions.Configuration.Abstractions" Version="[9.0.0]" />
+    <PackageDownload Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="[9.0.0]" />
+    <PackageDownload Include="Microsoft.Extensions.Logging.Abstractions" Version="[9.0.0]" />
+    <PackageDownload Include="Microsoft.Extensions.Options" Version="[9.0.0]" />
+  </ItemGroup>
+</Project>
--- a/deploy/tools/ci/nuget-prime/nuget-prime.csproj
+++ b/deploy/tools/ci/nuget-prime/nuget-prime.csproj
@@ -0,0 +1,45 @@
+<Project Sdk="Microsoft.NET.Sdk">
+  <PropertyGroup>
+    <TargetFramework>net10.0</TargetFramework>
+    <RestorePackagesPath>../../.nuget/packages</RestorePackagesPath>
+    <DisableImplicitFrameworkReferences>true</DisableImplicitFrameworkReferences>
+    <EnableDefaultItems>false</EnableDefaultItems>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageDownload Include="AWSSDK.Core" Version="[4.0.1.3]" />
+    <PackageDownload Include="AWSSDK.KeyManagementService" Version="[4.0.6]" />
+    <PackageDownload Include="AWSSDK.S3" Version="[3.7.305.6]" />
+    <PackageDownload Include="CycloneDX.Core" Version="[10.0.2]" />
+    <PackageDownload Include="Google.Protobuf" Version="[3.27.2]" />
+    <PackageDownload Include="Grpc.Net.Client" Version="[2.65.0]" />
+    <PackageDownload Include="Grpc.Tools" Version="[2.65.0]" />
+    <PackageDownload Include="Microsoft.Data.Sqlite" Version="[9.0.0-rc.1.24451.1]" />
+    <PackageDownload Include="Microsoft.Extensions.Configuration.Abstractions" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Configuration.Binder" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Diagnostics.Abstractions" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Diagnostics.HealthChecks.Abstractions" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Diagnostics.HealthChecks" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Hosting.Abstractions" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Http.Polly" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Http" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Logging.Abstractions" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Options.ConfigurationExtensions" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Microsoft.Extensions.Options" Version="[10.0.0-rc.2.25502.107]" />
+    <PackageDownload Include="Npgsql" Version="[9.0.3]" />
+    <PackageDownload Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="[9.0.3]" />
+    <PackageDownload Include="RoaringBitmap" Version="[0.0.9]" />
+    <PackageDownload Include="Serilog.AspNetCore" Version="[8.0.1]" />
+    <PackageDownload Include="Serilog.Extensions.Hosting" Version="[8.0.0]" />
+    <PackageDownload Include="Serilog.Sinks.Console" Version="[5.0.1]" />
+    <PackageDownload Include="StackExchange.Redis" Version="[2.8.37]" />
+    <PackageDownload Include="System.Text.Json" Version="[10.0.0-preview.7.25380.108]" />
+    <PackageDownload Include="Google.Api.CommonProtos" Version="[2.17.0]" />
+    <PackageDownload Include="Google.Api.Gax" Version="[4.11.0]" />
+    <PackageDownload Include="Google.Api.Gax.Grpc" Version="[4.11.0]" />
+    <PackageDownload Include="Google.Api.Gax.Grpc.GrpcCore" Version="[4.11.0]" />
+    <PackageDownload Include="Google.Apis" Version="[1.69.0]" />
+    <PackageDownload Include="Google.Apis.Auth" Version="[1.69.0]" />
+    <PackageDownload Include="Google.Apis.Core" Version="[1.64.0]" />
+  </ItemGroup>
+</Project>