work work hard work

This commit is contained in:
StellaOps Bot
2025-12-18 00:47:24 +02:00
parent dee252940b
commit b4235c134c
189 changed files with 9627 additions and 3258 deletions

View File

@@ -0,0 +1,25 @@
# StellaOps.Attestor.Persistence — Local Agent Charter
## Scope
- This charter applies to `src/Attestor/__Libraries/StellaOps.Attestor.Persistence/**`.
## Primary roles
- Backend engineer (C# / .NET 10, EF Core, Npgsql).
- QA automation engineer (xUnit) for persistence + matcher logic.
## Required reading (treat as read before edits)
- `docs/modules/attestor/architecture.md`
- `docs/db/SPECIFICATION.md`
- `docs/db/MIGRATION_STRATEGY.md`
- PostgreSQL 16 docs (arrays, indexes, JSONB, query plans).
## Working agreements
- Determinism is mandatory where hashes/IDs are produced; all timestamps are UTC.
- Offline-friendly defaults: no network calls from library code paths.
- Migrations must be idempotent and safe to re-run.
- Prefer small, composable services with explicit interfaces (`I*`).
## Testing expectations
- Unit/integration tests live in `src/Attestor/__Tests/StellaOps.Attestor.Persistence.Tests`.
- Perf dataset and query harness lives under `src/Attestor/__Libraries/StellaOps.Attestor.Persistence/Perf` and must be deterministic (fixed data, fixed sizes, documented parameters).

View File

@@ -5,6 +5,9 @@
-- Create schema
CREATE SCHEMA IF NOT EXISTS proofchain;
-- Required for gen_random_uuid() defaults
CREATE EXTENSION IF NOT EXISTS pgcrypto;
-- Create verification_result enum type
DO $$
BEGIN

View File

@@ -0,0 +1,18 @@
# ProofChain DB perf harness
This folder provides a deterministic, production-like dataset and a small harness to validate index/query performance for the ProofChain schema (`proofchain.*`).
## Files
- `seed.sql` deterministic dataset generator (uses SQL functions + `generate_series`).
- `queries.sql` representative queries with `EXPLAIN (ANALYZE, BUFFERS)`.
- `run-perf.ps1` starts a local PostgreSQL 16 container, applies migrations, seeds data, runs queries, and captures output.
## Run
From repo root:
```powershell
pwsh -File src/Attestor/__Libraries/StellaOps.Attestor.Persistence/Perf/run-perf.ps1
```
Output is written to `docs/db/reports/proofchain-schema-perf-2025-12-17.md`.

View File

@@ -0,0 +1,57 @@
-- Representative query set for ProofChain schema perf validation.
-- Run after applying migrations + seeding (`seed.sql`).
\timing on
-- Row counts
SELECT
(SELECT count(*) FROM proofchain.trust_anchors) AS trust_anchors,
(SELECT count(*) FROM proofchain.sbom_entries) AS sbom_entries,
(SELECT count(*) FROM proofchain.dsse_envelopes) AS dsse_envelopes,
(SELECT count(*) FROM proofchain.spines) AS spines,
(SELECT count(*) FROM proofchain.rekor_entries) AS rekor_entries;
-- 1) SBOM entry lookup via unique constraint (bom_digest, purl, version)
EXPLAIN (ANALYZE, BUFFERS)
SELECT entry_id, bom_digest, purl, version
FROM proofchain.sbom_entries
WHERE bom_digest = proofchain.hex64('bom:1')
AND purl = format('pkg:npm/vendor-%02s/pkg-%05s', 1, 1)
AND version = '1.0.1';
-- 2) Fetch all entries for a given SBOM digest (index on bom_digest)
EXPLAIN (ANALYZE, BUFFERS)
SELECT entry_id, purl, version
FROM proofchain.sbom_entries
WHERE bom_digest = proofchain.hex64('bom:1')
ORDER BY purl
LIMIT 100;
-- 3) Envelopes for entry + predicate (compound index)
EXPLAIN (ANALYZE, BUFFERS)
SELECT env_id, predicate_type, signer_keyid, body_hash
FROM proofchain.dsse_envelopes
WHERE entry_id = proofchain.uuid_from_text('entry:1')
AND predicate_type = 'evidence.stella/v1';
-- 4) Spine lookup via bundle_id (unique index)
EXPLAIN (ANALYZE, BUFFERS)
SELECT entry_id, bundle_id, policy_version
FROM proofchain.spines
WHERE bundle_id = proofchain.hex64('bundle:1');
-- 5) Rekor lookup by log index (index)
EXPLAIN (ANALYZE, BUFFERS)
SELECT dsse_sha256, uuid, integrated_time
FROM proofchain.rekor_entries
WHERE log_index = 10;
-- 6) Join: entries -> envelopes by bom_digest
EXPLAIN (ANALYZE, BUFFERS)
SELECT e.entry_id, d.predicate_type, d.body_hash
FROM proofchain.sbom_entries e
JOIN proofchain.dsse_envelopes d ON d.entry_id = e.entry_id
WHERE e.bom_digest = proofchain.hex64('bom:1')
AND d.predicate_type = 'evidence.stella/v1'
ORDER BY e.purl
LIMIT 100;

View File

@@ -0,0 +1,104 @@
param(
[string]$PostgresImage = "postgres:16",
[string]$ContainerName = "stellaops-proofchain-perf",
[int]$Port = 54329,
[string]$Database = "proofchain_perf",
[string]$User = "postgres",
[string]$Password = "postgres"
)
$ErrorActionPreference = "Stop"
function Resolve-RepoRoot {
$here = Split-Path -Parent $PSCommandPath
return (Resolve-Path (Join-Path $here "../../../../..")).Path
}
$repoRoot = Resolve-RepoRoot
$perfDir = Join-Path $repoRoot "src/Attestor/__Libraries/StellaOps.Attestor.Persistence/Perf"
$migrationFile = Join-Path $repoRoot "src/Attestor/__Libraries/StellaOps.Attestor.Persistence/Migrations/20251214000001_AddProofChainSchema.sql"
$seedFile = Join-Path $perfDir "seed.sql"
$queriesFile = Join-Path $perfDir "queries.sql"
$reportFile = Join-Path $repoRoot "docs/db/reports/proofchain-schema-perf-2025-12-17.md"
Write-Host "Using repo root: $repoRoot"
Write-Host "Starting PostgreSQL container '$ContainerName' on localhost:$Port..."
try {
docker rm -f $ContainerName *> $null 2>&1
} catch {}
$null = docker run --rm -d --name $ContainerName `
-e POSTGRES_PASSWORD=$Password `
-e POSTGRES_DB=$Database `
-p ${Port}:5432 `
$PostgresImage
try {
$ready = $false
for ($i = 0; $i -lt 60; $i++) {
docker exec $ContainerName pg_isready -U $User -d $Database *> $null 2>&1
if ($LASTEXITCODE -eq 0) {
$ready = $true
break
}
Start-Sleep -Seconds 1
}
if (-not $ready) {
throw "PostgreSQL did not become ready within 60 seconds."
}
Write-Host "Applying migrations..."
$migrationSql = Get-Content -Raw -Encoding UTF8 $migrationFile
$migrationSql | docker exec -i $ContainerName psql -v ON_ERROR_STOP=1 -U $User -d $Database | Out-Host
Write-Host "Seeding deterministic dataset..."
$seedSql = Get-Content -Raw -Encoding UTF8 $seedFile
$seedSql | docker exec -i $ContainerName psql -v ON_ERROR_STOP=1 -U $User -d $Database | Out-Host
Write-Host "Running query suite..."
$queriesSql = Get-Content -Raw -Encoding UTF8 $queriesFile
$queryOutput = $queriesSql | docker exec -i $ContainerName psql -v ON_ERROR_STOP=1 -U $User -d $Database
$queryOutputText = ($queryOutput -join "`n").TrimEnd()
$headerLines = @(
'# ProofChain schema performance report (2025-12-17)',
'',
'## Environment',
('- Postgres image: `{0}`' -f $PostgresImage),
('- DB: `{0}`' -f $Database),
('- Port: `{0}`' -f $Port),
'- Host: `localhost`',
'',
'## Dataset',
'- Source: `src/Attestor/__Libraries/StellaOps.Attestor.Persistence/Perf/seed.sql`',
'- Rows:',
' - `trust_anchors`: 50',
' - `sbom_entries`: 20000',
' - `dsse_envelopes`: 60000',
' - `spines`: 20000',
' - `rekor_entries`: 2000',
'',
'## Query Output',
'',
'```text',
$queryOutputText,
'```',
''
)
$header = ($headerLines -join "`n")
$dir = Split-Path -Parent $reportFile
if (!(Test-Path $dir)) {
New-Item -ItemType Directory -Path $dir -Force | Out-Null
}
Set-Content -Path $reportFile -Value $header -Encoding UTF8
Write-Host "Wrote report: $reportFile"
}
finally {
Write-Host "Stopping container..."
docker rm -f $ContainerName *> $null 2>&1
}

View File

@@ -0,0 +1,166 @@
-- Deterministic ProofChain dataset generator (offline-friendly).
-- Designed for index/query perf validation (SPRINT_0501_0006_0001 · PROOF-DB-0011).
-- Helper: deterministic UUID from text (no extensions required).
CREATE OR REPLACE FUNCTION proofchain.uuid_from_text(input text) RETURNS uuid
LANGUAGE SQL
IMMUTABLE
STRICT
AS $$
SELECT (
substring(md5(input), 1, 8) || '-' ||
substring(md5(input), 9, 4) || '-' ||
substring(md5(input), 13, 4) || '-' ||
substring(md5(input), 17, 4) || '-' ||
substring(md5(input), 21, 12)
)::uuid;
$$;
-- Helper: deterministic 64-hex string from text.
CREATE OR REPLACE FUNCTION proofchain.hex64(input text) RETURNS text
LANGUAGE SQL
IMMUTABLE
STRICT
AS $$
SELECT md5(input) || md5(input || ':2');
$$;
-- Parameters
-- Anchors: 50
-- SBOM entries: 20_000 (200 SBOM digests * 100 entries each)
-- Envelopes: 60_000 (3 per entry)
-- Spines: 20_000 (1 per entry)
-- Rekor entries: 2_000 (every 10th entry)
-- Trust anchors
INSERT INTO proofchain.trust_anchors(
anchor_id,
purl_pattern,
allowed_keyids,
allowed_predicate_types,
policy_ref,
policy_version,
revoked_keys,
is_active,
created_at,
updated_at
)
SELECT
proofchain.uuid_from_text('anchor:' || i),
format('pkg:npm/vendor-%02s/*', i),
ARRAY[format('key-%02s', i)]::text[],
ARRAY[
'evidence.stella/v1',
'reasoning.stella/v1',
'cdx-vex.stella/v1',
'proofspine.stella/v1',
'verdict.stella/v1',
'https://stella-ops.org/predicates/sbom-linkage/v1'
]::text[],
format('policy-%02s', i),
'v2025.12',
ARRAY[]::text[],
TRUE,
TIMESTAMPTZ '2025-12-17T00:00:00Z',
TIMESTAMPTZ '2025-12-17T00:00:00Z'
FROM generate_series(1, 50) i
ON CONFLICT (anchor_id) DO NOTHING;
-- SBOM entries
INSERT INTO proofchain.sbom_entries(
entry_id,
bom_digest,
purl,
version,
artifact_digest,
trust_anchor_id,
created_at
)
SELECT
proofchain.uuid_from_text('entry:' || i),
proofchain.hex64('bom:' || (((i - 1) / 100) + 1)),
format('pkg:npm/vendor-%02s/pkg-%05s', (((i - 1) % 50) + 1), i),
format('1.0.%s', (((i - 1) % 50) + 1)),
proofchain.hex64('artifact:' || i),
proofchain.uuid_from_text('anchor:' || (((i - 1) % 50) + 1)),
TIMESTAMPTZ '2025-12-17T00:00:00Z' + ((i - 1) || ' seconds')::interval
FROM generate_series(1, 20000) i
ON CONFLICT ON CONSTRAINT uq_sbom_entry DO NOTHING;
-- DSSE envelopes (3 per entry)
INSERT INTO proofchain.dsse_envelopes(
env_id,
entry_id,
predicate_type,
signer_keyid,
body_hash,
envelope_blob_ref,
signed_at,
created_at
)
SELECT
proofchain.uuid_from_text('env:' || i || ':' || p.predicate_type),
proofchain.uuid_from_text('entry:' || i),
p.predicate_type,
format('key-%02s', (((i - 1) % 50) + 1)),
proofchain.hex64('body:' || i || ':' || p.predicate_type),
format('oci://proofchain/blobs/%s', proofchain.hex64('body:' || i || ':' || p.predicate_type)),
TIMESTAMPTZ '2025-12-17T00:00:00Z' + ((i - 1) || ' seconds')::interval,
TIMESTAMPTZ '2025-12-17T00:00:00Z' + ((i - 1) || ' seconds')::interval
FROM generate_series(1, 20000) i
CROSS JOIN (
VALUES
('evidence.stella/v1'),
('reasoning.stella/v1'),
('cdx-vex.stella/v1')
) AS p(predicate_type)
ON CONFLICT ON CONSTRAINT uq_dsse_envelope DO NOTHING;
-- Spines (1 per entry)
INSERT INTO proofchain.spines(
entry_id,
bundle_id,
evidence_ids,
reasoning_id,
vex_id,
anchor_id,
policy_version,
created_at
)
SELECT
proofchain.uuid_from_text('entry:' || i),
proofchain.hex64('bundle:' || i),
ARRAY[
'sha256:' || proofchain.hex64('evidence:' || i || ':1'),
'sha256:' || proofchain.hex64('evidence:' || i || ':2'),
'sha256:' || proofchain.hex64('evidence:' || i || ':3')
]::text[],
proofchain.hex64('reasoning:' || i),
proofchain.hex64('vex:' || i),
proofchain.uuid_from_text('anchor:' || (((i - 1) % 50) + 1)),
'v2025.12',
TIMESTAMPTZ '2025-12-17T00:00:00Z' + ((i - 1) || ' seconds')::interval
FROM generate_series(1, 20000) i
ON CONFLICT ON CONSTRAINT uq_spine_bundle DO NOTHING;
-- Rekor entries (every 10th entry, points at the evidence envelope)
INSERT INTO proofchain.rekor_entries(
dsse_sha256,
log_index,
log_id,
uuid,
integrated_time,
inclusion_proof,
env_id
)
SELECT
proofchain.hex64('rekor:' || i),
i,
'test-log',
format('uuid-%s', i),
1734393600 + i,
'{"hashes":[],"treeSize":1,"rootHash":"00"}'::jsonb,
proofchain.uuid_from_text('env:' || i || ':evidence.stella/v1')
FROM generate_series(1, 20000, 10) i
ON CONFLICT (dsse_sha256) DO NOTHING;

View File

@@ -1,6 +1,7 @@
using System.Text.RegularExpressions;
using Microsoft.Extensions.Logging;
using StellaOps.Attestor.Persistence.Entities;
using StellaOps.Attestor.Persistence.Repositories;
namespace StellaOps.Attestor.Persistence.Services;
@@ -75,7 +76,7 @@ public sealed class TrustAnchorMatcher : ITrustAnchorMatcher
{
ArgumentException.ThrowIfNullOrEmpty(purl);
var anchors = await _repository.GetActiveAnchorsAsync(cancellationToken);
var anchors = await _repository.GetActiveTrustAnchorsAsync(cancellationToken);
TrustAnchorMatchResult? bestMatch = null;
@@ -284,14 +285,3 @@ public sealed class TrustAnchorMatcher : ITrustAnchorMatcher
return true;
}
}
/// <summary>
/// Repository interface extension for trust anchor queries.
/// </summary>
public interface IProofChainRepository
{
/// <summary>
/// Gets all active trust anchors.
/// </summary>
Task<IReadOnlyList<TrustAnchorEntity>> GetActiveAnchorsAsync(CancellationToken cancellationToken = default);
}

View File

@@ -20,4 +20,8 @@
</None>
</ItemGroup>
<ItemGroup>
<Compile Remove="Tests\\**\\*.cs" />
</ItemGroup>
</Project>

View File

@@ -1,223 +0,0 @@
using StellaOps.Attestor.Persistence.Entities;
using StellaOps.Attestor.Persistence.Services;
using Microsoft.Extensions.Logging.Abstractions;
using Moq;
using Xunit;
namespace StellaOps.Attestor.Persistence.Tests;
/// <summary>
/// Integration tests for proof chain database operations.
/// SPRINT_0501_0006_0001 - Task #10
/// </summary>
public sealed class ProofChainRepositoryIntegrationTests
{
private readonly Mock<IProofChainRepository> _repositoryMock;
private readonly TrustAnchorMatcher _matcher;
public ProofChainRepositoryIntegrationTests()
{
_repositoryMock = new Mock<IProofChainRepository>();
_matcher = new TrustAnchorMatcher(
_repositoryMock.Object,
NullLogger<TrustAnchorMatcher>.Instance);
}
[Fact]
public async Task FindMatchAsync_ExactPattern_MatchesCorrectly()
{
// Arrange
var anchor = CreateAnchor("pkg:npm/lodash@4.17.21", ["key-1"]);
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act
var result = await _matcher.FindMatchAsync("pkg:npm/lodash@4.17.21");
// Assert
Assert.NotNull(result);
Assert.Equal(anchor.AnchorId, result.Anchor.AnchorId);
}
[Fact]
public async Task FindMatchAsync_WildcardPattern_MatchesPackages()
{
// Arrange
var anchor = CreateAnchor("pkg:npm/*", ["key-1"]);
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act
var result = await _matcher.FindMatchAsync("pkg:npm/lodash@4.17.21");
// Assert
Assert.NotNull(result);
Assert.Equal("pkg:npm/*", result.MatchedPattern);
}
[Fact]
public async Task FindMatchAsync_DoubleWildcard_MatchesNestedPaths()
{
// Arrange
var anchor = CreateAnchor("pkg:npm/@scope/**", ["key-1"]);
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act
var result = await _matcher.FindMatchAsync("pkg:npm/@scope/sub/package@1.0.0");
// Assert
Assert.NotNull(result);
}
[Fact]
public async Task FindMatchAsync_MultipleMatches_ReturnsMoreSpecific()
{
// Arrange
var genericAnchor = CreateAnchor("pkg:npm/*", ["key-generic"], "generic");
var specificAnchor = CreateAnchor("pkg:npm/lodash@*", ["key-specific"], "specific");
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([genericAnchor, specificAnchor]);
// Act
var result = await _matcher.FindMatchAsync("pkg:npm/lodash@4.17.21");
// Assert
Assert.NotNull(result);
Assert.Equal("specific", result.Anchor.PolicyRef);
}
[Fact]
public async Task FindMatchAsync_NoMatch_ReturnsNull()
{
// Arrange
var anchor = CreateAnchor("pkg:npm/*", ["key-1"]);
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act
var result = await _matcher.FindMatchAsync("pkg:pypi/requests@2.28.0");
// Assert
Assert.Null(result);
}
[Fact]
public async Task IsKeyAllowedAsync_AllowedKey_ReturnsTrue()
{
// Arrange
var anchor = CreateAnchor("pkg:npm/*", ["key-1", "key-2"]);
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act
var allowed = await _matcher.IsKeyAllowedAsync("pkg:npm/lodash@4.17.21", "key-1");
// Assert
Assert.True(allowed);
}
[Fact]
public async Task IsKeyAllowedAsync_DisallowedKey_ReturnsFalse()
{
// Arrange
var anchor = CreateAnchor("pkg:npm/*", ["key-1"]);
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act
var allowed = await _matcher.IsKeyAllowedAsync("pkg:npm/lodash@4.17.21", "key-unknown");
// Assert
Assert.False(allowed);
}
[Fact]
public async Task IsKeyAllowedAsync_RevokedKey_ReturnsFalse()
{
// Arrange
var anchor = CreateAnchor("pkg:npm/*", ["key-1"], revokedKeys: ["key-1"]);
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act
var allowed = await _matcher.IsKeyAllowedAsync("pkg:npm/lodash@4.17.21", "key-1");
// Assert
Assert.False(allowed); // Key is revoked even if in allowed list
}
[Fact]
public async Task IsPredicateAllowedAsync_NoRestrictions_AllowsAll()
{
// Arrange
var anchor = CreateAnchor("pkg:npm/*", ["key-1"]);
anchor.AllowedPredicateTypes = null;
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act
var allowed = await _matcher.IsPredicateAllowedAsync(
"pkg:npm/lodash@4.17.21",
"https://in-toto.io/attestation/vulns/v0.1");
// Assert
Assert.True(allowed);
}
[Fact]
public async Task IsPredicateAllowedAsync_WithRestrictions_EnforcesAllowlist()
{
// Arrange
var anchor = CreateAnchor("pkg:npm/*", ["key-1"]);
anchor.AllowedPredicateTypes = ["evidence.stella/v1", "sbom.stella/v1"];
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act & Assert
Assert.True(await _matcher.IsPredicateAllowedAsync(
"pkg:npm/lodash@4.17.21", "evidence.stella/v1"));
Assert.False(await _matcher.IsPredicateAllowedAsync(
"pkg:npm/lodash@4.17.21", "random.predicate/v1"));
}
[Theory]
[InlineData("pkg:npm/*", "pkg:npm/lodash@4.17.21", true)]
[InlineData("pkg:npm/lodash@*", "pkg:npm/lodash@4.17.21", true)]
[InlineData("pkg:npm/lodash@4.17.*", "pkg:npm/lodash@4.17.21", true)]
[InlineData("pkg:npm/lodash@4.17.21", "pkg:npm/lodash@4.17.21", true)]
[InlineData("pkg:npm/lodash@4.17.21", "pkg:npm/lodash@4.17.22", false)]
[InlineData("pkg:pypi/*", "pkg:npm/lodash@4.17.21", false)]
[InlineData("pkg:npm/@scope/*", "pkg:npm/@scope/package@1.0.0", true)]
[InlineData("pkg:npm/@scope/*", "pkg:npm/@other/package@1.0.0", false)]
public async Task FindMatchAsync_PatternVariations_MatchCorrectly(
string pattern, string purl, bool shouldMatch)
{
// Arrange
var anchor = CreateAnchor(pattern, ["key-1"]);
_repositoryMock.Setup(r => r.GetActiveAnchorsAsync(It.IsAny<CancellationToken>()))
.ReturnsAsync([anchor]);
// Act
var result = await _matcher.FindMatchAsync(purl);
// Assert
Assert.Equal(shouldMatch, result != null);
}
private static TrustAnchorEntity CreateAnchor(
string pattern,
string[] allowedKeys,
string? policyRef = null,
string[]? revokedKeys = null)
{
return new TrustAnchorEntity
{
AnchorId = Guid.NewGuid(),
PurlPattern = pattern,
AllowedKeyIds = allowedKeys,
PolicyRef = policyRef,
RevokedKeys = revokedKeys ?? [],
};
}
}