stella-ops.org/git.stella-ops.org
1
0
Fork 0
Code Issues Pull Requests Actions 2 Releases Wiki Activity

UI work to fill SBOM sourcing management gap. UI planning remaining functionality exposure. Work on CI/Tests stabilization

Browse Source 
Introduces CGS determinism test runs to CI workflows for Windows, macOS, Linux, Alpine, and Debian, fulfilling CGS-008 cross-platform requirements. Updates local-ci scripts to support new smoke steps, test timeouts, progress intervals, and project slicing for improved test isolation and diagnostics.
This commit is contained in:
master
2025-12-29 19:12:38 +02:00
parent 41552d26ec
commit a4badc275e
286 changed files with 50918 additions and 992 deletions
Download Patch File Download Diff File Expand all files Collapse all files

17
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Migrations/001_initial_schema.sql
View File

@@ -2,8 +2,6 @@
-- Consolidated from migrations 001-017 (pre_1.0 archived)
-- Creates the complete vuln and concelier schemas for vulnerability advisory management
BEGIN;
-- ============================================================================
-- SECTION 1: Schema and Extension Creation
-- ============================================================================
@@ -44,6 +42,14 @@ BEGIN
END;
$$ LANGUAGE plpgsql;
CREATE OR REPLACE FUNCTION vuln.sync_advisory_provenance_ingested_at()
RETURNS TRIGGER AS $$
BEGIN
NEW.provenance_ingested_at = NULLIF(NEW.provenance->>'ingested_at', '')::TIMESTAMPTZ;
RETURN NEW;
END;
$$ LANGUAGE plpgsql;
-- ============================================================================
-- SECTION 3: Core vuln Tables
-- ============================================================================
@@ -118,7 +124,7 @@ CREATE TABLE IF NOT EXISTS vuln.advisories (
-- Generated columns for provenance
provenance_source_key TEXT GENERATED ALWAYS AS (provenance->>'source_key') STORED,
provenance_feed_id TEXT GENERATED ALWAYS AS (provenance->>'feed_id') STORED,
provenance_ingested_at TIMESTAMPTZ GENERATED ALWAYS AS ((provenance->>'ingested_at')::TIMESTAMPTZ) STORED
provenance_ingested_at TIMESTAMPTZ
);
CREATE INDEX idx_advisories_vuln_id ON vuln.advisories(primary_vuln_id);
@@ -136,6 +142,10 @@ CREATE TRIGGER trg_advisories_search_vector
BEFORE INSERT OR UPDATE ON vuln.advisories
FOR EACH ROW EXECUTE FUNCTION vuln.update_advisory_search_vector();
CREATE TRIGGER trg_advisories_provenance_ingested_at
BEFORE INSERT OR UPDATE OF provenance ON vuln.advisories
FOR EACH ROW EXECUTE FUNCTION vuln.sync_advisory_provenance_ingested_at();
CREATE TRIGGER trg_advisories_updated_at
BEFORE UPDATE ON vuln.advisories
FOR EACH ROW EXECUTE FUNCTION vuln.update_updated_at();
@@ -725,4 +735,3 @@ AS $$
WHERE cve LIKE 'CVE-' || p_year::TEXT || '-%' AND status = 'active';
$$;
COMMIT;

22
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Advisories/PostgresAdvisoryStore.cs
View File

@@ -33,9 +33,17 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
private static readonly JsonSerializerOptions JsonOptions = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
PropertyNameCaseInsensitive = true,
WriteIndented = false
};
private static readonly JsonSerializerOptions RawPayloadOptions = new()
{
PropertyNameCaseInsensitive = true,
ReadCommentHandling = JsonCommentHandling.Skip,
AllowTrailingCommas = true
};
public PostgresAdvisoryStore(
IAdvisoryRepository advisoryRepository,
IAdvisoryAliasRepository aliasRepository,
@@ -186,13 +194,23 @@ public sealed class PostgresAdvisoryStore : IPostgresAdvisoryStore, AdvisoryCont
{
try
{
var advisory = JsonSerializer.Deserialize<Advisory>(entity.RawPayload, JsonOptions);
var advisory = CanonicalJsonSerializer.Deserialize<Advisory>(entity.RawPayload);
return advisory;
}
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(ex, "Failed to deserialize raw payload for advisory {AdvisoryKey}, attempting fallback JSON parse", entity.AdvisoryKey);
}
try
{
var advisory = JsonSerializer.Deserialize<Advisory>(entity.RawPayload, RawPayloadOptions);
if (advisory is not null)
{
return advisory;
}
}
catch (JsonException ex)
catch (Exception ex) when (ex is JsonException or InvalidOperationException)
{
_logger.LogWarning(ex, "Failed to deserialize raw payload for advisory {AdvisoryKey}, reconstructing from entities", entity.AdvisoryKey);
}

2
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/AdvisoryRepository.cs
View File

@@ -370,7 +370,7 @@ public sealed class AdvisoryRepository : RepositoryBase<ConcelierDataSource>, IA
)
VALUES (
@id, @advisory_key, @primary_vuln_id, @source_id, @title, @summary, @description,
@severity, @published_at, @modified_at, @withdrawn_at, @provenance::jsonb, @raw_Payload::jsonb
@severity, @published_at, @modified_at, @withdrawn_at, @provenance::jsonb, @raw_payload::jsonb
)
ON CONFLICT (advisory_key) DO UPDATE SET
primary_vuln_id = EXCLUDED.primary_vuln_id,

2
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/DocumentRepository.cs
View File

@@ -58,7 +58,7 @@ INSERT INTO concelier.source_documents (
headers_json, metadata_json, etag, last_modified, payload, created_at, updated_at, expires_at)
VALUES (
@Id, @SourceId, @SourceName, @Uri, @Sha256, @Status, @ContentType,
@HeadersJson, @MetadataJson, @Etag, @LastModified, @Payload, @CreatedAt, @UpdatedAt, @ExpiresAt)
@HeadersJson::jsonb, @MetadataJson::jsonb, @Etag, @LastModified, @Payload, @CreatedAt, @UpdatedAt, @ExpiresAt)
ON CONFLICT (source_name, uri) DO UPDATE SET
sha256 = EXCLUDED.sha256,
status = EXCLUDED.status,

58
src/Concelier/__Libraries/StellaOps.Concelier.Persistence/Postgres/Repositories/PostgresDtoStore.cs
View File

@@ -24,14 +24,22 @@ internal sealed class PostgresDtoStore : IDtoStore, Contracts.IStorageDtoStore
{
const string sql = """
INSERT INTO concelier.dtos (id, document_id, source_name, format, payload_json, schema_version, created_at, validated_at)
VALUES (@Id, @DocumentId, @SourceName, @Format, @PayloadJson, @SchemaVersion, @CreatedAt, @ValidatedAt)
VALUES (@Id, @DocumentId, @SourceName, @Format, @PayloadJson::jsonb, @SchemaVersion, @CreatedAt, @ValidatedAt)
ON CONFLICT (document_id) DO UPDATE
SET payload_json = EXCLUDED.payload_json,
schema_version = EXCLUDED.schema_version,
source_name = EXCLUDED.source_name,
format = EXCLUDED.format,
validated_at = EXCLUDED.validated_at
RETURNING id, document_id, source_name, format, payload_json, schema_version, created_at, validated_at;
RETURNING
id AS "Id",
document_id AS "DocumentId",
source_name AS "SourceName",
format AS "Format",
payload_json::text AS "PayloadJson",
schema_version AS "SchemaVersion",
created_at AS "CreatedAt",
validated_at AS "ValidatedAt";
""";
var payloadJson = record.Payload.ToJson();
@@ -55,7 +63,15 @@ internal sealed class PostgresDtoStore : IDtoStore, Contracts.IStorageDtoStore
public async Task<DtoRecord?> FindByDocumentIdAsync(Guid documentId, CancellationToken cancellationToken)
{
const string sql = """
SELECT id, document_id, source_name, format, payload_json, schema_version, created_at, validated_at
SELECT
id AS "Id",
document_id AS "DocumentId",
source_name AS "SourceName",
format AS "Format",
payload_json::text AS "PayloadJson",
schema_version AS "SchemaVersion",
created_at AS "CreatedAt",
validated_at AS "ValidatedAt"
FROM concelier.dtos
WHERE document_id = @DocumentId
LIMIT 1;
@@ -69,7 +85,15 @@ internal sealed class PostgresDtoStore : IDtoStore, Contracts.IStorageDtoStore
public async Task<IReadOnlyList<DtoRecord>> GetBySourceAsync(string sourceName, int limit, CancellationToken cancellationToken)
{
const string sql = """
SELECT id, document_id, source_name, format, payload_json, schema_version, created_at, validated_at
SELECT
id AS "Id",
document_id AS "DocumentId",
source_name AS "SourceName",
format AS "Format",
payload_json::text AS "PayloadJson",
schema_version AS "SchemaVersion",
created_at AS "CreatedAt",
validated_at AS "ValidatedAt"
FROM concelier.dtos
WHERE source_name = @SourceName
ORDER BY created_at DESC
@@ -84,15 +108,17 @@ internal sealed class PostgresDtoStore : IDtoStore, Contracts.IStorageDtoStore
private DtoRecord ToRecord(DtoRow row)
{
var payload = StellaOps.Concelier.Documents.DocumentObject.Parse(row.PayloadJson);
var createdAtUtc = DateTime.SpecifyKind(row.CreatedAt, DateTimeKind.Utc);
var validatedAtUtc = DateTime.SpecifyKind(row.ValidatedAt, DateTimeKind.Utc);
return new DtoRecord(
row.Id,
row.DocumentId,
row.SourceName,
row.Format,
payload,
row.CreatedAt,
new DateTimeOffset(createdAtUtc),
row.SchemaVersion,
row.ValidatedAt);
new DateTimeOffset(validatedAtUtc));
}
async Task<Contracts.StorageDto> Contracts.IStorageDtoStore.UpsertAsync(Contracts.StorageDto record, CancellationToken cancellationToken)
@@ -106,13 +132,15 @@ internal sealed class PostgresDtoStore : IDtoStore, Contracts.IStorageDtoStore
.Select(dto => dto.ToStorageDto())
.ToArray();
private sealed record DtoRow(
Guid Id,
Guid DocumentId,
string SourceName,
string Format,
string PayloadJson,
string SchemaVersion,
DateTimeOffset CreatedAt,
DateTimeOffset ValidatedAt);
private sealed class DtoRow
{
public Guid Id { get; init; }
public Guid DocumentId { get; init; }
public string SourceName { get; init; } = string.Empty;
public string Format { get; init; } = string.Empty;
public string PayloadJson { get; init; } = string.Empty;
public string SchemaVersion { get; init; } = string.Empty;
public DateTime CreatedAt { get; init; }
public DateTime ValidatedAt { get; init; }
}
}