53508ceccb
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
- Implemented RecordingLogger and RecordingLoggerFactory for capturing log entries in tests. - Added unit tests for InMemoryChannel, covering constructor behavior, property assignments, channel communication, and disposal. - Created InMemoryTransportOptionsTests to validate default values and customizable options for InMemory transport. - Developed RabbitMqFrameProtocolTests to ensure correct parsing and property creation for RabbitMQ frames. - Added RabbitMqTransportOptionsTests to verify default settings and customization options for RabbitMQ transport. - Updated project files for testing libraries and dependencies.
280 lines
8.8 KiB
JSON
280 lines
8.8 KiB
JSON
{
|
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
|
"$id": "https://stellaops.org/schemas/object-storage.schema.json",
|
|
"title": "StellaOps Object Storage Contract",
|
|
"description": "Contract for S3-compatible object storage used by Concelier for large raw payloads. Defines the interface for deterministic pointers, provenance metadata, and migration from GridFS.",
|
|
"version": "1.0.0",
|
|
"definitions": {
|
|
"ObjectPointer": {
|
|
"type": "object",
|
|
"description": "Deterministic pointer to an object in storage",
|
|
"required": ["bucket", "key", "sha256", "size"],
|
|
"properties": {
|
|
"bucket": {
|
|
"type": "string",
|
|
"description": "S3 bucket name (tenant-prefixed)",
|
|
"pattern": "^[a-z0-9][a-z0-9.-]{1,61}[a-z0-9]$"
|
|
},
|
|
"key": {
|
|
"type": "string",
|
|
"description": "Object key (deterministic, content-addressed)",
|
|
"pattern": "^[a-zA-Z0-9/._-]+$"
|
|
},
|
|
"sha256": {
|
|
"type": "string",
|
|
"description": "SHA-256 hash of object content (hex encoded)",
|
|
"pattern": "^[a-f0-9]{64}$"
|
|
},
|
|
"size": {
|
|
"type": "integer",
|
|
"description": "Object size in bytes",
|
|
"minimum": 0
|
|
},
|
|
"contentType": {
|
|
"type": "string",
|
|
"description": "MIME type of the object",
|
|
"default": "application/octet-stream"
|
|
},
|
|
"encoding": {
|
|
"type": "string",
|
|
"description": "Content encoding if compressed",
|
|
"enum": ["identity", "gzip", "zstd"]
|
|
}
|
|
}
|
|
},
|
|
"ProvenanceMetadata": {
|
|
"type": "object",
|
|
"description": "Provenance metadata preserved from original ingestion",
|
|
"required": ["sourceId", "ingestedAt", "tenantId"],
|
|
"properties": {
|
|
"sourceId": {
|
|
"type": "string",
|
|
"description": "Identifier of the original data source",
|
|
"format": "uri"
|
|
},
|
|
"ingestedAt": {
|
|
"type": "string",
|
|
"description": "UTC ISO-8601 timestamp of original ingestion",
|
|
"format": "date-time"
|
|
},
|
|
"tenantId": {
|
|
"type": "string",
|
|
"description": "Tenant identifier for multi-tenant isolation",
|
|
"pattern": "^[a-zA-Z0-9_-]+$"
|
|
},
|
|
"originalFormat": {
|
|
"type": "string",
|
|
"description": "Original format before normalization",
|
|
"enum": ["json", "xml", "csv", "ndjson", "yaml"]
|
|
},
|
|
"originalSize": {
|
|
"type": "integer",
|
|
"description": "Original size before any transformation",
|
|
"minimum": 0
|
|
},
|
|
"transformations": {
|
|
"type": "array",
|
|
"description": "List of transformations applied",
|
|
"items": {
|
|
"type": "object",
|
|
"properties": {
|
|
"type": {
|
|
"type": "string",
|
|
"enum": ["compression", "normalization", "redaction", "migration"]
|
|
},
|
|
"timestamp": {
|
|
"type": "string",
|
|
"format": "date-time"
|
|
},
|
|
"agent": {
|
|
"type": "string",
|
|
"description": "Agent/service that performed transformation"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"gridFsLegacyId": {
|
|
"type": "string",
|
|
"description": "Original GridFS ObjectId for migration tracking",
|
|
"pattern": "^[a-f0-9]{24}$"
|
|
}
|
|
}
|
|
},
|
|
"StorageConfig": {
|
|
"type": "object",
|
|
"description": "Configuration for S3-compatible object storage endpoint",
|
|
"required": ["endpoint", "region"],
|
|
"properties": {
|
|
"endpoint": {
|
|
"type": "string",
|
|
"description": "S3-compatible endpoint URL (MinIO, AWS S3, etc.)",
|
|
"format": "uri"
|
|
},
|
|
"region": {
|
|
"type": "string",
|
|
"description": "Storage region (use 'us-east-1' for MinIO)",
|
|
"default": "us-east-1"
|
|
},
|
|
"usePathStyle": {
|
|
"type": "boolean",
|
|
"description": "Use path-style addressing (required for MinIO)",
|
|
"default": true
|
|
},
|
|
"bucketPrefix": {
|
|
"type": "string",
|
|
"description": "Prefix for tenant bucket names",
|
|
"default": "stellaops-"
|
|
},
|
|
"maxObjectSize": {
|
|
"type": "integer",
|
|
"description": "Maximum object size in bytes (default 5GB)",
|
|
"default": 5368709120
|
|
},
|
|
"compressionThreshold": {
|
|
"type": "integer",
|
|
"description": "Objects larger than this (bytes) will be compressed",
|
|
"default": 1048576
|
|
}
|
|
}
|
|
},
|
|
"MigrationRecord": {
|
|
"type": "object",
|
|
"description": "Record of a migration from GridFS to S3",
|
|
"required": ["gridFsId", "pointer", "migratedAt", "status"],
|
|
"properties": {
|
|
"gridFsId": {
|
|
"type": "string",
|
|
"description": "Original GridFS ObjectId",
|
|
"pattern": "^[a-f0-9]{24}$"
|
|
},
|
|
"pointer": {
|
|
"$ref": "#/definitions/ObjectPointer"
|
|
},
|
|
"migratedAt": {
|
|
"type": "string",
|
|
"format": "date-time"
|
|
},
|
|
"status": {
|
|
"type": "string",
|
|
"enum": ["pending", "migrated", "verified", "failed", "tombstoned"]
|
|
},
|
|
"verifiedAt": {
|
|
"type": "string",
|
|
"format": "date-time",
|
|
"description": "Timestamp when content hash was verified post-migration"
|
|
},
|
|
"rollbackAvailable": {
|
|
"type": "boolean",
|
|
"description": "Whether GridFS tombstone still exists for rollback",
|
|
"default": true
|
|
}
|
|
}
|
|
},
|
|
"PayloadReference": {
|
|
"type": "object",
|
|
"description": "Reference to a large payload stored in object storage (used in advisory_observations)",
|
|
"required": ["type", "pointer", "provenance"],
|
|
"properties": {
|
|
"type": {
|
|
"type": "string",
|
|
"const": "object-storage-ref",
|
|
"description": "Discriminator for payload type"
|
|
},
|
|
"pointer": {
|
|
"$ref": "#/definitions/ObjectPointer"
|
|
},
|
|
"provenance": {
|
|
"$ref": "#/definitions/ProvenanceMetadata"
|
|
},
|
|
"inline": {
|
|
"type": "boolean",
|
|
"description": "If true, payload is small enough to be inline (not in object storage)",
|
|
"default": false
|
|
},
|
|
"inlineData": {
|
|
"type": "string",
|
|
"description": "Base64-encoded inline data (only if inline=true and size < threshold)",
|
|
"contentEncoding": "base64"
|
|
}
|
|
}
|
|
}
|
|
},
|
|
"type": "object",
|
|
"properties": {
|
|
"$schema": {
|
|
"type": "string"
|
|
},
|
|
"config": {
|
|
"$ref": "#/definitions/StorageConfig"
|
|
},
|
|
"pointers": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/PayloadReference"
|
|
}
|
|
},
|
|
"migrations": {
|
|
"type": "array",
|
|
"items": {
|
|
"$ref": "#/definitions/MigrationRecord"
|
|
}
|
|
}
|
|
},
|
|
"examples": [
|
|
{
|
|
"config": {
|
|
"endpoint": "http://minio.stellaops.local:9000",
|
|
"region": "us-east-1",
|
|
"usePathStyle": true,
|
|
"bucketPrefix": "stellaops-",
|
|
"compressionThreshold": 1048576
|
|
},
|
|
"pointers": [
|
|
{
|
|
"type": "object-storage-ref",
|
|
"pointer": {
|
|
"bucket": "stellaops-tenant-abc123",
|
|
"key": "advisories/raw/2025/12/05/sha256-a1b2c3d4.json.zst",
|
|
"sha256": "a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2",
|
|
"size": 524288,
|
|
"contentType": "application/json",
|
|
"encoding": "zstd"
|
|
},
|
|
"provenance": {
|
|
"sourceId": "https://nvd.nist.gov/feeds/json/cve/1.1",
|
|
"ingestedAt": "2025-12-05T10:30:00Z",
|
|
"tenantId": "tenant-abc123",
|
|
"originalFormat": "json",
|
|
"originalSize": 2097152,
|
|
"transformations": [
|
|
{
|
|
"type": "compression",
|
|
"timestamp": "2025-12-05T10:30:01Z",
|
|
"agent": "concelier-ingest-v1.2.0"
|
|
}
|
|
]
|
|
},
|
|
"inline": false
|
|
}
|
|
],
|
|
"migrations": [
|
|
{
|
|
"gridFsId": "507f1f77bcf86cd799439011",
|
|
"pointer": {
|
|
"bucket": "stellaops-tenant-abc123",
|
|
"key": "advisories/migrated/507f1f77bcf86cd799439011.json",
|
|
"sha256": "b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3d4e5f6a1b2c3",
|
|
"size": 102400,
|
|
"contentType": "application/json",
|
|
"encoding": "identity"
|
|
},
|
|
"migratedAt": "2025-12-05T11:00:00Z",
|
|
"status": "verified",
|
|
"verifiedAt": "2025-12-05T11:00:05Z",
|
|
"rollbackAvailable": true
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|