CD/CD consolidation
This commit is contained in:
StellaOps Bot
16
devops/database/mongo/indices/README.md
Normal file
16
devops/database/mongo/indices/README.md
Normal file
@@ -0,0 +1,16 @@
|
||||
# MongoDB Provenance Indexes
|
||||
|
||||
Indexes supporting Sprint 401 reachability/provenance queries.
|
||||
|
||||
## Available indexes
|
||||
- `events_by_subject_kind_provenance`: `(subject.digest.sha256, kind, provenance.dsse.rekor.logIndex)` for subject/kind lookups with Rekor presence.
|
||||
- `events_unproven_by_kind`: `(kind, trust.verified, provenance.dsse.rekor.logIndex)` to find unverified or missing-Rekor events per kind.
|
||||
- `events_by_rekor_logindex`: `(provenance.dsse.rekor.logIndex)` to audit Rekor alignment.
|
||||
|
||||
## Apply
|
||||
```js
|
||||
// From mongo shell (connected to provenance database)
|
||||
load('ops/mongo/indices/events_provenance_indices.js');
|
||||
```
|
||||
|
||||
Indexes are idempotent; rerunning is safe.
|
||||
89
devops/database/mongo/indices/events_provenance_indices.js
Normal file
89
devops/database/mongo/indices/events_provenance_indices.js
Normal file
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* MongoDB indexes for DSSE provenance queries on the events collection.
|
||||
* Run with: mongosh stellaops_db < events_provenance_indices.js
|
||||
*
|
||||
* These indexes support:
|
||||
* - Proven VEX/SBOM/SCAN lookup by subject digest
|
||||
* - Compliance gap queries (unverified events)
|
||||
* - Rekor log index lookups
|
||||
* - Backfill service queries
|
||||
*
|
||||
* Created: 2025-11-27 (PROV-INDEX-401-030)
|
||||
* C# equivalent: src/StellaOps.Events.Mongo/MongoIndexes.cs
|
||||
*/
|
||||
|
||||
// Switch to the target database (override via --eval "var dbName='custom'" if needed)
|
||||
const targetDb = typeof dbName !== 'undefined' ? dbName : 'stellaops';
|
||||
db = db.getSiblingDB(targetDb);
|
||||
|
||||
print(`Creating provenance indexes on ${targetDb}.events...`);
|
||||
|
||||
// Index 1: Lookup proven events by subject digest + kind
|
||||
db.events.createIndex(
|
||||
{
|
||||
"subject.digest.sha256": 1,
|
||||
"kind": 1,
|
||||
"provenance.dsse.rekor.logIndex": 1
|
||||
},
|
||||
{
|
||||
name: "events_by_subject_kind_provenance",
|
||||
background: true
|
||||
}
|
||||
);
|
||||
print(" - events_by_subject_kind_provenance");
|
||||
|
||||
// Index 2: Find unproven evidence by kind (compliance gap queries)
|
||||
db.events.createIndex(
|
||||
{
|
||||
"kind": 1,
|
||||
"trust.verified": 1,
|
||||
"provenance.dsse.rekor.logIndex": 1
|
||||
},
|
||||
{
|
||||
name: "events_unproven_by_kind",
|
||||
background: true
|
||||
}
|
||||
);
|
||||
print(" - events_unproven_by_kind");
|
||||
|
||||
// Index 3: Direct Rekor log index lookup
|
||||
db.events.createIndex(
|
||||
{
|
||||
"provenance.dsse.rekor.logIndex": 1
|
||||
},
|
||||
{
|
||||
name: "events_by_rekor_logindex",
|
||||
background: true
|
||||
}
|
||||
);
|
||||
print(" - events_by_rekor_logindex");
|
||||
|
||||
// Index 4: Envelope digest lookup (for backfill deduplication)
|
||||
db.events.createIndex(
|
||||
{
|
||||
"provenance.dsse.envelopeDigest": 1
|
||||
},
|
||||
{
|
||||
name: "events_by_envelope_digest",
|
||||
background: true,
|
||||
sparse: true
|
||||
}
|
||||
);
|
||||
print(" - events_by_envelope_digest");
|
||||
|
||||
// Index 5: Timestamp + kind for compliance reporting time ranges
|
||||
db.events.createIndex(
|
||||
{
|
||||
"ts": -1,
|
||||
"kind": 1,
|
||||
"trust.verified": 1
|
||||
},
|
||||
{
|
||||
name: "events_by_ts_kind_verified",
|
||||
background: true
|
||||
}
|
||||
);
|
||||
print(" - events_by_ts_kind_verified");
|
||||
|
||||
print("\nProvenance indexes created successfully.");
|
||||
print("Run 'db.events.getIndexes()' to verify.");
|
||||
67
devops/database/mongo/indices/reachability_store_indices.js
Normal file
67
devops/database/mongo/indices/reachability_store_indices.js
Normal file
@@ -0,0 +1,67 @@
|
||||
/**
|
||||
* MongoDB indexes for the shared reachability store collections used by Signals/Policy/Scanner.
|
||||
* Run with: mongosh stellaops_db < reachability_store_indices.js
|
||||
*
|
||||
* Collections:
|
||||
* - func_nodes: canonical function nodes keyed by graph + symbol ID and joinable by (purl, symbolDigest)
|
||||
* - call_edges: canonical call edges keyed by graph and joinable by (purl, symbolDigest)
|
||||
* - cve_func_hits: per-subject mapping of CVE -> affected/reachable functions with evidence pointers
|
||||
*
|
||||
* Created: 2025-12-13 (SIG-STORE-401-016)
|
||||
*/
|
||||
|
||||
// Switch to the target database (override via --eval "var dbName='custom'" if needed)
|
||||
const targetDb = typeof dbName !== 'undefined' ? dbName : 'stellaops';
|
||||
db = db.getSiblingDB(targetDb);
|
||||
|
||||
print(`Creating reachability store indexes on ${targetDb}...`);
|
||||
|
||||
print(`- func_nodes`);
|
||||
db.func_nodes.createIndex(
|
||||
{ "graphHash": 1, "symbolId": 1 },
|
||||
{ name: "func_nodes_by_graph_symbol", unique: true, background: true }
|
||||
);
|
||||
db.func_nodes.createIndex(
|
||||
{ "purl": 1, "symbolDigest": 1 },
|
||||
{ name: "func_nodes_by_purl_symboldigest", background: true, sparse: true }
|
||||
);
|
||||
db.func_nodes.createIndex(
|
||||
{ "codeId": 1 },
|
||||
{ name: "func_nodes_by_code_id", background: true, sparse: true }
|
||||
);
|
||||
|
||||
print(`- call_edges`);
|
||||
db.call_edges.createIndex(
|
||||
{ "graphHash": 1, "sourceId": 1, "targetId": 1, "type": 1 },
|
||||
{ name: "call_edges_by_graph_edge", unique: true, background: true }
|
||||
);
|
||||
db.call_edges.createIndex(
|
||||
{ "graphHash": 1, "sourceId": 1 },
|
||||
{ name: "call_edges_by_graph_source", background: true }
|
||||
);
|
||||
db.call_edges.createIndex(
|
||||
{ "graphHash": 1, "targetId": 1 },
|
||||
{ name: "call_edges_by_graph_target", background: true }
|
||||
);
|
||||
db.call_edges.createIndex(
|
||||
{ "purl": 1, "symbolDigest": 1 },
|
||||
{ name: "call_edges_by_purl_symboldigest", background: true, sparse: true }
|
||||
);
|
||||
|
||||
print(`- cve_func_hits`);
|
||||
db.cve_func_hits.createIndex(
|
||||
{ "subjectKey": 1, "cveId": 1 },
|
||||
{ name: "cve_func_hits_by_subject_cve", background: true }
|
||||
);
|
||||
db.cve_func_hits.createIndex(
|
||||
{ "cveId": 1, "purl": 1, "symbolDigest": 1 },
|
||||
{ name: "cve_func_hits_by_cve_purl_symboldigest", background: true, sparse: true }
|
||||
);
|
||||
db.cve_func_hits.createIndex(
|
||||
{ "graphHash": 1 },
|
||||
{ name: "cve_func_hits_by_graph", background: true, sparse: true }
|
||||
);
|
||||
|
||||
print("\nReachability store indexes created successfully.");
|
||||
print("Run db.func_nodes.getIndexes(), db.call_edges.getIndexes(), db.cve_func_hits.getIndexes() to verify.");
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
// Task Runner baseline collections and indexes
|
||||
// Mirrors docs/modules/taskrunner/migrations/pack-run-collections.md (last updated 2025-11-06)
|
||||
|
||||
function ensureCollection(name, validator) {
|
||||
const existing = db.getCollectionNames();
|
||||
if (!existing.includes(name)) {
|
||||
db.createCollection(name, { validator, validationLevel: "moderate" });
|
||||
} else if (validator) {
|
||||
db.runCommand({ collMod: name, validator, validationLevel: "moderate" });
|
||||
}
|
||||
}
|
||||
|
||||
const runValidator = {
|
||||
$jsonSchema: {
|
||||
bsonType: "object",
|
||||
required: ["planHash", "plan", "failurePolicy", "requestedAt", "createdAt", "updatedAt", "steps"],
|
||||
properties: {
|
||||
_id: { bsonType: "string" },
|
||||
planHash: { bsonType: "string" },
|
||||
plan: { bsonType: "object" },
|
||||
failurePolicy: { bsonType: "object" },
|
||||
requestedAt: { bsonType: "date" },
|
||||
createdAt: { bsonType: "date" },
|
||||
updatedAt: { bsonType: "date" },
|
||||
steps: {
|
||||
bsonType: "array",
|
||||
items: {
|
||||
bsonType: "object",
|
||||
required: ["stepId", "status", "attempts"],
|
||||
properties: {
|
||||
stepId: { bsonType: "string" },
|
||||
status: { bsonType: "string" },
|
||||
attempts: { bsonType: "int" },
|
||||
kind: { bsonType: "string" },
|
||||
enabled: { bsonType: "bool" },
|
||||
continueOnError: { bsonType: "bool" },
|
||||
maxParallel: { bsonType: ["int", "null"] },
|
||||
approvalId: { bsonType: ["string", "null"] },
|
||||
gateMessage: { bsonType: ["string", "null"] },
|
||||
lastTransitionAt: { bsonType: ["date", "null"] },
|
||||
nextAttemptAt: { bsonType: ["date", "null"] },
|
||||
statusReason: { bsonType: ["string", "null"] }
|
||||
}
|
||||
}
|
||||
},
|
||||
tenantId: { bsonType: ["string", "null"] }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const logValidator = {
|
||||
$jsonSchema: {
|
||||
bsonType: "object",
|
||||
required: ["runId", "sequence", "timestamp", "level", "eventType", "message"],
|
||||
properties: {
|
||||
runId: { bsonType: "string" },
|
||||
sequence: { bsonType: "long" },
|
||||
timestamp: { bsonType: "date" },
|
||||
level: { bsonType: "string" },
|
||||
eventType: { bsonType: "string" },
|
||||
message: { bsonType: "string" },
|
||||
stepId: { bsonType: ["string", "null"] },
|
||||
metadata: { bsonType: ["object", "null"] }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const artifactsValidator = {
|
||||
$jsonSchema: {
|
||||
bsonType: "object",
|
||||
required: ["runId", "name", "type", "status", "capturedAt"],
|
||||
properties: {
|
||||
runId: { bsonType: "string" },
|
||||
name: { bsonType: "string" },
|
||||
type: { bsonType: "string" },
|
||||
status: { bsonType: "string" },
|
||||
capturedAt: { bsonType: "date" },
|
||||
sourcePath: { bsonType: ["string", "null"] },
|
||||
storedPath: { bsonType: ["string", "null"] },
|
||||
notes: { bsonType: ["string", "null"] },
|
||||
expression: { bsonType: ["object", "null"] }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
const approvalsValidator = {
|
||||
$jsonSchema: {
|
||||
bsonType: "object",
|
||||
required: ["runId", "approvalId", "requestedAt", "status"],
|
||||
properties: {
|
||||
runId: { bsonType: "string" },
|
||||
approvalId: { bsonType: "string" },
|
||||
requiredGrants: { bsonType: "array", items: { bsonType: "string" } },
|
||||
stepIds: { bsonType: "array", items: { bsonType: "string" } },
|
||||
messages: { bsonType: "array", items: { bsonType: "string" } },
|
||||
reasonTemplate: { bsonType: ["string", "null"] },
|
||||
requestedAt: { bsonType: "date" },
|
||||
status: { bsonType: "string" },
|
||||
actorId: { bsonType: ["string", "null"] },
|
||||
completedAt: { bsonType: ["date", "null"] },
|
||||
summary: { bsonType: ["string", "null"] }
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
ensureCollection("pack_runs", runValidator);
|
||||
ensureCollection("pack_run_logs", logValidator);
|
||||
ensureCollection("pack_artifacts", artifactsValidator);
|
||||
ensureCollection("pack_run_approvals", approvalsValidator);
|
||||
|
||||
// Indexes for pack_runs
|
||||
db.pack_runs.createIndex({ updatedAt: -1 }, { name: "pack_runs_updatedAt_desc" });
|
||||
db.pack_runs.createIndex({ tenantId: 1, updatedAt: -1 }, { name: "pack_runs_tenant_updatedAt_desc", sparse: true });
|
||||
|
||||
// Indexes for pack_run_logs
|
||||
db.pack_run_logs.createIndex({ runId: 1, sequence: 1 }, { unique: true, name: "pack_run_logs_run_sequence" });
|
||||
db.pack_run_logs.createIndex({ runId: 1, timestamp: 1 }, { name: "pack_run_logs_run_timestamp" });
|
||||
|
||||
// Indexes for pack_artifacts
|
||||
db.pack_artifacts.createIndex({ runId: 1, name: 1 }, { unique: true, name: "pack_artifacts_run_name" });
|
||||
db.pack_artifacts.createIndex({ runId: 1 }, { name: "pack_artifacts_run" });
|
||||
|
||||
// Indexes for pack_run_approvals
|
||||
db.pack_run_approvals.createIndex({ runId: 1, approvalId: 1 }, { unique: true, name: "pack_run_approvals_run_approval" });
|
||||
db.pack_run_approvals.createIndex({ runId: 1, status: 1 }, { name: "pack_run_approvals_run_status" });
|
||||
Reference in New Issue
Block a user