docs consoliation work
This commit is contained in:
StellaOps Bot
471
docs/modules/vex-lens/operations/deployment.md
Normal file
471
docs/modules/vex-lens/operations/deployment.md
Normal file
@@ -0,0 +1,471 @@
|
||||
# VexLens Deployment Runbook
|
||||
|
||||
> Operational runbook for deploying and configuring VexLens consensus engine.
|
||||
|
||||
---
|
||||
|
||||
## 1) Prerequisites
|
||||
|
||||
### 1.1 Infrastructure Requirements
|
||||
|
||||
| Component | Requirement | Notes |
|
||||
|-----------|-------------|-------|
|
||||
| Runtime | .NET 10.0+ | LTS recommended |
|
||||
| Database | PostgreSQL 15.0+ | For projections and issuer directory |
|
||||
| Cache | Redis 7.0+ (optional) | For caching consensus results |
|
||||
| Memory | 512MB minimum | 2GB recommended for production |
|
||||
| CPU | 2 cores minimum | 4 cores for high throughput |
|
||||
|
||||
### 1.2 Dependencies
|
||||
|
||||
- **Excititor**: VEX document ingestion service
|
||||
- **Authority**: OIDC token validation
|
||||
- **Policy Engine**: (optional) For VEX-aware policy evaluation
|
||||
|
||||
---
|
||||
|
||||
## 2) Configuration
|
||||
|
||||
### 2.1 Environment Variables
|
||||
|
||||
```bash
|
||||
# Core Settings
|
||||
VEXLENS_CONSENSUS_DEFAULT_MODE=WeightedVote
|
||||
VEXLENS_CONSENSUS_MINIMUM_CONFIDENCE=0.1
|
||||
VEXLENS_CONSENSUS_CONFLICT_THRESHOLD=0.3
|
||||
|
||||
# Trust Settings
|
||||
VEXLENS_TRUST_FRESHNESS_HALFLIFE_DAYS=90
|
||||
VEXLENS_TRUST_MINIMUM_FRESHNESS=0.3
|
||||
VEXLENS_TRUST_ALLOW_UNSIGNED=true
|
||||
VEXLENS_TRUST_UNSIGNED_PENALTY=0.3
|
||||
VEXLENS_TRUST_ALLOW_UNKNOWN_ISSUERS=true
|
||||
VEXLENS_TRUST_UNKNOWN_ISSUER_PENALTY=0.5
|
||||
|
||||
# Storage
|
||||
VEXLENS_STORAGE_POSTGRESQL_CONNECTION_STRING=Host=localhost;Database=vexlens;Username=stellaops;Password=secret
|
||||
VEXLENS_STORAGE_PROJECTION_RETENTION_DAYS=365
|
||||
VEXLENS_STORAGE_EVENT_RETENTION_DAYS=90
|
||||
|
||||
# Issuer Directory
|
||||
VEXLENS_ISSUER_DIRECTORY_SOURCE=postgresql
|
||||
VEXLENS_ISSUER_DIRECTORY_REFRESH_INTERVAL_MINUTES=60
|
||||
|
||||
# Observability
|
||||
VEXLENS_OTEL_EXPORTER_ENDPOINT=http://otel-collector:4317
|
||||
VEXLENS_OTEL_SERVICE_NAME=vexlens
|
||||
```
|
||||
|
||||
### 2.2 Configuration File (vexlens.yaml)
|
||||
|
||||
```yaml
|
||||
vexlens:
|
||||
consensus:
|
||||
defaultMode: WeightedVote
|
||||
minimumConfidence: 0.1
|
||||
conflictThreshold: 0.3
|
||||
requireJustificationForNotAffected: false
|
||||
|
||||
trust:
|
||||
freshnessHalfLifeDays: 90
|
||||
minimumFreshness: 0.3
|
||||
allowUnsigned: true
|
||||
unsignedPenalty: 0.3
|
||||
allowUnknownIssuers: true
|
||||
unknownIssuerPenalty: 0.5
|
||||
factorWeights:
|
||||
IssuerBase: 0.25
|
||||
SignatureStatus: 0.15
|
||||
Freshness: 0.15
|
||||
IssuerCategory: 0.10
|
||||
IssuerTier: 0.10
|
||||
StatusQuality: 0.10
|
||||
TransparencyLog: 0.05
|
||||
SourceMatch: 0.05
|
||||
ProductAuthority: 0.05
|
||||
|
||||
storage:
|
||||
postgresql:
|
||||
connectionString: Host=localhost;Database=vexlens;Username=stellaops;Password=secret
|
||||
projectionsTable: consensus_projections
|
||||
issuersTable: issuers
|
||||
projectionRetentionDays: 365
|
||||
eventRetentionDays: 90
|
||||
|
||||
issuerDirectory:
|
||||
source: postgresql
|
||||
refreshIntervalMinutes: 60
|
||||
seedFile: /etc/vexlens/issuers.json
|
||||
|
||||
observability:
|
||||
metrics:
|
||||
enabled: true
|
||||
exporterEndpoint: http://otel-collector:4317
|
||||
tracing:
|
||||
enabled: true
|
||||
samplingRatio: 0.1
|
||||
logging:
|
||||
level: Information
|
||||
format: json
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3) Deployment Steps
|
||||
|
||||
### 3.1 Docker Deployment
|
||||
|
||||
```bash
|
||||
# Pull the image
|
||||
docker pull stellaops/vexlens:latest
|
||||
|
||||
# Run with configuration
|
||||
docker run -d \
|
||||
--name vexlens \
|
||||
-p 8080:8080 \
|
||||
-v /etc/vexlens:/etc/vexlens:ro \
|
||||
-e VEXLENS_STORAGE_POSTGRESQL_CONNECTION_STRING="Host=postgres;Database=vexlens;Username=stellaops;Password=secret" \
|
||||
stellaops/vexlens:latest
|
||||
```
|
||||
|
||||
### 3.2 Kubernetes Deployment
|
||||
|
||||
```yaml
|
||||
apiVersion: apps/v1
|
||||
kind: Deployment
|
||||
metadata:
|
||||
name: vexlens
|
||||
namespace: stellaops
|
||||
spec:
|
||||
replicas: 2
|
||||
selector:
|
||||
matchLabels:
|
||||
app: vexlens
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app: vexlens
|
||||
spec:
|
||||
containers:
|
||||
- name: vexlens
|
||||
image: stellaops/vexlens:latest
|
||||
ports:
|
||||
- containerPort: 8080
|
||||
env:
|
||||
- name: VEXLENS_STORAGE_POSTGRESQL_CONNECTION_STRING
|
||||
valueFrom:
|
||||
secretKeyRef:
|
||||
name: vexlens-secrets
|
||||
key: postgresql-connection-string
|
||||
resources:
|
||||
requests:
|
||||
memory: "512Mi"
|
||||
cpu: "500m"
|
||||
limits:
|
||||
memory: "2Gi"
|
||||
cpu: "2000m"
|
||||
livenessProbe:
|
||||
httpGet:
|
||||
path: /health/live
|
||||
port: 8080
|
||||
initialDelaySeconds: 10
|
||||
periodSeconds: 30
|
||||
readinessProbe:
|
||||
httpGet:
|
||||
path: /health/ready
|
||||
port: 8080
|
||||
initialDelaySeconds: 5
|
||||
periodSeconds: 10
|
||||
volumeMounts:
|
||||
- name: config
|
||||
mountPath: /etc/vexlens
|
||||
readOnly: true
|
||||
volumes:
|
||||
- name: config
|
||||
configMap:
|
||||
name: vexlens-config
|
||||
---
|
||||
apiVersion: v1
|
||||
kind: Service
|
||||
metadata:
|
||||
name: vexlens
|
||||
namespace: stellaops
|
||||
spec:
|
||||
selector:
|
||||
app: vexlens
|
||||
ports:
|
||||
- port: 80
|
||||
targetPort: 8080
|
||||
```
|
||||
|
||||
### 3.3 Helm Deployment
|
||||
|
||||
```bash
|
||||
helm install vexlens stellaops/vexlens \
|
||||
--namespace stellaops \
|
||||
--set postgresql.connectionString="Host=postgres;Database=vexlens;Username=stellaops;Password=secret" \
|
||||
--set replicas=2 \
|
||||
--set resources.requests.memory=512Mi \
|
||||
--set resources.limits.memory=2Gi
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 4) Issuer Directory Setup
|
||||
|
||||
### 4.1 Seed Issuers File
|
||||
|
||||
Create `/etc/vexlens/issuers.json`:
|
||||
|
||||
```json
|
||||
{
|
||||
"issuers": [
|
||||
{
|
||||
"issuerId": "npm-security",
|
||||
"name": "npm Security Team",
|
||||
"category": "Vendor",
|
||||
"trustTier": "Authoritative",
|
||||
"keyFingerprints": [
|
||||
{
|
||||
"fingerprint": "ABCD1234EFGH5678",
|
||||
"keyType": "Pgp",
|
||||
"algorithm": "EdDSA"
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"description": "Official npm security advisories",
|
||||
"uri": "https://www.npmjs.com/advisories"
|
||||
}
|
||||
},
|
||||
{
|
||||
"issuerId": "github-security",
|
||||
"name": "GitHub Security Lab",
|
||||
"category": "Aggregator",
|
||||
"trustTier": "Trusted",
|
||||
"metadata": {
|
||||
"description": "GitHub Security Advisories",
|
||||
"uri": "https://github.com/advisories"
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
```
|
||||
|
||||
### 4.2 Register Issuer via API
|
||||
|
||||
```bash
|
||||
curl -X POST http://vexlens:8080/api/v1/vexlens/issuers \
|
||||
-H "Content-Type: application/json" \
|
||||
-H "X-StellaOps-Tenant: tenant-001" \
|
||||
-d '{
|
||||
"issuerId": "vendor-acme",
|
||||
"name": "ACME Corporation",
|
||||
"category": "Vendor",
|
||||
"trustTier": "Authoritative",
|
||||
"initialKeys": [
|
||||
{
|
||||
"fingerprint": "1234ABCD5678EFGH",
|
||||
"keyType": "Pgp",
|
||||
"algorithm": "RSA"
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"description": "ACME security advisories",
|
||||
"uri": "https://security.acme.example.com"
|
||||
}
|
||||
}'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 5) Health Checks
|
||||
|
||||
### 5.1 Liveness Probe
|
||||
|
||||
```bash
|
||||
curl http://vexlens:8080/health/live
|
||||
# Response: {"status": "Healthy"}
|
||||
```
|
||||
|
||||
### 5.2 Readiness Probe
|
||||
|
||||
```bash
|
||||
curl http://vexlens:8080/health/ready
|
||||
# Response: {"status": "Healthy", "checks": {"postgresql": "Healthy", "issuerDirectory": "Healthy"}}
|
||||
```
|
||||
|
||||
### 5.3 Detailed Health
|
||||
|
||||
```bash
|
||||
curl http://vexlens:8080/health/detailed
|
||||
# Full health check with component details
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 6) Monitoring
|
||||
|
||||
### 6.1 Key Metrics to Monitor
|
||||
|
||||
| Metric | Alert Threshold | Description |
|
||||
|--------|-----------------|-------------|
|
||||
| `vexlens.consensus.duration_seconds` | p99 > 5s | Consensus computation latency |
|
||||
| `vexlens.consensus.conflicts_total` | rate > 100/min | High conflict rate |
|
||||
| `vexlens.normalization.errors_total` | rate > 10/min | Normalization failures |
|
||||
| `vexlens.projection.query_duration_seconds` | p99 > 1s | Slow projection queries |
|
||||
|
||||
### 6.2 Grafana Dashboard
|
||||
|
||||
Import the VexLens dashboard from `deploy/grafana/vexlens-dashboard.json`.
|
||||
|
||||
### 6.3 Alerting Rules
|
||||
|
||||
```yaml
|
||||
groups:
|
||||
- name: vexlens
|
||||
rules:
|
||||
- alert: VexLensHighLatency
|
||||
expr: histogram_quantile(0.99, rate(vexlens_consensus_duration_seconds_bucket[5m])) > 5
|
||||
for: 5m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "VexLens consensus latency is high"
|
||||
|
||||
- alert: VexLensHighConflictRate
|
||||
expr: rate(vexlens_consensus_conflicts_total[5m]) > 100
|
||||
for: 10m
|
||||
labels:
|
||||
severity: warning
|
||||
annotations:
|
||||
summary: "VexLens detecting high conflict rate"
|
||||
|
||||
- alert: VexLensNormalizationErrors
|
||||
expr: rate(vexlens_normalization_errors_total[5m]) > 10
|
||||
for: 5m
|
||||
labels:
|
||||
severity: critical
|
||||
annotations:
|
||||
summary: "VexLens normalization errors increasing"
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 7) Backup and Recovery
|
||||
|
||||
### 7.1 Backup Projections
|
||||
|
||||
```bash
|
||||
# PostgreSQL backup
|
||||
pg_dump -h localhost -U stellaops -d vexlens \
|
||||
-t consensus_projections \
|
||||
-F c -f /backup/vexlens-projections-$(date +%Y%m%d).dump
|
||||
```
|
||||
|
||||
### 7.2 Backup Issuer Directory
|
||||
|
||||
```bash
|
||||
# Export issuers to JSON
|
||||
curl http://vexlens:8080/api/v1/vexlens/issuers?limit=1000 \
|
||||
> /backup/issuers-$(date +%Y%m%d).json
|
||||
```
|
||||
|
||||
### 7.3 Restore
|
||||
|
||||
```bash
|
||||
# Restore PostgreSQL
|
||||
pg_restore -h localhost -U stellaops -d vexlens \
|
||||
/backup/vexlens-projections-20251206.dump
|
||||
|
||||
# Re-seed issuers if needed
|
||||
# Issuers are automatically loaded from seed file on startup
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 8) Scaling
|
||||
|
||||
### 8.1 Horizontal Scaling
|
||||
|
||||
VexLens is stateless for compute operations. Scale horizontally by adding replicas:
|
||||
|
||||
```bash
|
||||
kubectl scale deployment vexlens --replicas=4 -n stellaops
|
||||
```
|
||||
|
||||
### 8.2 Performance Tuning
|
||||
|
||||
```yaml
|
||||
# For high-throughput deployments
|
||||
vexlens:
|
||||
consensus:
|
||||
# Enable batch processing
|
||||
batchSize: 100
|
||||
batchTimeoutMs: 50
|
||||
|
||||
storage:
|
||||
postgresql:
|
||||
# Connection pool
|
||||
maxPoolSize: 100
|
||||
minPoolSize: 10
|
||||
|
||||
caching:
|
||||
enabled: true
|
||||
redis:
|
||||
connectionString: redis://redis:6379
|
||||
consensusTtlMinutes: 5
|
||||
issuerTtlMinutes: 60
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 9) Troubleshooting
|
||||
|
||||
### 9.1 Common Issues
|
||||
|
||||
| Issue | Cause | Resolution |
|
||||
|-------|-------|------------|
|
||||
| Slow consensus | Many statements | Enable caching, increase batch size |
|
||||
| High conflict rate | Inconsistent sources | Review issuer trust tiers |
|
||||
| Normalization failures | Invalid VEX format | Check Excititor connector config |
|
||||
| Low confidence scores | Missing signatures | Configure issuer keys |
|
||||
|
||||
### 9.2 Debug Logging
|
||||
|
||||
```bash
|
||||
# Enable debug logging
|
||||
export VEXLENS_OBSERVABILITY_LOGGING_LEVEL=Debug
|
||||
```
|
||||
|
||||
### 9.3 Determinism Verification
|
||||
|
||||
```bash
|
||||
# Run determinism harness
|
||||
curl -X POST http://vexlens:8080/api/v1/vexlens/test/determinism \
|
||||
-H "Content-Type: application/json" \
|
||||
-d '{"vexContent": "..."}'
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 10) Upgrade Procedure
|
||||
|
||||
### 10.1 Rolling Upgrade
|
||||
|
||||
```bash
|
||||
# Update image
|
||||
kubectl set image deployment/vexlens vexlens=stellaops/vexlens:v1.2.0 -n stellaops
|
||||
|
||||
# Monitor rollout
|
||||
kubectl rollout status deployment/vexlens -n stellaops
|
||||
```
|
||||
|
||||
### 10.2 Database Migrations
|
||||
|
||||
VexLens uses automatic schema migrations. No manual intervention required for minor versions.
|
||||
|
||||
For major version upgrades:
|
||||
1. Backup all data
|
||||
2. Review migration notes in release changelog
|
||||
3. Apply migrations: `vexlens migrate --apply`
|
||||
4. Verify: `vexlens migrate --verify`
|
||||
Reference in New Issue
Block a user