release orchestrator pivot, architecture and planning
This commit is contained in:
656
docs/modules/release-orchestrator/deployment/strategies.md
Normal file
656
docs/modules/release-orchestrator/deployment/strategies.md
Normal file
@@ -0,0 +1,656 @@
|
||||
# Deployment Strategies
|
||||
|
||||
## Overview
|
||||
|
||||
Release Orchestrator supports multiple deployment strategies to balance deployment speed, risk, and availability requirements.
|
||||
|
||||
## Strategy Comparison
|
||||
|
||||
| Strategy | Description | Risk Level | Downtime | Rollback Speed |
|
||||
|----------|-------------|------------|----------|----------------|
|
||||
| All-at-once | Deploy to all targets simultaneously | High | Brief | Fast |
|
||||
| Rolling | Deploy to targets in batches | Medium | None | Medium |
|
||||
| Canary | Deploy to subset, then expand | Low | None | Fast |
|
||||
| Blue-Green | Deploy to parallel environment | Low | None | Instant |
|
||||
|
||||
## All-at-Once Strategy
|
||||
|
||||
### Description
|
||||
|
||||
Deploys to all targets simultaneously. Simple and fast, but highest risk.
|
||||
|
||||
```
|
||||
ALL-AT-ONCE DEPLOYMENT
|
||||
|
||||
Time 0 Time 1
|
||||
┌─────────────────┐ ┌─────────────────┐
|
||||
│ Target 1 [v1] │ │ Target 1 [v2] │
|
||||
├─────────────────┤ ├─────────────────┤
|
||||
│ Target 2 [v1] │ ───► │ Target 2 [v2] │
|
||||
├─────────────────┤ ├─────────────────┤
|
||||
│ Target 3 [v1] │ │ Target 3 [v2] │
|
||||
└─────────────────┘ └─────────────────┘
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```typescript
|
||||
interface AllAtOnceConfig {
|
||||
strategy: "all-at-once";
|
||||
|
||||
// Concurrency limit (0 = unlimited)
|
||||
maxConcurrent: number;
|
||||
|
||||
// Health check after deployment
|
||||
healthCheck: HealthCheckConfig;
|
||||
|
||||
// Failure behavior
|
||||
failureBehavior: "rollback" | "continue" | "pause";
|
||||
}
|
||||
|
||||
// Example
|
||||
const config: AllAtOnceConfig = {
|
||||
strategy: "all-at-once",
|
||||
maxConcurrent: 0,
|
||||
healthCheck: {
|
||||
type: "http",
|
||||
path: "/health",
|
||||
timeout: 30,
|
||||
retries: 3,
|
||||
interval: 10
|
||||
},
|
||||
failureBehavior: "rollback"
|
||||
};
|
||||
```
|
||||
|
||||
### Execution
|
||||
|
||||
```typescript
|
||||
class AllAtOnceExecutor {
|
||||
async execute(job: DeploymentJob, config: AllAtOnceConfig): Promise<void> {
|
||||
const tasks = job.tasks;
|
||||
const concurrency = config.maxConcurrent || tasks.length;
|
||||
|
||||
// Execute all tasks with concurrency limit
|
||||
const results = await pMap(
|
||||
tasks,
|
||||
async (task) => {
|
||||
try {
|
||||
await this.executeTask(task);
|
||||
return { taskId: task.id, success: true };
|
||||
} catch (error) {
|
||||
return { taskId: task.id, success: false, error };
|
||||
}
|
||||
},
|
||||
{ concurrency }
|
||||
);
|
||||
|
||||
// Check for failures
|
||||
const failures = results.filter(r => !r.success);
|
||||
|
||||
if (failures.length > 0) {
|
||||
if (config.failureBehavior === "rollback") {
|
||||
await this.rollbackAll(job);
|
||||
throw new DeploymentFailedError(failures);
|
||||
} else if (config.failureBehavior === "pause") {
|
||||
job.status = "failed";
|
||||
throw new DeploymentFailedError(failures);
|
||||
}
|
||||
// "continue" - proceed despite failures
|
||||
}
|
||||
|
||||
// Health check all targets
|
||||
await this.verifyAllTargets(job, config.healthCheck);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Use Cases
|
||||
|
||||
- Development environments
|
||||
- Small deployments
|
||||
- Time-critical updates
|
||||
- Stateless services with fast startup
|
||||
|
||||
## Rolling Strategy
|
||||
|
||||
### Description
|
||||
|
||||
Deploys to targets in configurable batches, maintaining availability throughout.
|
||||
|
||||
```
|
||||
ROLLING DEPLOYMENT (batch size: 1)
|
||||
|
||||
Time 0 Time 1 Time 2 Time 3
|
||||
┌─────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐
|
||||
│ T1 [v1] │ │ T1 [v2] ✓ │ │ T1 [v2] ✓ │ │ T1 [v2] ✓ │
|
||||
├─────────────┤ ├─────────────┤ ├─────────────┤ ├─────────────┤
|
||||
│ T2 [v1] │──►│ T2 [v1] │──►│ T2 [v2] ✓ │──►│ T2 [v2] ✓ │
|
||||
├─────────────┤ ├─────────────┤ ├─────────────┤ ├─────────────┤
|
||||
│ T3 [v1] │ │ T3 [v1] │ │ T3 [v1] │ │ T3 [v2] ✓ │
|
||||
└─────────────┘ └─────────────┘ └─────────────┘ └─────────────┘
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```typescript
|
||||
interface RollingConfig {
|
||||
strategy: "rolling";
|
||||
|
||||
// Batch configuration
|
||||
batchSize: number; // Targets per batch
|
||||
batchPercent?: number; // Alternative: percentage of targets
|
||||
|
||||
// Timing
|
||||
batchDelay: number; // Seconds between batches
|
||||
stabilizationTime: number; // Wait after health check passes
|
||||
|
||||
// Health check
|
||||
healthCheck: HealthCheckConfig;
|
||||
|
||||
// Failure handling
|
||||
maxFailedBatches: number; // Failures before stopping
|
||||
failureBehavior: "rollback" | "pause" | "skip";
|
||||
|
||||
// Ordering
|
||||
targetOrder: "default" | "shuffle" | "priority";
|
||||
}
|
||||
|
||||
// Example
|
||||
const config: RollingConfig = {
|
||||
strategy: "rolling",
|
||||
batchSize: 2,
|
||||
batchDelay: 30,
|
||||
stabilizationTime: 60,
|
||||
healthCheck: {
|
||||
type: "http",
|
||||
path: "/health",
|
||||
timeout: 30,
|
||||
retries: 5,
|
||||
interval: 10
|
||||
},
|
||||
maxFailedBatches: 1,
|
||||
failureBehavior: "rollback",
|
||||
targetOrder: "default"
|
||||
};
|
||||
```
|
||||
|
||||
### Execution
|
||||
|
||||
```typescript
|
||||
class RollingExecutor {
|
||||
async execute(job: DeploymentJob, config: RollingConfig): Promise<void> {
|
||||
const tasks = this.orderTasks(job.tasks, config.targetOrder);
|
||||
const batches = this.createBatches(tasks, config);
|
||||
let failedBatches = 0;
|
||||
const completedTasks: DeploymentTask[] = [];
|
||||
|
||||
for (const batch of batches) {
|
||||
this.emitProgress(job, {
|
||||
phase: "deploying",
|
||||
currentBatch: batches.indexOf(batch) + 1,
|
||||
totalBatches: batches.length,
|
||||
completedTargets: completedTasks.length,
|
||||
totalTargets: tasks.length
|
||||
});
|
||||
|
||||
// Execute batch
|
||||
const results = await Promise.all(
|
||||
batch.map(task => this.executeTask(task))
|
||||
);
|
||||
|
||||
// Check batch results
|
||||
const failures = results.filter(r => !r.success);
|
||||
|
||||
if (failures.length > 0) {
|
||||
failedBatches++;
|
||||
|
||||
if (failedBatches > config.maxFailedBatches) {
|
||||
if (config.failureBehavior === "rollback") {
|
||||
await this.rollbackCompleted(completedTasks);
|
||||
}
|
||||
throw new DeploymentFailedError(failures);
|
||||
}
|
||||
|
||||
if (config.failureBehavior === "pause") {
|
||||
job.status = "failed";
|
||||
throw new DeploymentFailedError(failures);
|
||||
}
|
||||
// "skip" - continue to next batch
|
||||
}
|
||||
|
||||
// Health check batch targets
|
||||
await this.verifyBatch(batch, config.healthCheck);
|
||||
|
||||
// Wait for stabilization
|
||||
if (config.stabilizationTime > 0) {
|
||||
await sleep(config.stabilizationTime * 1000);
|
||||
}
|
||||
|
||||
completedTasks.push(...batch);
|
||||
|
||||
// Wait before next batch
|
||||
if (batches.indexOf(batch) < batches.length - 1) {
|
||||
await sleep(config.batchDelay * 1000);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private createBatches(
|
||||
tasks: DeploymentTask[],
|
||||
config: RollingConfig
|
||||
): DeploymentTask[][] {
|
||||
const batchSize = config.batchPercent
|
||||
? Math.ceil(tasks.length * config.batchPercent / 100)
|
||||
: config.batchSize;
|
||||
|
||||
const batches: DeploymentTask[][] = [];
|
||||
for (let i = 0; i < tasks.length; i += batchSize) {
|
||||
batches.push(tasks.slice(i, i + batchSize));
|
||||
}
|
||||
|
||||
return batches;
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Use Cases
|
||||
|
||||
- Production deployments
|
||||
- High-availability requirements
|
||||
- Large target counts
|
||||
- Services requiring gradual rollout
|
||||
|
||||
## Canary Strategy
|
||||
|
||||
### Description
|
||||
|
||||
Deploys to a small subset of targets first, validates, then expands to remaining targets.
|
||||
|
||||
```
|
||||
CANARY DEPLOYMENT
|
||||
|
||||
Phase 1: Canary (10%) Phase 2: Expand (50%) Phase 3: Full (100%)
|
||||
|
||||
┌─────────────┐ ┌─────────────┐ ┌─────────────┐
|
||||
│ T1 [v2] ✓ │ ◄─canary │ T1 [v2] ✓ │ │ T1 [v2] ✓ │
|
||||
├─────────────┤ ├─────────────┤ ├─────────────┤
|
||||
│ T2 [v1] │ │ T2 [v2] ✓ │ │ T2 [v2] ✓ │
|
||||
├─────────────┤ ├─────────────┤ ├─────────────┤
|
||||
│ T3 [v1] │ │ T3 [v2] ✓ │ │ T3 [v2] ✓ │
|
||||
├─────────────┤ ├─────────────┤ ├─────────────┤
|
||||
│ T4 [v1] │ │ T4 [v2] ✓ │ │ T4 [v2] ✓ │
|
||||
├─────────────┤ ├─────────────┤ ├─────────────┤
|
||||
│ T5 [v1] │ │ T5 [v1] │ │ T5 [v2] ✓ │
|
||||
└─────────────┘ └─────────────┘ └─────────────┘
|
||||
|
||||
│ │ │
|
||||
▼ ▼ ▼
|
||||
Health Check Health Check Health Check
|
||||
Error Rate Check Error Rate Check Error Rate Check
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```typescript
|
||||
interface CanaryConfig {
|
||||
strategy: "canary";
|
||||
|
||||
// Canary stages
|
||||
stages: CanaryStage[];
|
||||
|
||||
// Canary selection
|
||||
canarySelector: "random" | "labeled" | "first";
|
||||
canaryLabel?: string; // Label for canary targets
|
||||
|
||||
// Automatic vs manual progression
|
||||
autoProgress: boolean;
|
||||
|
||||
// Health and metrics checks
|
||||
healthCheck: HealthCheckConfig;
|
||||
metricsCheck?: MetricsCheckConfig;
|
||||
}
|
||||
|
||||
interface CanaryStage {
|
||||
name: string;
|
||||
percentage: number; // Target percentage
|
||||
duration: number; // Minimum time at this stage (seconds)
|
||||
autoProgress: boolean; // Auto-advance after duration
|
||||
}
|
||||
|
||||
interface MetricsCheckConfig {
|
||||
integrationId: UUID; // Metrics integration
|
||||
queries: MetricQuery[];
|
||||
failureThreshold: number; // Percentage deviation to fail
|
||||
}
|
||||
|
||||
interface MetricQuery {
|
||||
name: string;
|
||||
query: string; // PromQL or similar
|
||||
operator: "lt" | "gt" | "eq";
|
||||
threshold: number;
|
||||
}
|
||||
|
||||
// Example
|
||||
const config: CanaryConfig = {
|
||||
strategy: "canary",
|
||||
stages: [
|
||||
{ name: "canary", percentage: 10, duration: 300, autoProgress: false },
|
||||
{ name: "expand", percentage: 50, duration: 300, autoProgress: true },
|
||||
{ name: "full", percentage: 100, duration: 0, autoProgress: true }
|
||||
],
|
||||
canarySelector: "labeled",
|
||||
canaryLabel: "canary=true",
|
||||
autoProgress: false,
|
||||
healthCheck: {
|
||||
type: "http",
|
||||
path: "/health",
|
||||
timeout: 30,
|
||||
retries: 5,
|
||||
interval: 10
|
||||
},
|
||||
metricsCheck: {
|
||||
integrationId: "prometheus-uuid",
|
||||
queries: [
|
||||
{
|
||||
name: "error_rate",
|
||||
query: "rate(http_requests_total{status=~\"5..\"}[5m]) / rate(http_requests_total[5m])",
|
||||
operator: "lt",
|
||||
threshold: 0.01 // Less than 1% error rate
|
||||
}
|
||||
],
|
||||
failureThreshold: 10
|
||||
}
|
||||
};
|
||||
```
|
||||
|
||||
### Execution
|
||||
|
||||
```typescript
|
||||
class CanaryExecutor {
|
||||
async execute(job: DeploymentJob, config: CanaryConfig): Promise<void> {
|
||||
const tasks = this.orderTasks(job.tasks, config);
|
||||
|
||||
for (const stage of config.stages) {
|
||||
const targetCount = Math.ceil(tasks.length * stage.percentage / 100);
|
||||
const stageTasks = tasks.slice(0, targetCount);
|
||||
const newTasks = stageTasks.filter(t => t.status === "pending");
|
||||
|
||||
this.emitProgress(job, {
|
||||
phase: "canary",
|
||||
stage: stage.name,
|
||||
percentage: stage.percentage,
|
||||
targets: stageTasks.length
|
||||
});
|
||||
|
||||
// Deploy to new targets in this stage
|
||||
await Promise.all(newTasks.map(task => this.executeTask(task)));
|
||||
|
||||
// Health check stage targets
|
||||
await this.verifyTargets(stageTasks, config.healthCheck);
|
||||
|
||||
// Metrics check if configured
|
||||
if (config.metricsCheck) {
|
||||
await this.checkMetrics(stageTasks, config.metricsCheck);
|
||||
}
|
||||
|
||||
// Wait for stage duration
|
||||
if (stage.duration > 0) {
|
||||
await this.waitWithMonitoring(
|
||||
stageTasks,
|
||||
stage.duration,
|
||||
config.metricsCheck
|
||||
);
|
||||
}
|
||||
|
||||
// Wait for manual approval if not auto-progress
|
||||
if (!stage.autoProgress && stage.percentage < 100) {
|
||||
await this.waitForApproval(job, stage.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async checkMetrics(
|
||||
targets: DeploymentTask[],
|
||||
config: MetricsCheckConfig
|
||||
): Promise<void> {
|
||||
const metricsClient = await this.getMetricsClient(config.integrationId);
|
||||
|
||||
for (const query of config.queries) {
|
||||
const result = await metricsClient.query(query.query);
|
||||
|
||||
const passed = this.evaluateMetric(result, query);
|
||||
|
||||
if (!passed) {
|
||||
throw new CanaryMetricsFailedError(query.name, result, query.threshold);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Use Cases
|
||||
|
||||
- Risk-sensitive deployments
|
||||
- Services with real user traffic
|
||||
- Deployments with metrics-based validation
|
||||
- Gradual feature rollouts
|
||||
|
||||
## Blue-Green Strategy
|
||||
|
||||
### Description
|
||||
|
||||
Deploys to a parallel "green" environment while "blue" continues serving traffic, then switches.
|
||||
|
||||
```
|
||||
BLUE-GREEN DEPLOYMENT
|
||||
|
||||
Phase 1: Deploy Green Phase 2: Switch Traffic
|
||||
|
||||
┌─────────────────────────┐ ┌─────────────────────────┐
|
||||
│ Load Balancer │ │ Load Balancer │
|
||||
│ │ │ │ │ │
|
||||
│ ▼ │ │ ▼ │
|
||||
│ ┌─────────────┐ │ │ ┌─────────────┐ │
|
||||
│ │ Blue [v1] │◄─active│ │ │ Blue [v1] │ │
|
||||
│ │ T1, T2, T3 │ │ │ │ T1, T2, T3 │ │
|
||||
│ └─────────────┘ │ │ └─────────────┘ │
|
||||
│ │ │ │
|
||||
│ ┌─────────────┐ │ │ ┌─────────────┐ │
|
||||
│ │ Green [v2] │◄─deploy│ │ │ Green [v2] │◄─active│
|
||||
│ │ T4, T5, T6 │ │ │ │ T4, T5, T6 │ │
|
||||
│ └─────────────┘ │ │ └─────────────┘ │
|
||||
│ │ │ │
|
||||
└─────────────────────────┘ └─────────────────────────┘
|
||||
```
|
||||
|
||||
### Configuration
|
||||
|
||||
```typescript
|
||||
interface BlueGreenConfig {
|
||||
strategy: "blue-green";
|
||||
|
||||
// Environment labels
|
||||
blueLabel: string; // Label for blue targets
|
||||
greenLabel: string; // Label for green targets
|
||||
|
||||
// Traffic routing
|
||||
routerIntegration: UUID; // Router/LB integration
|
||||
routingConfig: RoutingConfig;
|
||||
|
||||
// Validation
|
||||
healthCheck: HealthCheckConfig;
|
||||
warmupTime: number; // Seconds to warm up green
|
||||
validationTests?: string[]; // Test suites to run
|
||||
|
||||
// Switchover
|
||||
switchoverMode: "instant" | "gradual";
|
||||
gradualSteps?: number[]; // Percentage steps for gradual
|
||||
|
||||
// Rollback
|
||||
keepBlueActive: number; // Seconds to keep blue ready
|
||||
}
|
||||
|
||||
// Example
|
||||
const config: BlueGreenConfig = {
|
||||
strategy: "blue-green",
|
||||
blueLabel: "deployment=blue",
|
||||
greenLabel: "deployment=green",
|
||||
routerIntegration: "nginx-lb-uuid",
|
||||
routingConfig: {
|
||||
upstreamName: "myapp",
|
||||
healthEndpoint: "/health"
|
||||
},
|
||||
healthCheck: {
|
||||
type: "http",
|
||||
path: "/health",
|
||||
timeout: 30,
|
||||
retries: 5,
|
||||
interval: 10
|
||||
},
|
||||
warmupTime: 60,
|
||||
validationTests: ["smoke-test-suite"],
|
||||
switchoverMode: "instant",
|
||||
keepBlueActive: 1800 // 30 minutes
|
||||
};
|
||||
```
|
||||
|
||||
### Execution
|
||||
|
||||
```typescript
|
||||
class BlueGreenExecutor {
|
||||
async execute(job: DeploymentJob, config: BlueGreenConfig): Promise<void> {
|
||||
// Identify blue and green targets
|
||||
const { blue, green } = this.categorizeTargets(job.tasks, config);
|
||||
|
||||
// Phase 1: Deploy to green
|
||||
this.emitProgress(job, { phase: "deploying-green" });
|
||||
|
||||
await Promise.all(green.map(task => this.executeTask(task)));
|
||||
|
||||
// Health check green targets
|
||||
await this.verifyTargets(green, config.healthCheck);
|
||||
|
||||
// Warmup period
|
||||
if (config.warmupTime > 0) {
|
||||
this.emitProgress(job, { phase: "warming-up" });
|
||||
await sleep(config.warmupTime * 1000);
|
||||
}
|
||||
|
||||
// Run validation tests
|
||||
if (config.validationTests?.length) {
|
||||
this.emitProgress(job, { phase: "validating" });
|
||||
await this.runValidationTests(green, config.validationTests);
|
||||
}
|
||||
|
||||
// Phase 2: Switch traffic
|
||||
this.emitProgress(job, { phase: "switching-traffic" });
|
||||
|
||||
if (config.switchoverMode === "instant") {
|
||||
await this.instantSwitchover(config, blue, green);
|
||||
} else {
|
||||
await this.gradualSwitchover(config, blue, green);
|
||||
}
|
||||
|
||||
// Verify traffic routing
|
||||
await this.verifyRouting(green, config);
|
||||
|
||||
// Schedule blue decommission
|
||||
if (config.keepBlueActive > 0) {
|
||||
this.scheduleBlueDecommission(blue, config.keepBlueActive);
|
||||
}
|
||||
}
|
||||
|
||||
private async instantSwitchover(
|
||||
config: BlueGreenConfig,
|
||||
blue: DeploymentTask[],
|
||||
green: DeploymentTask[]
|
||||
): Promise<void> {
|
||||
const router = await this.getRouter(config.routerIntegration);
|
||||
|
||||
// Update upstream to green targets
|
||||
await router.updateUpstream(config.routingConfig.upstreamName, {
|
||||
servers: green.map(t => ({
|
||||
address: t.target.address,
|
||||
weight: 1
|
||||
}))
|
||||
});
|
||||
|
||||
// Remove blue from rotation
|
||||
await router.removeServers(
|
||||
config.routingConfig.upstreamName,
|
||||
blue.map(t => t.target.address)
|
||||
);
|
||||
}
|
||||
|
||||
private async gradualSwitchover(
|
||||
config: BlueGreenConfig,
|
||||
blue: DeploymentTask[],
|
||||
green: DeploymentTask[]
|
||||
): Promise<void> {
|
||||
const router = await this.getRouter(config.routerIntegration);
|
||||
const steps = config.gradualSteps || [25, 50, 75, 100];
|
||||
|
||||
for (const percentage of steps) {
|
||||
await router.setTrafficSplit(config.routingConfig.upstreamName, {
|
||||
blue: 100 - percentage,
|
||||
green: percentage
|
||||
});
|
||||
|
||||
// Monitor for errors
|
||||
await this.monitorTraffic(30);
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### Use Cases
|
||||
|
||||
- Zero-downtime deployments
|
||||
- Database migration deployments
|
||||
- High-stakes production updates
|
||||
- Instant rollback requirements
|
||||
|
||||
## Strategy Selection Guide
|
||||
|
||||
```
|
||||
STRATEGY SELECTION
|
||||
|
||||
START
|
||||
│
|
||||
▼
|
||||
┌────────────────────────┐
|
||||
│ Zero downtime needed? │
|
||||
└───────────┬────────────┘
|
||||
│
|
||||
No │ Yes
|
||||
│ │ │
|
||||
▼ │ ▼
|
||||
┌──────────┐ │ ┌───────────────────┐
|
||||
│ All-at- │ │ │ Metrics-based │
|
||||
│ once │ │ │ validation needed?│
|
||||
└──────────┘ │ └─────────┬─────────┘
|
||||
│ │
|
||||
│ No │ Yes
|
||||
│ │ │ │
|
||||
│ ▼ │ ▼
|
||||
│ ┌──────────┐│ ┌──────────┐
|
||||
│ │ Instant ││ │ Canary │
|
||||
│ │ rollback? ││ │ │
|
||||
│ └────┬─────┘│ └──────────┘
|
||||
│ │ │
|
||||
│ No │ Yes │
|
||||
│ │ │ │ │
|
||||
│ ▼ │ ▼ │
|
||||
│┌──────┐│┌────┴─────┐
|
||||
││Rolling│││Blue-Green│
|
||||
│└──────┘│└──────────┘
|
||||
│ │
|
||||
└───────┘
|
||||
```
|
||||
|
||||
## References
|
||||
|
||||
- [Deployment Overview](overview.md)
|
||||
- [Progressive Delivery](../modules/progressive-delivery.md)
|
||||
- [Rollback Management](overview.md#rollback-management)
|
||||
Reference in New Issue
Block a user