Add integration tests for migration categories and execution
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
- Implemented MigrationCategoryTests to validate migration categorization for startup, release, seed, and data migrations. - Added tests for edge cases, including null, empty, and whitespace migration names. - Created StartupMigrationHostTests to verify the behavior of the migration host with real PostgreSQL instances using Testcontainers. - Included tests for migration execution, schema creation, and handling of pending release migrations. - Added SQL migration files for testing: creating a test table, adding a column, a release migration, and seeding data.
This commit is contained in:
master
903
docs/router/17-Step.md
Normal file
903
docs/router/17-Step.md
Normal file
@@ -0,0 +1,903 @@
|
||||
# Step 17: S3/Storage Handler Implementation
|
||||
|
||||
**Phase 4: Handler Plugins**
|
||||
**Estimated Complexity:** Medium
|
||||
**Dependencies:** Step 10 (Microservice Handler)
|
||||
|
||||
---
|
||||
|
||||
## Overview
|
||||
|
||||
The S3/Storage handler routes file operations to object storage backends (S3, MinIO, Azure Blob, GCS). It handles presigned URL generation, multipart uploads, streaming downloads, and integrates with claim-based access control.
|
||||
|
||||
---
|
||||
|
||||
## Goals
|
||||
|
||||
1. Route file operations to appropriate storage backends
|
||||
2. Generate presigned URLs for direct client uploads/downloads
|
||||
3. Support multipart uploads for large files
|
||||
4. Stream files without buffering in gateway
|
||||
5. Enforce claim-based access control on storage operations
|
||||
|
||||
---
|
||||
|
||||
## Core Architecture
|
||||
|
||||
```
|
||||
┌────────────────────────────────────────────────────────────────┐
|
||||
│ Storage Handler │
|
||||
├────────────────────────────────────────────────────────────────┤
|
||||
│ │
|
||||
│ HTTP Request │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌───────────────┐ ┌─────────────────────┐ │
|
||||
│ │ Path Resolver │───►│ Bucket/Key Mapping │ │
|
||||
│ └───────┬───────┘ └─────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌───────────────┐ ┌─────────────────────┐ │
|
||||
│ │Access Control │───►│ Claim-Based Policy │ │
|
||||
│ └───────┬───────┘ └─────────────────────┘ │
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ ┌───────────────────────────────────────────┐ │
|
||||
│ │ Storage Backend │ │
|
||||
│ │ ┌─────┐ ┌───────┐ ┌──────┐ ┌─────┐ │ │
|
||||
│ │ │ S3 │ │ MinIO │ │Azure │ │ GCS │ │ │
|
||||
│ │ └─────┘ └───────┘ └──────┘ └─────┘ │ │
|
||||
│ └───────────────────────────────────────────┘ │
|
||||
│ │
|
||||
└────────────────────────────────────────────────────────────────┘
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Configuration
|
||||
|
||||
```csharp
|
||||
namespace StellaOps.Router.Handlers.Storage;
|
||||
|
||||
public class StorageHandlerConfig
|
||||
{
|
||||
/// <summary>Path prefix for storage routes.</summary>
|
||||
public string PathPrefix { get; set; } = "/files";
|
||||
|
||||
/// <summary>Default storage backend.</summary>
|
||||
public string DefaultBackend { get; set; } = "s3";
|
||||
|
||||
/// <summary>Maximum upload size (bytes).</summary>
|
||||
public long MaxUploadSize { get; set; } = 5L * 1024 * 1024 * 1024; // 5GB
|
||||
|
||||
/// <summary>Multipart threshold (bytes).</summary>
|
||||
public long MultipartThreshold { get; set; } = 100 * 1024 * 1024; // 100MB
|
||||
|
||||
/// <summary>Presigned URL expiration.</summary>
|
||||
public TimeSpan PresignedUrlExpiration { get; set; } = TimeSpan.FromHours(1);
|
||||
|
||||
/// <summary>Whether to use presigned URLs for uploads.</summary>
|
||||
public bool UsePresignedUploads { get; set; } = true;
|
||||
|
||||
/// <summary>Whether to use presigned URLs for downloads.</summary>
|
||||
public bool UsePresignedDownloads { get; set; } = true;
|
||||
|
||||
/// <summary>Storage backends configuration.</summary>
|
||||
public Dictionary<string, StorageBackendConfig> Backends { get; set; } = new();
|
||||
|
||||
/// <summary>Bucket mappings (path pattern to bucket).</summary>
|
||||
public List<BucketMapping> BucketMappings { get; set; } = new();
|
||||
}
|
||||
|
||||
public class StorageBackendConfig
|
||||
{
|
||||
public string Type { get; set; } = "S3"; // S3, Azure, GCS
|
||||
public string Endpoint { get; set; } = "";
|
||||
public string Region { get; set; } = "us-east-1";
|
||||
public string AccessKey { get; set; } = "";
|
||||
public string SecretKey { get; set; } = "";
|
||||
public bool UsePathStyle { get; set; } = false;
|
||||
public bool UseSsl { get; set; } = true;
|
||||
}
|
||||
|
||||
public class BucketMapping
|
||||
{
|
||||
public string PathPattern { get; set; } = "";
|
||||
public string Bucket { get; set; } = "";
|
||||
public string? KeyPrefix { get; set; }
|
||||
public string Backend { get; set; } = "default";
|
||||
public StorageAccessPolicy Policy { get; set; } = new();
|
||||
}
|
||||
|
||||
public class StorageAccessPolicy
|
||||
{
|
||||
public bool RequireAuthentication { get; set; } = true;
|
||||
public List<string> AllowedClaims { get; set; } = new();
|
||||
public string? OwnerClaimPath { get; set; }
|
||||
public bool EnforceOwnership { get; set; } = false;
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Storage Handler Implementation
|
||||
|
||||
```csharp
|
||||
namespace StellaOps.Router.Handlers.Storage;
|
||||
|
||||
public sealed class StorageHandler : IRouteHandler
|
||||
{
|
||||
public string HandlerType => "Storage";
|
||||
public int Priority => 90;
|
||||
|
||||
private readonly StorageHandlerConfig _config;
|
||||
private readonly IStorageBackendFactory _backendFactory;
|
||||
private readonly IAccessControlEvaluator _accessControl;
|
||||
private readonly ILogger<StorageHandler> _logger;
|
||||
|
||||
public StorageHandler(
|
||||
IOptions<StorageHandlerConfig> config,
|
||||
IStorageBackendFactory backendFactory,
|
||||
IAccessControlEvaluator accessControl,
|
||||
ILogger<StorageHandler> logger)
|
||||
{
|
||||
_config = config.Value;
|
||||
_backendFactory = backendFactory;
|
||||
_accessControl = accessControl;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public bool CanHandle(RouteMatchResult match)
|
||||
{
|
||||
return match.Handler == "Storage" ||
|
||||
match.Route.Path.StartsWith(_config.PathPrefix, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public async Task<RouteHandlerResult> HandleAsync(
|
||||
HttpContext context,
|
||||
RouteMatchResult match,
|
||||
IReadOnlyDictionary<string, string> claims,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
try
|
||||
{
|
||||
// Resolve storage location
|
||||
var location = ResolveLocation(context.Request.Path, context.Request.Query);
|
||||
|
||||
// Check access
|
||||
var accessResult = _accessControl.Evaluate(location, claims, context.Request.Method);
|
||||
if (!accessResult.Allowed)
|
||||
{
|
||||
return new RouteHandlerResult
|
||||
{
|
||||
Handled = true,
|
||||
StatusCode = 403,
|
||||
Body = Encoding.UTF8.GetBytes(accessResult.Reason ?? "Access denied")
|
||||
};
|
||||
}
|
||||
|
||||
// Get backend
|
||||
var backend = _backendFactory.GetBackend(location.Backend);
|
||||
|
||||
return context.Request.Method.ToUpper() switch
|
||||
{
|
||||
"GET" => await HandleGetAsync(context, backend, location, cancellationToken),
|
||||
"HEAD" => await HandleHeadAsync(context, backend, location, cancellationToken),
|
||||
"PUT" => await HandlePutAsync(context, backend, location, claims, cancellationToken),
|
||||
"POST" => await HandlePostAsync(context, backend, location, claims, cancellationToken),
|
||||
"DELETE" => await HandleDeleteAsync(context, backend, location, cancellationToken),
|
||||
_ => new RouteHandlerResult { Handled = true, StatusCode = 405 }
|
||||
};
|
||||
}
|
||||
catch (StorageNotFoundException)
|
||||
{
|
||||
return new RouteHandlerResult { Handled = true, StatusCode = 404 };
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Storage operation error");
|
||||
return new RouteHandlerResult
|
||||
{
|
||||
Handled = true,
|
||||
StatusCode = 500,
|
||||
Body = Encoding.UTF8.GetBytes("Storage operation failed")
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
private StorageLocation ResolveLocation(PathString path, IQueryCollection query)
|
||||
{
|
||||
var relativePath = path.Value?.Substring(_config.PathPrefix.Length).TrimStart('/') ?? "";
|
||||
|
||||
foreach (var mapping in _config.BucketMappings)
|
||||
{
|
||||
if (IsMatch(relativePath, mapping.PathPattern))
|
||||
{
|
||||
var key = ExtractKey(relativePath, mapping);
|
||||
return new StorageLocation
|
||||
{
|
||||
Backend = mapping.Backend,
|
||||
Bucket = mapping.Bucket,
|
||||
Key = key,
|
||||
Policy = mapping.Policy
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Default: first segment is bucket, rest is key
|
||||
var segments = relativePath.Split('/', 2);
|
||||
return new StorageLocation
|
||||
{
|
||||
Backend = _config.DefaultBackend,
|
||||
Bucket = segments[0],
|
||||
Key = segments.Length > 1 ? segments[1] : ""
|
||||
};
|
||||
}
|
||||
|
||||
private bool IsMatch(string path, string pattern)
|
||||
{
|
||||
var regex = new Regex("^" + Regex.Escape(pattern).Replace("\\*", ".*") + "$");
|
||||
return regex.IsMatch(path);
|
||||
}
|
||||
|
||||
private string ExtractKey(string path, BucketMapping mapping)
|
||||
{
|
||||
var key = path;
|
||||
if (!string.IsNullOrEmpty(mapping.KeyPrefix))
|
||||
{
|
||||
key = mapping.KeyPrefix.TrimEnd('/') + "/" + key;
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
private async Task<RouteHandlerResult> HandleGetAsync(
|
||||
HttpContext context,
|
||||
IStorageBackend backend,
|
||||
StorageLocation location,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
// Check for presigned download
|
||||
if (_config.UsePresignedDownloads && !IsRangeRequest(context.Request))
|
||||
{
|
||||
var presignedUrl = await backend.GetPresignedDownloadUrlAsync(
|
||||
location.Bucket,
|
||||
location.Key,
|
||||
_config.PresignedUrlExpiration,
|
||||
cancellationToken);
|
||||
|
||||
return new RouteHandlerResult
|
||||
{
|
||||
Handled = true,
|
||||
StatusCode = 307, // Temporary Redirect
|
||||
Headers = new Dictionary<string, string>
|
||||
{
|
||||
["Location"] = presignedUrl,
|
||||
["Cache-Control"] = "no-store"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
// Stream directly
|
||||
var metadata = await backend.GetObjectMetadataAsync(location.Bucket, location.Key, cancellationToken);
|
||||
var stream = await backend.GetObjectStreamAsync(location.Bucket, location.Key, cancellationToken);
|
||||
|
||||
context.Response.StatusCode = 200;
|
||||
context.Response.ContentType = metadata.ContentType;
|
||||
context.Response.ContentLength = metadata.ContentLength;
|
||||
|
||||
if (!string.IsNullOrEmpty(metadata.ETag))
|
||||
{
|
||||
context.Response.Headers["ETag"] = metadata.ETag;
|
||||
}
|
||||
|
||||
await stream.CopyToAsync(context.Response.Body, cancellationToken);
|
||||
|
||||
return new RouteHandlerResult { Handled = true, StatusCode = 200 };
|
||||
}
|
||||
|
||||
private bool IsRangeRequest(HttpRequest request)
|
||||
{
|
||||
return request.Headers.ContainsKey("Range");
|
||||
}
|
||||
|
||||
private async Task<RouteHandlerResult> HandleHeadAsync(
|
||||
HttpContext context,
|
||||
IStorageBackend backend,
|
||||
StorageLocation location,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var metadata = await backend.GetObjectMetadataAsync(location.Bucket, location.Key, cancellationToken);
|
||||
|
||||
return new RouteHandlerResult
|
||||
{
|
||||
Handled = true,
|
||||
StatusCode = 200,
|
||||
Headers = new Dictionary<string, string>
|
||||
{
|
||||
["Content-Type"] = metadata.ContentType,
|
||||
["Content-Length"] = metadata.ContentLength.ToString(),
|
||||
["ETag"] = metadata.ETag ?? "",
|
||||
["Last-Modified"] = metadata.LastModified.ToString("R")
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<RouteHandlerResult> HandlePutAsync(
|
||||
HttpContext context,
|
||||
IStorageBackend backend,
|
||||
StorageLocation location,
|
||||
IReadOnlyDictionary<string, string> claims,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var contentLength = context.Request.ContentLength ?? 0;
|
||||
|
||||
// Validate size
|
||||
if (contentLength > _config.MaxUploadSize)
|
||||
{
|
||||
return new RouteHandlerResult
|
||||
{
|
||||
Handled = true,
|
||||
StatusCode = 413,
|
||||
Body = Encoding.UTF8.GetBytes($"File too large. Max size: {_config.MaxUploadSize}")
|
||||
};
|
||||
}
|
||||
|
||||
// Use presigned upload for large files
|
||||
if (_config.UsePresignedUploads && contentLength > _config.MultipartThreshold)
|
||||
{
|
||||
var uploadInfo = await backend.InitiateMultipartUploadAsync(
|
||||
location.Bucket,
|
||||
location.Key,
|
||||
context.Request.ContentType ?? "application/octet-stream",
|
||||
cancellationToken);
|
||||
|
||||
return new RouteHandlerResult
|
||||
{
|
||||
Handled = true,
|
||||
StatusCode = 200,
|
||||
ContentType = "application/json",
|
||||
Body = JsonSerializer.SerializeToUtf8Bytes(new
|
||||
{
|
||||
uploadId = uploadInfo.UploadId,
|
||||
parts = uploadInfo.PresignedPartUrls
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
// Direct upload
|
||||
var contentType = context.Request.ContentType ?? "application/octet-stream";
|
||||
var metadata = new Dictionary<string, string>();
|
||||
|
||||
// Add owner metadata if enforced
|
||||
if (location.Policy?.EnforceOwnership == true && location.Policy.OwnerClaimPath != null)
|
||||
{
|
||||
if (claims.TryGetValue(location.Policy.OwnerClaimPath, out var owner))
|
||||
{
|
||||
metadata["x-owner"] = owner;
|
||||
}
|
||||
}
|
||||
|
||||
await backend.PutObjectAsync(
|
||||
location.Bucket,
|
||||
location.Key,
|
||||
context.Request.Body,
|
||||
contentLength,
|
||||
contentType,
|
||||
metadata,
|
||||
cancellationToken);
|
||||
|
||||
return new RouteHandlerResult
|
||||
{
|
||||
Handled = true,
|
||||
StatusCode = 201,
|
||||
Headers = new Dictionary<string, string>
|
||||
{
|
||||
["Location"] = $"{_config.PathPrefix}/{location.Bucket}/{location.Key}"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<RouteHandlerResult> HandlePostAsync(
|
||||
HttpContext context,
|
||||
IStorageBackend backend,
|
||||
StorageLocation location,
|
||||
IReadOnlyDictionary<string, string> claims,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var action = context.Request.Query["action"].ToString();
|
||||
|
||||
return action switch
|
||||
{
|
||||
"presign" => await HandlePresignRequestAsync(context, backend, location, cancellationToken),
|
||||
"complete" => await HandleCompleteMultipartAsync(context, backend, location, cancellationToken),
|
||||
"abort" => await HandleAbortMultipartAsync(context, backend, location, cancellationToken),
|
||||
_ => await HandlePutAsync(context, backend, location, claims, cancellationToken)
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<RouteHandlerResult> HandlePresignRequestAsync(
|
||||
HttpContext context,
|
||||
IStorageBackend backend,
|
||||
StorageLocation location,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var method = context.Request.Query["method"].ToString().ToUpper();
|
||||
var expiration = _config.PresignedUrlExpiration;
|
||||
|
||||
string presignedUrl;
|
||||
if (method == "PUT")
|
||||
{
|
||||
var contentType = context.Request.Query["contentType"].ToString();
|
||||
presignedUrl = await backend.GetPresignedUploadUrlAsync(
|
||||
location.Bucket,
|
||||
location.Key,
|
||||
contentType,
|
||||
expiration,
|
||||
cancellationToken);
|
||||
}
|
||||
else
|
||||
{
|
||||
presignedUrl = await backend.GetPresignedDownloadUrlAsync(
|
||||
location.Bucket,
|
||||
location.Key,
|
||||
expiration,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
return new RouteHandlerResult
|
||||
{
|
||||
Handled = true,
|
||||
StatusCode = 200,
|
||||
ContentType = "application/json",
|
||||
Body = JsonSerializer.SerializeToUtf8Bytes(new
|
||||
{
|
||||
url = presignedUrl,
|
||||
expiresAt = DateTimeOffset.UtcNow.Add(expiration)
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
private async Task<RouteHandlerResult> HandleCompleteMultipartAsync(
|
||||
HttpContext context,
|
||||
IStorageBackend backend,
|
||||
StorageLocation location,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var body = await JsonSerializer.DeserializeAsync<CompleteMultipartRequest>(
|
||||
context.Request.Body,
|
||||
cancellationToken: cancellationToken);
|
||||
|
||||
if (body == null)
|
||||
{
|
||||
return new RouteHandlerResult { Handled = true, StatusCode = 400 };
|
||||
}
|
||||
|
||||
await backend.CompleteMultipartUploadAsync(
|
||||
location.Bucket,
|
||||
location.Key,
|
||||
body.UploadId,
|
||||
body.Parts,
|
||||
cancellationToken);
|
||||
|
||||
return new RouteHandlerResult { Handled = true, StatusCode = 200 };
|
||||
}
|
||||
|
||||
private async Task<RouteHandlerResult> HandleAbortMultipartAsync(
|
||||
HttpContext context,
|
||||
IStorageBackend backend,
|
||||
StorageLocation location,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var uploadId = context.Request.Query["uploadId"].ToString();
|
||||
|
||||
await backend.AbortMultipartUploadAsync(
|
||||
location.Bucket,
|
||||
location.Key,
|
||||
uploadId,
|
||||
cancellationToken);
|
||||
|
||||
return new RouteHandlerResult { Handled = true, StatusCode = 204 };
|
||||
}
|
||||
|
||||
private async Task<RouteHandlerResult> HandleDeleteAsync(
|
||||
HttpContext context,
|
||||
IStorageBackend backend,
|
||||
StorageLocation location,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await backend.DeleteObjectAsync(location.Bucket, location.Key, cancellationToken);
|
||||
return new RouteHandlerResult { Handled = true, StatusCode = 204 };
|
||||
}
|
||||
}
|
||||
|
||||
internal class CompleteMultipartRequest
|
||||
{
|
||||
public string UploadId { get; set; } = "";
|
||||
public List<UploadPart> Parts { get; set; } = new();
|
||||
}
|
||||
|
||||
internal class StorageLocation
|
||||
{
|
||||
public string Backend { get; set; } = "";
|
||||
public string Bucket { get; set; } = "";
|
||||
public string Key { get; set; } = "";
|
||||
public StorageAccessPolicy? Policy { get; set; }
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Storage Backend Interface
|
||||
|
||||
```csharp
|
||||
namespace StellaOps.Router.Handlers.Storage;
|
||||
|
||||
public interface IStorageBackend
|
||||
{
|
||||
Task<ObjectMetadata> GetObjectMetadataAsync(
|
||||
string bucket, string key, CancellationToken cancellationToken);
|
||||
|
||||
Task<Stream> GetObjectStreamAsync(
|
||||
string bucket, string key, CancellationToken cancellationToken);
|
||||
|
||||
Task PutObjectAsync(
|
||||
string bucket, string key, Stream content, long contentLength,
|
||||
string contentType, Dictionary<string, string>? metadata,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task DeleteObjectAsync(
|
||||
string bucket, string key, CancellationToken cancellationToken);
|
||||
|
||||
Task<string> GetPresignedDownloadUrlAsync(
|
||||
string bucket, string key, TimeSpan expiration,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<string> GetPresignedUploadUrlAsync(
|
||||
string bucket, string key, string contentType, TimeSpan expiration,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task<MultipartUploadInfo> InitiateMultipartUploadAsync(
|
||||
string bucket, string key, string contentType,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task CompleteMultipartUploadAsync(
|
||||
string bucket, string key, string uploadId, List<UploadPart> parts,
|
||||
CancellationToken cancellationToken);
|
||||
|
||||
Task AbortMultipartUploadAsync(
|
||||
string bucket, string key, string uploadId,
|
||||
CancellationToken cancellationToken);
|
||||
}
|
||||
|
||||
public class ObjectMetadata
|
||||
{
|
||||
public string ContentType { get; set; } = "application/octet-stream";
|
||||
public long ContentLength { get; set; }
|
||||
public string? ETag { get; set; }
|
||||
public DateTimeOffset LastModified { get; set; }
|
||||
public Dictionary<string, string> CustomMetadata { get; set; } = new();
|
||||
}
|
||||
|
||||
public class MultipartUploadInfo
|
||||
{
|
||||
public string UploadId { get; set; } = "";
|
||||
public List<PresignedPartUrl> PresignedPartUrls { get; set; } = new();
|
||||
}
|
||||
|
||||
public class PresignedPartUrl
|
||||
{
|
||||
public int PartNumber { get; set; }
|
||||
public string Url { get; set; } = "";
|
||||
}
|
||||
|
||||
public class UploadPart
|
||||
{
|
||||
public int PartNumber { get; set; }
|
||||
public string ETag { get; set; } = "";
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## S3 Backend Implementation
|
||||
|
||||
```csharp
|
||||
namespace StellaOps.Router.Handlers.Storage;
|
||||
|
||||
public sealed class S3StorageBackend : IStorageBackend
|
||||
{
|
||||
private readonly IAmazonS3 _client;
|
||||
private readonly ILogger<S3StorageBackend> _logger;
|
||||
|
||||
public S3StorageBackend(IAmazonS3 client, ILogger<S3StorageBackend> logger)
|
||||
{
|
||||
_client = client;
|
||||
_logger = logger;
|
||||
}
|
||||
|
||||
public async Task<ObjectMetadata> GetObjectMetadataAsync(
|
||||
string bucket, string key, CancellationToken cancellationToken)
|
||||
{
|
||||
var response = await _client.GetObjectMetadataAsync(bucket, key, cancellationToken);
|
||||
|
||||
return new ObjectMetadata
|
||||
{
|
||||
ContentType = response.Headers.ContentType,
|
||||
ContentLength = response.ContentLength,
|
||||
ETag = response.ETag,
|
||||
LastModified = response.LastModified,
|
||||
CustomMetadata = response.Metadata.Keys
|
||||
.ToDictionary(k => k, k => response.Metadata[k])
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<Stream> GetObjectStreamAsync(
|
||||
string bucket, string key, CancellationToken cancellationToken)
|
||||
{
|
||||
var response = await _client.GetObjectAsync(bucket, key, cancellationToken);
|
||||
return response.ResponseStream;
|
||||
}
|
||||
|
||||
public async Task PutObjectAsync(
|
||||
string bucket, string key, Stream content, long contentLength,
|
||||
string contentType, Dictionary<string, string>? metadata,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new PutObjectRequest
|
||||
{
|
||||
BucketName = bucket,
|
||||
Key = key,
|
||||
InputStream = content,
|
||||
ContentType = contentType
|
||||
};
|
||||
|
||||
if (metadata != null)
|
||||
{
|
||||
foreach (var (k, v) in metadata)
|
||||
{
|
||||
request.Metadata.Add(k, v);
|
||||
}
|
||||
}
|
||||
|
||||
await _client.PutObjectAsync(request, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task DeleteObjectAsync(
|
||||
string bucket, string key, CancellationToken cancellationToken)
|
||||
{
|
||||
await _client.DeleteObjectAsync(bucket, key, cancellationToken);
|
||||
}
|
||||
|
||||
public Task<string> GetPresignedDownloadUrlAsync(
|
||||
string bucket, string key, TimeSpan expiration,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new GetPreSignedUrlRequest
|
||||
{
|
||||
BucketName = bucket,
|
||||
Key = key,
|
||||
Expires = DateTime.UtcNow.Add(expiration),
|
||||
Verb = HttpVerb.GET
|
||||
};
|
||||
|
||||
var url = _client.GetPreSignedURL(request);
|
||||
return Task.FromResult(url);
|
||||
}
|
||||
|
||||
public Task<string> GetPresignedUploadUrlAsync(
|
||||
string bucket, string key, string contentType, TimeSpan expiration,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new GetPreSignedUrlRequest
|
||||
{
|
||||
BucketName = bucket,
|
||||
Key = key,
|
||||
Expires = DateTime.UtcNow.Add(expiration),
|
||||
Verb = HttpVerb.PUT,
|
||||
ContentType = contentType
|
||||
};
|
||||
|
||||
var url = _client.GetPreSignedURL(request);
|
||||
return Task.FromResult(url);
|
||||
}
|
||||
|
||||
public async Task<MultipartUploadInfo> InitiateMultipartUploadAsync(
|
||||
string bucket, string key, string contentType,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var initResponse = await _client.InitiateMultipartUploadAsync(
|
||||
bucket, key, cancellationToken);
|
||||
|
||||
// Generate presigned URLs for parts (assuming 100MB parts, 50 parts max)
|
||||
var partUrls = new List<PresignedPartUrl>();
|
||||
for (int i = 1; i <= 50; i++)
|
||||
{
|
||||
var url = _client.GetPreSignedURL(new GetPreSignedUrlRequest
|
||||
{
|
||||
BucketName = bucket,
|
||||
Key = key,
|
||||
Expires = DateTime.UtcNow.AddHours(24),
|
||||
Verb = HttpVerb.PUT,
|
||||
UploadId = initResponse.UploadId,
|
||||
PartNumber = i
|
||||
});
|
||||
|
||||
partUrls.Add(new PresignedPartUrl { PartNumber = i, Url = url });
|
||||
}
|
||||
|
||||
return new MultipartUploadInfo
|
||||
{
|
||||
UploadId = initResponse.UploadId,
|
||||
PresignedPartUrls = partUrls
|
||||
};
|
||||
}
|
||||
|
||||
public async Task CompleteMultipartUploadAsync(
|
||||
string bucket, string key, string uploadId, List<UploadPart> parts,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var request = new CompleteMultipartUploadRequest
|
||||
{
|
||||
BucketName = bucket,
|
||||
Key = key,
|
||||
UploadId = uploadId,
|
||||
PartETags = parts.Select(p => new PartETag(p.PartNumber, p.ETag)).ToList()
|
||||
};
|
||||
|
||||
await _client.CompleteMultipartUploadAsync(request, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task AbortMultipartUploadAsync(
|
||||
string bucket, string key, string uploadId,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
await _client.AbortMultipartUploadAsync(bucket, key, uploadId, cancellationToken);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Access Control Evaluator
|
||||
|
||||
```csharp
|
||||
namespace StellaOps.Router.Handlers.Storage;
|
||||
|
||||
public interface IAccessControlEvaluator
|
||||
{
|
||||
AccessResult Evaluate(
|
||||
StorageLocation location,
|
||||
IReadOnlyDictionary<string, string> claims,
|
||||
string httpMethod);
|
||||
}
|
||||
|
||||
public class AccessResult
|
||||
{
|
||||
public bool Allowed { get; set; }
|
||||
public string? Reason { get; set; }
|
||||
}
|
||||
|
||||
public sealed class ClaimBasedAccessControlEvaluator : IAccessControlEvaluator
|
||||
{
|
||||
public AccessResult Evaluate(
|
||||
StorageLocation location,
|
||||
IReadOnlyDictionary<string, string> claims,
|
||||
string httpMethod)
|
||||
{
|
||||
var policy = location.Policy ?? new StorageAccessPolicy();
|
||||
|
||||
// Check authentication requirement
|
||||
if (policy.RequireAuthentication && !claims.Any())
|
||||
{
|
||||
return new AccessResult { Allowed = false, Reason = "Authentication required" };
|
||||
}
|
||||
|
||||
// Check allowed claims
|
||||
if (policy.AllowedClaims.Any())
|
||||
{
|
||||
var hasRequiredClaim = policy.AllowedClaims.Any(c =>
|
||||
{
|
||||
var parts = c.Split('=', 2);
|
||||
if (parts.Length == 2)
|
||||
{
|
||||
return claims.TryGetValue(parts[0], out var value) && value == parts[1];
|
||||
}
|
||||
return claims.ContainsKey(c);
|
||||
});
|
||||
|
||||
if (!hasRequiredClaim)
|
||||
{
|
||||
return new AccessResult { Allowed = false, Reason = "Required claim not present" };
|
||||
}
|
||||
}
|
||||
|
||||
// Check ownership for write operations
|
||||
if (policy.EnforceOwnership && IsWriteOperation(httpMethod))
|
||||
{
|
||||
if (string.IsNullOrEmpty(policy.OwnerClaimPath))
|
||||
{
|
||||
return new AccessResult { Allowed = false, Reason = "Owner claim path not configured" };
|
||||
}
|
||||
|
||||
if (!claims.ContainsKey(policy.OwnerClaimPath))
|
||||
{
|
||||
return new AccessResult { Allowed = false, Reason = "Owner claim required" };
|
||||
}
|
||||
}
|
||||
|
||||
return new AccessResult { Allowed = true };
|
||||
}
|
||||
|
||||
private bool IsWriteOperation(string method)
|
||||
{
|
||||
return method.ToUpper() is "PUT" or "POST" or "DELETE" or "PATCH";
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## YAML Configuration
|
||||
|
||||
```yaml
|
||||
Storage:
|
||||
PathPrefix: "/files"
|
||||
DefaultBackend: "s3"
|
||||
MaxUploadSize: 5368709120 # 5GB
|
||||
MultipartThreshold: 104857600 # 100MB
|
||||
PresignedUrlExpiration: "01:00:00"
|
||||
UsePresignedUploads: true
|
||||
UsePresignedDownloads: true
|
||||
|
||||
Backends:
|
||||
s3:
|
||||
Type: "S3"
|
||||
Endpoint: "https://s3.amazonaws.com"
|
||||
Region: "us-east-1"
|
||||
AccessKey: "${AWS_ACCESS_KEY}"
|
||||
SecretKey: "${AWS_SECRET_KEY}"
|
||||
minio:
|
||||
Type: "S3"
|
||||
Endpoint: "https://minio.internal:9000"
|
||||
Region: "us-east-1"
|
||||
AccessKey: "${MINIO_ACCESS_KEY}"
|
||||
SecretKey: "${MINIO_SECRET_KEY}"
|
||||
UsePathStyle: true
|
||||
|
||||
BucketMappings:
|
||||
- PathPattern: "uploads/*"
|
||||
Bucket: "user-uploads"
|
||||
KeyPrefix: "files/"
|
||||
Backend: "s3"
|
||||
Policy:
|
||||
RequireAuthentication: true
|
||||
EnforceOwnership: true
|
||||
OwnerClaimPath: "sub"
|
||||
|
||||
- PathPattern: "public/*"
|
||||
Bucket: "public-assets"
|
||||
Backend: "s3"
|
||||
Policy:
|
||||
RequireAuthentication: false
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Deliverables
|
||||
|
||||
1. `StellaOps.Router.Handlers.Storage/StorageHandler.cs`
|
||||
2. `StellaOps.Router.Handlers.Storage/StorageHandlerConfig.cs`
|
||||
3. `StellaOps.Router.Handlers.Storage/IStorageBackend.cs`
|
||||
4. `StellaOps.Router.Handlers.Storage/S3StorageBackend.cs`
|
||||
5. `StellaOps.Router.Handlers.Storage/IAccessControlEvaluator.cs`
|
||||
6. `StellaOps.Router.Handlers.Storage/ClaimBasedAccessControlEvaluator.cs`
|
||||
7. `StellaOps.Router.Handlers.Storage/StorageBackendFactory.cs`
|
||||
8. Presigned URL generation tests
|
||||
9. Multipart upload tests
|
||||
10. Access control tests
|
||||
|
||||
---
|
||||
|
||||
## Next Step
|
||||
|
||||
Proceed to [Step 18: Reverse Proxy Handler Implementation](18-Step.md) to implement direct reverse proxy routing.
|
||||
Reference in New Issue
Block a user