Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled
Policy Lint & Smoke / policy-lint (push) Has been cancelled
Concelier Attestation Tests / attestation-tests (push) Has been cancelled
AOC Guard CI / aoc-guard (push) Has been cancelled
AOC Guard CI / aoc-verify (push) Has been cancelled
- Implemented MigrationCategoryTests to validate migration categorization for startup, release, seed, and data migrations. - Added tests for edge cases, including null, empty, and whitespace migration names. - Created StartupMigrationHostTests to verify the behavior of the migration host with real PostgreSQL instances using Testcontainers. - Included tests for migration execution, schema creation, and handling of pending release migrations. - Added SQL migration files for testing: creating a test table, adding a column, a release migration, and seeding data.
904 lines
30 KiB
Markdown
904 lines
30 KiB
Markdown
# Step 17: S3/Storage Handler Implementation
|
|
|
|
**Phase 4: Handler Plugins**
|
|
**Estimated Complexity:** Medium
|
|
**Dependencies:** Step 10 (Microservice Handler)
|
|
|
|
---
|
|
|
|
## Overview
|
|
|
|
The S3/Storage handler routes file operations to object storage backends (S3, MinIO, Azure Blob, GCS). It handles presigned URL generation, multipart uploads, streaming downloads, and integrates with claim-based access control.
|
|
|
|
---
|
|
|
|
## Goals
|
|
|
|
1. Route file operations to appropriate storage backends
|
|
2. Generate presigned URLs for direct client uploads/downloads
|
|
3. Support multipart uploads for large files
|
|
4. Stream files without buffering in gateway
|
|
5. Enforce claim-based access control on storage operations
|
|
|
|
---
|
|
|
|
## Core Architecture
|
|
|
|
```
|
|
┌────────────────────────────────────────────────────────────────┐
|
|
│ Storage Handler │
|
|
├────────────────────────────────────────────────────────────────┤
|
|
│ │
|
|
│ HTTP Request │
|
|
│ │ │
|
|
│ ▼ │
|
|
│ ┌───────────────┐ ┌─────────────────────┐ │
|
|
│ │ Path Resolver │───►│ Bucket/Key Mapping │ │
|
|
│ └───────┬───────┘ └─────────────────────┘ │
|
|
│ │ │
|
|
│ ▼ │
|
|
│ ┌───────────────┐ ┌─────────────────────┐ │
|
|
│ │Access Control │───►│ Claim-Based Policy │ │
|
|
│ └───────┬───────┘ └─────────────────────┘ │
|
|
│ │ │
|
|
│ ▼ │
|
|
│ ┌───────────────────────────────────────────┐ │
|
|
│ │ Storage Backend │ │
|
|
│ │ ┌─────┐ ┌───────┐ ┌──────┐ ┌─────┐ │ │
|
|
│ │ │ S3 │ │ MinIO │ │Azure │ │ GCS │ │ │
|
|
│ │ └─────┘ └───────┘ └──────┘ └─────┘ │ │
|
|
│ └───────────────────────────────────────────┘ │
|
|
│ │
|
|
└────────────────────────────────────────────────────────────────┘
|
|
```
|
|
|
|
---
|
|
|
|
## Configuration
|
|
|
|
```csharp
|
|
namespace StellaOps.Router.Handlers.Storage;
|
|
|
|
public class StorageHandlerConfig
|
|
{
|
|
/// <summary>Path prefix for storage routes.</summary>
|
|
public string PathPrefix { get; set; } = "/files";
|
|
|
|
/// <summary>Default storage backend.</summary>
|
|
public string DefaultBackend { get; set; } = "s3";
|
|
|
|
/// <summary>Maximum upload size (bytes).</summary>
|
|
public long MaxUploadSize { get; set; } = 5L * 1024 * 1024 * 1024; // 5GB
|
|
|
|
/// <summary>Multipart threshold (bytes).</summary>
|
|
public long MultipartThreshold { get; set; } = 100 * 1024 * 1024; // 100MB
|
|
|
|
/// <summary>Presigned URL expiration.</summary>
|
|
public TimeSpan PresignedUrlExpiration { get; set; } = TimeSpan.FromHours(1);
|
|
|
|
/// <summary>Whether to use presigned URLs for uploads.</summary>
|
|
public bool UsePresignedUploads { get; set; } = true;
|
|
|
|
/// <summary>Whether to use presigned URLs for downloads.</summary>
|
|
public bool UsePresignedDownloads { get; set; } = true;
|
|
|
|
/// <summary>Storage backends configuration.</summary>
|
|
public Dictionary<string, StorageBackendConfig> Backends { get; set; } = new();
|
|
|
|
/// <summary>Bucket mappings (path pattern to bucket).</summary>
|
|
public List<BucketMapping> BucketMappings { get; set; } = new();
|
|
}
|
|
|
|
public class StorageBackendConfig
|
|
{
|
|
public string Type { get; set; } = "S3"; // S3, Azure, GCS
|
|
public string Endpoint { get; set; } = "";
|
|
public string Region { get; set; } = "us-east-1";
|
|
public string AccessKey { get; set; } = "";
|
|
public string SecretKey { get; set; } = "";
|
|
public bool UsePathStyle { get; set; } = false;
|
|
public bool UseSsl { get; set; } = true;
|
|
}
|
|
|
|
public class BucketMapping
|
|
{
|
|
public string PathPattern { get; set; } = "";
|
|
public string Bucket { get; set; } = "";
|
|
public string? KeyPrefix { get; set; }
|
|
public string Backend { get; set; } = "default";
|
|
public StorageAccessPolicy Policy { get; set; } = new();
|
|
}
|
|
|
|
public class StorageAccessPolicy
|
|
{
|
|
public bool RequireAuthentication { get; set; } = true;
|
|
public List<string> AllowedClaims { get; set; } = new();
|
|
public string? OwnerClaimPath { get; set; }
|
|
public bool EnforceOwnership { get; set; } = false;
|
|
}
|
|
```
|
|
|
|
---
|
|
|
|
## Storage Handler Implementation
|
|
|
|
```csharp
|
|
namespace StellaOps.Router.Handlers.Storage;
|
|
|
|
public sealed class StorageHandler : IRouteHandler
|
|
{
|
|
public string HandlerType => "Storage";
|
|
public int Priority => 90;
|
|
|
|
private readonly StorageHandlerConfig _config;
|
|
private readonly IStorageBackendFactory _backendFactory;
|
|
private readonly IAccessControlEvaluator _accessControl;
|
|
private readonly ILogger<StorageHandler> _logger;
|
|
|
|
public StorageHandler(
|
|
IOptions<StorageHandlerConfig> config,
|
|
IStorageBackendFactory backendFactory,
|
|
IAccessControlEvaluator accessControl,
|
|
ILogger<StorageHandler> logger)
|
|
{
|
|
_config = config.Value;
|
|
_backendFactory = backendFactory;
|
|
_accessControl = accessControl;
|
|
_logger = logger;
|
|
}
|
|
|
|
public bool CanHandle(RouteMatchResult match)
|
|
{
|
|
return match.Handler == "Storage" ||
|
|
match.Route.Path.StartsWith(_config.PathPrefix, StringComparison.OrdinalIgnoreCase);
|
|
}
|
|
|
|
public async Task<RouteHandlerResult> HandleAsync(
|
|
HttpContext context,
|
|
RouteMatchResult match,
|
|
IReadOnlyDictionary<string, string> claims,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
try
|
|
{
|
|
// Resolve storage location
|
|
var location = ResolveLocation(context.Request.Path, context.Request.Query);
|
|
|
|
// Check access
|
|
var accessResult = _accessControl.Evaluate(location, claims, context.Request.Method);
|
|
if (!accessResult.Allowed)
|
|
{
|
|
return new RouteHandlerResult
|
|
{
|
|
Handled = true,
|
|
StatusCode = 403,
|
|
Body = Encoding.UTF8.GetBytes(accessResult.Reason ?? "Access denied")
|
|
};
|
|
}
|
|
|
|
// Get backend
|
|
var backend = _backendFactory.GetBackend(location.Backend);
|
|
|
|
return context.Request.Method.ToUpper() switch
|
|
{
|
|
"GET" => await HandleGetAsync(context, backend, location, cancellationToken),
|
|
"HEAD" => await HandleHeadAsync(context, backend, location, cancellationToken),
|
|
"PUT" => await HandlePutAsync(context, backend, location, claims, cancellationToken),
|
|
"POST" => await HandlePostAsync(context, backend, location, claims, cancellationToken),
|
|
"DELETE" => await HandleDeleteAsync(context, backend, location, cancellationToken),
|
|
_ => new RouteHandlerResult { Handled = true, StatusCode = 405 }
|
|
};
|
|
}
|
|
catch (StorageNotFoundException)
|
|
{
|
|
return new RouteHandlerResult { Handled = true, StatusCode = 404 };
|
|
}
|
|
catch (Exception ex)
|
|
{
|
|
_logger.LogError(ex, "Storage operation error");
|
|
return new RouteHandlerResult
|
|
{
|
|
Handled = true,
|
|
StatusCode = 500,
|
|
Body = Encoding.UTF8.GetBytes("Storage operation failed")
|
|
};
|
|
}
|
|
}
|
|
|
|
private StorageLocation ResolveLocation(PathString path, IQueryCollection query)
|
|
{
|
|
var relativePath = path.Value?.Substring(_config.PathPrefix.Length).TrimStart('/') ?? "";
|
|
|
|
foreach (var mapping in _config.BucketMappings)
|
|
{
|
|
if (IsMatch(relativePath, mapping.PathPattern))
|
|
{
|
|
var key = ExtractKey(relativePath, mapping);
|
|
return new StorageLocation
|
|
{
|
|
Backend = mapping.Backend,
|
|
Bucket = mapping.Bucket,
|
|
Key = key,
|
|
Policy = mapping.Policy
|
|
};
|
|
}
|
|
}
|
|
|
|
// Default: first segment is bucket, rest is key
|
|
var segments = relativePath.Split('/', 2);
|
|
return new StorageLocation
|
|
{
|
|
Backend = _config.DefaultBackend,
|
|
Bucket = segments[0],
|
|
Key = segments.Length > 1 ? segments[1] : ""
|
|
};
|
|
}
|
|
|
|
private bool IsMatch(string path, string pattern)
|
|
{
|
|
var regex = new Regex("^" + Regex.Escape(pattern).Replace("\\*", ".*") + "$");
|
|
return regex.IsMatch(path);
|
|
}
|
|
|
|
private string ExtractKey(string path, BucketMapping mapping)
|
|
{
|
|
var key = path;
|
|
if (!string.IsNullOrEmpty(mapping.KeyPrefix))
|
|
{
|
|
key = mapping.KeyPrefix.TrimEnd('/') + "/" + key;
|
|
}
|
|
return key;
|
|
}
|
|
|
|
private async Task<RouteHandlerResult> HandleGetAsync(
|
|
HttpContext context,
|
|
IStorageBackend backend,
|
|
StorageLocation location,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
// Check for presigned download
|
|
if (_config.UsePresignedDownloads && !IsRangeRequest(context.Request))
|
|
{
|
|
var presignedUrl = await backend.GetPresignedDownloadUrlAsync(
|
|
location.Bucket,
|
|
location.Key,
|
|
_config.PresignedUrlExpiration,
|
|
cancellationToken);
|
|
|
|
return new RouteHandlerResult
|
|
{
|
|
Handled = true,
|
|
StatusCode = 307, // Temporary Redirect
|
|
Headers = new Dictionary<string, string>
|
|
{
|
|
["Location"] = presignedUrl,
|
|
["Cache-Control"] = "no-store"
|
|
}
|
|
};
|
|
}
|
|
|
|
// Stream directly
|
|
var metadata = await backend.GetObjectMetadataAsync(location.Bucket, location.Key, cancellationToken);
|
|
var stream = await backend.GetObjectStreamAsync(location.Bucket, location.Key, cancellationToken);
|
|
|
|
context.Response.StatusCode = 200;
|
|
context.Response.ContentType = metadata.ContentType;
|
|
context.Response.ContentLength = metadata.ContentLength;
|
|
|
|
if (!string.IsNullOrEmpty(metadata.ETag))
|
|
{
|
|
context.Response.Headers["ETag"] = metadata.ETag;
|
|
}
|
|
|
|
await stream.CopyToAsync(context.Response.Body, cancellationToken);
|
|
|
|
return new RouteHandlerResult { Handled = true, StatusCode = 200 };
|
|
}
|
|
|
|
private bool IsRangeRequest(HttpRequest request)
|
|
{
|
|
return request.Headers.ContainsKey("Range");
|
|
}
|
|
|
|
private async Task<RouteHandlerResult> HandleHeadAsync(
|
|
HttpContext context,
|
|
IStorageBackend backend,
|
|
StorageLocation location,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var metadata = await backend.GetObjectMetadataAsync(location.Bucket, location.Key, cancellationToken);
|
|
|
|
return new RouteHandlerResult
|
|
{
|
|
Handled = true,
|
|
StatusCode = 200,
|
|
Headers = new Dictionary<string, string>
|
|
{
|
|
["Content-Type"] = metadata.ContentType,
|
|
["Content-Length"] = metadata.ContentLength.ToString(),
|
|
["ETag"] = metadata.ETag ?? "",
|
|
["Last-Modified"] = metadata.LastModified.ToString("R")
|
|
}
|
|
};
|
|
}
|
|
|
|
private async Task<RouteHandlerResult> HandlePutAsync(
|
|
HttpContext context,
|
|
IStorageBackend backend,
|
|
StorageLocation location,
|
|
IReadOnlyDictionary<string, string> claims,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var contentLength = context.Request.ContentLength ?? 0;
|
|
|
|
// Validate size
|
|
if (contentLength > _config.MaxUploadSize)
|
|
{
|
|
return new RouteHandlerResult
|
|
{
|
|
Handled = true,
|
|
StatusCode = 413,
|
|
Body = Encoding.UTF8.GetBytes($"File too large. Max size: {_config.MaxUploadSize}")
|
|
};
|
|
}
|
|
|
|
// Use presigned upload for large files
|
|
if (_config.UsePresignedUploads && contentLength > _config.MultipartThreshold)
|
|
{
|
|
var uploadInfo = await backend.InitiateMultipartUploadAsync(
|
|
location.Bucket,
|
|
location.Key,
|
|
context.Request.ContentType ?? "application/octet-stream",
|
|
cancellationToken);
|
|
|
|
return new RouteHandlerResult
|
|
{
|
|
Handled = true,
|
|
StatusCode = 200,
|
|
ContentType = "application/json",
|
|
Body = JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
uploadId = uploadInfo.UploadId,
|
|
parts = uploadInfo.PresignedPartUrls
|
|
})
|
|
};
|
|
}
|
|
|
|
// Direct upload
|
|
var contentType = context.Request.ContentType ?? "application/octet-stream";
|
|
var metadata = new Dictionary<string, string>();
|
|
|
|
// Add owner metadata if enforced
|
|
if (location.Policy?.EnforceOwnership == true && location.Policy.OwnerClaimPath != null)
|
|
{
|
|
if (claims.TryGetValue(location.Policy.OwnerClaimPath, out var owner))
|
|
{
|
|
metadata["x-owner"] = owner;
|
|
}
|
|
}
|
|
|
|
await backend.PutObjectAsync(
|
|
location.Bucket,
|
|
location.Key,
|
|
context.Request.Body,
|
|
contentLength,
|
|
contentType,
|
|
metadata,
|
|
cancellationToken);
|
|
|
|
return new RouteHandlerResult
|
|
{
|
|
Handled = true,
|
|
StatusCode = 201,
|
|
Headers = new Dictionary<string, string>
|
|
{
|
|
["Location"] = $"{_config.PathPrefix}/{location.Bucket}/{location.Key}"
|
|
}
|
|
};
|
|
}
|
|
|
|
private async Task<RouteHandlerResult> HandlePostAsync(
|
|
HttpContext context,
|
|
IStorageBackend backend,
|
|
StorageLocation location,
|
|
IReadOnlyDictionary<string, string> claims,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var action = context.Request.Query["action"].ToString();
|
|
|
|
return action switch
|
|
{
|
|
"presign" => await HandlePresignRequestAsync(context, backend, location, cancellationToken),
|
|
"complete" => await HandleCompleteMultipartAsync(context, backend, location, cancellationToken),
|
|
"abort" => await HandleAbortMultipartAsync(context, backend, location, cancellationToken),
|
|
_ => await HandlePutAsync(context, backend, location, claims, cancellationToken)
|
|
};
|
|
}
|
|
|
|
private async Task<RouteHandlerResult> HandlePresignRequestAsync(
|
|
HttpContext context,
|
|
IStorageBackend backend,
|
|
StorageLocation location,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var method = context.Request.Query["method"].ToString().ToUpper();
|
|
var expiration = _config.PresignedUrlExpiration;
|
|
|
|
string presignedUrl;
|
|
if (method == "PUT")
|
|
{
|
|
var contentType = context.Request.Query["contentType"].ToString();
|
|
presignedUrl = await backend.GetPresignedUploadUrlAsync(
|
|
location.Bucket,
|
|
location.Key,
|
|
contentType,
|
|
expiration,
|
|
cancellationToken);
|
|
}
|
|
else
|
|
{
|
|
presignedUrl = await backend.GetPresignedDownloadUrlAsync(
|
|
location.Bucket,
|
|
location.Key,
|
|
expiration,
|
|
cancellationToken);
|
|
}
|
|
|
|
return new RouteHandlerResult
|
|
{
|
|
Handled = true,
|
|
StatusCode = 200,
|
|
ContentType = "application/json",
|
|
Body = JsonSerializer.SerializeToUtf8Bytes(new
|
|
{
|
|
url = presignedUrl,
|
|
expiresAt = DateTimeOffset.UtcNow.Add(expiration)
|
|
})
|
|
};
|
|
}
|
|
|
|
private async Task<RouteHandlerResult> HandleCompleteMultipartAsync(
|
|
HttpContext context,
|
|
IStorageBackend backend,
|
|
StorageLocation location,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var body = await JsonSerializer.DeserializeAsync<CompleteMultipartRequest>(
|
|
context.Request.Body,
|
|
cancellationToken: cancellationToken);
|
|
|
|
if (body == null)
|
|
{
|
|
return new RouteHandlerResult { Handled = true, StatusCode = 400 };
|
|
}
|
|
|
|
await backend.CompleteMultipartUploadAsync(
|
|
location.Bucket,
|
|
location.Key,
|
|
body.UploadId,
|
|
body.Parts,
|
|
cancellationToken);
|
|
|
|
return new RouteHandlerResult { Handled = true, StatusCode = 200 };
|
|
}
|
|
|
|
private async Task<RouteHandlerResult> HandleAbortMultipartAsync(
|
|
HttpContext context,
|
|
IStorageBackend backend,
|
|
StorageLocation location,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var uploadId = context.Request.Query["uploadId"].ToString();
|
|
|
|
await backend.AbortMultipartUploadAsync(
|
|
location.Bucket,
|
|
location.Key,
|
|
uploadId,
|
|
cancellationToken);
|
|
|
|
return new RouteHandlerResult { Handled = true, StatusCode = 204 };
|
|
}
|
|
|
|
private async Task<RouteHandlerResult> HandleDeleteAsync(
|
|
HttpContext context,
|
|
IStorageBackend backend,
|
|
StorageLocation location,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await backend.DeleteObjectAsync(location.Bucket, location.Key, cancellationToken);
|
|
return new RouteHandlerResult { Handled = true, StatusCode = 204 };
|
|
}
|
|
}
|
|
|
|
internal class CompleteMultipartRequest
|
|
{
|
|
public string UploadId { get; set; } = "";
|
|
public List<UploadPart> Parts { get; set; } = new();
|
|
}
|
|
|
|
internal class StorageLocation
|
|
{
|
|
public string Backend { get; set; } = "";
|
|
public string Bucket { get; set; } = "";
|
|
public string Key { get; set; } = "";
|
|
public StorageAccessPolicy? Policy { get; set; }
|
|
}
|
|
```
|
|
|
|
---
|
|
|
|
## Storage Backend Interface
|
|
|
|
```csharp
|
|
namespace StellaOps.Router.Handlers.Storage;
|
|
|
|
public interface IStorageBackend
|
|
{
|
|
Task<ObjectMetadata> GetObjectMetadataAsync(
|
|
string bucket, string key, CancellationToken cancellationToken);
|
|
|
|
Task<Stream> GetObjectStreamAsync(
|
|
string bucket, string key, CancellationToken cancellationToken);
|
|
|
|
Task PutObjectAsync(
|
|
string bucket, string key, Stream content, long contentLength,
|
|
string contentType, Dictionary<string, string>? metadata,
|
|
CancellationToken cancellationToken);
|
|
|
|
Task DeleteObjectAsync(
|
|
string bucket, string key, CancellationToken cancellationToken);
|
|
|
|
Task<string> GetPresignedDownloadUrlAsync(
|
|
string bucket, string key, TimeSpan expiration,
|
|
CancellationToken cancellationToken);
|
|
|
|
Task<string> GetPresignedUploadUrlAsync(
|
|
string bucket, string key, string contentType, TimeSpan expiration,
|
|
CancellationToken cancellationToken);
|
|
|
|
Task<MultipartUploadInfo> InitiateMultipartUploadAsync(
|
|
string bucket, string key, string contentType,
|
|
CancellationToken cancellationToken);
|
|
|
|
Task CompleteMultipartUploadAsync(
|
|
string bucket, string key, string uploadId, List<UploadPart> parts,
|
|
CancellationToken cancellationToken);
|
|
|
|
Task AbortMultipartUploadAsync(
|
|
string bucket, string key, string uploadId,
|
|
CancellationToken cancellationToken);
|
|
}
|
|
|
|
public class ObjectMetadata
|
|
{
|
|
public string ContentType { get; set; } = "application/octet-stream";
|
|
public long ContentLength { get; set; }
|
|
public string? ETag { get; set; }
|
|
public DateTimeOffset LastModified { get; set; }
|
|
public Dictionary<string, string> CustomMetadata { get; set; } = new();
|
|
}
|
|
|
|
public class MultipartUploadInfo
|
|
{
|
|
public string UploadId { get; set; } = "";
|
|
public List<PresignedPartUrl> PresignedPartUrls { get; set; } = new();
|
|
}
|
|
|
|
public class PresignedPartUrl
|
|
{
|
|
public int PartNumber { get; set; }
|
|
public string Url { get; set; } = "";
|
|
}
|
|
|
|
public class UploadPart
|
|
{
|
|
public int PartNumber { get; set; }
|
|
public string ETag { get; set; } = "";
|
|
}
|
|
```
|
|
|
|
---
|
|
|
|
## S3 Backend Implementation
|
|
|
|
```csharp
|
|
namespace StellaOps.Router.Handlers.Storage;
|
|
|
|
public sealed class S3StorageBackend : IStorageBackend
|
|
{
|
|
private readonly IAmazonS3 _client;
|
|
private readonly ILogger<S3StorageBackend> _logger;
|
|
|
|
public S3StorageBackend(IAmazonS3 client, ILogger<S3StorageBackend> logger)
|
|
{
|
|
_client = client;
|
|
_logger = logger;
|
|
}
|
|
|
|
public async Task<ObjectMetadata> GetObjectMetadataAsync(
|
|
string bucket, string key, CancellationToken cancellationToken)
|
|
{
|
|
var response = await _client.GetObjectMetadataAsync(bucket, key, cancellationToken);
|
|
|
|
return new ObjectMetadata
|
|
{
|
|
ContentType = response.Headers.ContentType,
|
|
ContentLength = response.ContentLength,
|
|
ETag = response.ETag,
|
|
LastModified = response.LastModified,
|
|
CustomMetadata = response.Metadata.Keys
|
|
.ToDictionary(k => k, k => response.Metadata[k])
|
|
};
|
|
}
|
|
|
|
public async Task<Stream> GetObjectStreamAsync(
|
|
string bucket, string key, CancellationToken cancellationToken)
|
|
{
|
|
var response = await _client.GetObjectAsync(bucket, key, cancellationToken);
|
|
return response.ResponseStream;
|
|
}
|
|
|
|
public async Task PutObjectAsync(
|
|
string bucket, string key, Stream content, long contentLength,
|
|
string contentType, Dictionary<string, string>? metadata,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var request = new PutObjectRequest
|
|
{
|
|
BucketName = bucket,
|
|
Key = key,
|
|
InputStream = content,
|
|
ContentType = contentType
|
|
};
|
|
|
|
if (metadata != null)
|
|
{
|
|
foreach (var (k, v) in metadata)
|
|
{
|
|
request.Metadata.Add(k, v);
|
|
}
|
|
}
|
|
|
|
await _client.PutObjectAsync(request, cancellationToken);
|
|
}
|
|
|
|
public async Task DeleteObjectAsync(
|
|
string bucket, string key, CancellationToken cancellationToken)
|
|
{
|
|
await _client.DeleteObjectAsync(bucket, key, cancellationToken);
|
|
}
|
|
|
|
public Task<string> GetPresignedDownloadUrlAsync(
|
|
string bucket, string key, TimeSpan expiration,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var request = new GetPreSignedUrlRequest
|
|
{
|
|
BucketName = bucket,
|
|
Key = key,
|
|
Expires = DateTime.UtcNow.Add(expiration),
|
|
Verb = HttpVerb.GET
|
|
};
|
|
|
|
var url = _client.GetPreSignedURL(request);
|
|
return Task.FromResult(url);
|
|
}
|
|
|
|
public Task<string> GetPresignedUploadUrlAsync(
|
|
string bucket, string key, string contentType, TimeSpan expiration,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var request = new GetPreSignedUrlRequest
|
|
{
|
|
BucketName = bucket,
|
|
Key = key,
|
|
Expires = DateTime.UtcNow.Add(expiration),
|
|
Verb = HttpVerb.PUT,
|
|
ContentType = contentType
|
|
};
|
|
|
|
var url = _client.GetPreSignedURL(request);
|
|
return Task.FromResult(url);
|
|
}
|
|
|
|
public async Task<MultipartUploadInfo> InitiateMultipartUploadAsync(
|
|
string bucket, string key, string contentType,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var initResponse = await _client.InitiateMultipartUploadAsync(
|
|
bucket, key, cancellationToken);
|
|
|
|
// Generate presigned URLs for parts (assuming 100MB parts, 50 parts max)
|
|
var partUrls = new List<PresignedPartUrl>();
|
|
for (int i = 1; i <= 50; i++)
|
|
{
|
|
var url = _client.GetPreSignedURL(new GetPreSignedUrlRequest
|
|
{
|
|
BucketName = bucket,
|
|
Key = key,
|
|
Expires = DateTime.UtcNow.AddHours(24),
|
|
Verb = HttpVerb.PUT,
|
|
UploadId = initResponse.UploadId,
|
|
PartNumber = i
|
|
});
|
|
|
|
partUrls.Add(new PresignedPartUrl { PartNumber = i, Url = url });
|
|
}
|
|
|
|
return new MultipartUploadInfo
|
|
{
|
|
UploadId = initResponse.UploadId,
|
|
PresignedPartUrls = partUrls
|
|
};
|
|
}
|
|
|
|
public async Task CompleteMultipartUploadAsync(
|
|
string bucket, string key, string uploadId, List<UploadPart> parts,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
var request = new CompleteMultipartUploadRequest
|
|
{
|
|
BucketName = bucket,
|
|
Key = key,
|
|
UploadId = uploadId,
|
|
PartETags = parts.Select(p => new PartETag(p.PartNumber, p.ETag)).ToList()
|
|
};
|
|
|
|
await _client.CompleteMultipartUploadAsync(request, cancellationToken);
|
|
}
|
|
|
|
public async Task AbortMultipartUploadAsync(
|
|
string bucket, string key, string uploadId,
|
|
CancellationToken cancellationToken)
|
|
{
|
|
await _client.AbortMultipartUploadAsync(bucket, key, uploadId, cancellationToken);
|
|
}
|
|
}
|
|
```
|
|
|
|
---
|
|
|
|
## Access Control Evaluator
|
|
|
|
```csharp
|
|
namespace StellaOps.Router.Handlers.Storage;
|
|
|
|
public interface IAccessControlEvaluator
|
|
{
|
|
AccessResult Evaluate(
|
|
StorageLocation location,
|
|
IReadOnlyDictionary<string, string> claims,
|
|
string httpMethod);
|
|
}
|
|
|
|
public class AccessResult
|
|
{
|
|
public bool Allowed { get; set; }
|
|
public string? Reason { get; set; }
|
|
}
|
|
|
|
public sealed class ClaimBasedAccessControlEvaluator : IAccessControlEvaluator
|
|
{
|
|
public AccessResult Evaluate(
|
|
StorageLocation location,
|
|
IReadOnlyDictionary<string, string> claims,
|
|
string httpMethod)
|
|
{
|
|
var policy = location.Policy ?? new StorageAccessPolicy();
|
|
|
|
// Check authentication requirement
|
|
if (policy.RequireAuthentication && !claims.Any())
|
|
{
|
|
return new AccessResult { Allowed = false, Reason = "Authentication required" };
|
|
}
|
|
|
|
// Check allowed claims
|
|
if (policy.AllowedClaims.Any())
|
|
{
|
|
var hasRequiredClaim = policy.AllowedClaims.Any(c =>
|
|
{
|
|
var parts = c.Split('=', 2);
|
|
if (parts.Length == 2)
|
|
{
|
|
return claims.TryGetValue(parts[0], out var value) && value == parts[1];
|
|
}
|
|
return claims.ContainsKey(c);
|
|
});
|
|
|
|
if (!hasRequiredClaim)
|
|
{
|
|
return new AccessResult { Allowed = false, Reason = "Required claim not present" };
|
|
}
|
|
}
|
|
|
|
// Check ownership for write operations
|
|
if (policy.EnforceOwnership && IsWriteOperation(httpMethod))
|
|
{
|
|
if (string.IsNullOrEmpty(policy.OwnerClaimPath))
|
|
{
|
|
return new AccessResult { Allowed = false, Reason = "Owner claim path not configured" };
|
|
}
|
|
|
|
if (!claims.ContainsKey(policy.OwnerClaimPath))
|
|
{
|
|
return new AccessResult { Allowed = false, Reason = "Owner claim required" };
|
|
}
|
|
}
|
|
|
|
return new AccessResult { Allowed = true };
|
|
}
|
|
|
|
private bool IsWriteOperation(string method)
|
|
{
|
|
return method.ToUpper() is "PUT" or "POST" or "DELETE" or "PATCH";
|
|
}
|
|
}
|
|
```
|
|
|
|
---
|
|
|
|
## YAML Configuration
|
|
|
|
```yaml
|
|
Storage:
|
|
PathPrefix: "/files"
|
|
DefaultBackend: "s3"
|
|
MaxUploadSize: 5368709120 # 5GB
|
|
MultipartThreshold: 104857600 # 100MB
|
|
PresignedUrlExpiration: "01:00:00"
|
|
UsePresignedUploads: true
|
|
UsePresignedDownloads: true
|
|
|
|
Backends:
|
|
s3:
|
|
Type: "S3"
|
|
Endpoint: "https://s3.amazonaws.com"
|
|
Region: "us-east-1"
|
|
AccessKey: "${AWS_ACCESS_KEY}"
|
|
SecretKey: "${AWS_SECRET_KEY}"
|
|
minio:
|
|
Type: "S3"
|
|
Endpoint: "https://minio.internal:9000"
|
|
Region: "us-east-1"
|
|
AccessKey: "${MINIO_ACCESS_KEY}"
|
|
SecretKey: "${MINIO_SECRET_KEY}"
|
|
UsePathStyle: true
|
|
|
|
BucketMappings:
|
|
- PathPattern: "uploads/*"
|
|
Bucket: "user-uploads"
|
|
KeyPrefix: "files/"
|
|
Backend: "s3"
|
|
Policy:
|
|
RequireAuthentication: true
|
|
EnforceOwnership: true
|
|
OwnerClaimPath: "sub"
|
|
|
|
- PathPattern: "public/*"
|
|
Bucket: "public-assets"
|
|
Backend: "s3"
|
|
Policy:
|
|
RequireAuthentication: false
|
|
```
|
|
|
|
---
|
|
|
|
## Deliverables
|
|
|
|
1. `StellaOps.Router.Handlers.Storage/StorageHandler.cs`
|
|
2. `StellaOps.Router.Handlers.Storage/StorageHandlerConfig.cs`
|
|
3. `StellaOps.Router.Handlers.Storage/IStorageBackend.cs`
|
|
4. `StellaOps.Router.Handlers.Storage/S3StorageBackend.cs`
|
|
5. `StellaOps.Router.Handlers.Storage/IAccessControlEvaluator.cs`
|
|
6. `StellaOps.Router.Handlers.Storage/ClaimBasedAccessControlEvaluator.cs`
|
|
7. `StellaOps.Router.Handlers.Storage/StorageBackendFactory.cs`
|
|
8. Presigned URL generation tests
|
|
9. Multipart upload tests
|
|
10. Access control tests
|
|
|
|
---
|
|
|
|
## Next Step
|
|
|
|
Proceed to [Step 18: Reverse Proxy Handler Implementation](18-Step.md) to implement direct reverse proxy routing.
|