feat: Add RustFS artifact object store and migration tool
Some checks failed
Docs CI / lint-and-preview (push) Has been cancelled

- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS.
- Added unit tests for RustFsArtifactObjectStore functionality.
- Created a RustFS migrator tool to transfer objects from S3 to RustFS.
- Introduced policy preview and report models for API integration.
- Added fixtures and tests for policy preview and report functionality.
- Included necessary metadata and scripts for cache_pkg package.
This commit is contained in:
Vladimir Moushkov
2025-10-23 18:53:18 +03:00
parent aaa5fbfb78
commit f4d7a15a00
117 changed files with 4849 additions and 725 deletions

View File

@@ -93,15 +93,16 @@ services:
volumeClaims:
- name: concelier-jobs
claimName: stellaops-concelier-jobs
scanner-web:
scanner-web:
image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718
service:
port: 8444
env:
SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017"
SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000"
SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap"
SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret"
SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1"
SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222"
SCANNER__EVENTS__ENABLED: "false"
SCANNER__EVENTS__DRIVER: "redis"
@@ -109,13 +110,14 @@ services:
SCANNER__EVENTS__STREAM: "stella.events"
SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5"
SCANNER__EVENTS__MAXSTREAMLENGTH: "10000"
scanner-worker:
scanner-worker:
image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5
env:
SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-airgap:stellaops-airgap@stellaops-mongo:27017"
SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000"
SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-airgap"
SCANNER__STORAGE__S3__SECRETACCESSKEY: "airgap-minio-secret"
SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1"
SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222"
SCANNER__EVENTS__ENABLED: "false"
SCANNER__EVENTS__DRIVER: "redis"
@@ -163,11 +165,11 @@ services:
volumeClaims:
- name: mongo-data
claimName: stellaops-mongo-data
minio:
class: infrastructure
image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
service:
port: 9000
minio:
class: infrastructure
image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
service:
port: 9000
command:
- server
- /data
@@ -179,9 +181,29 @@ services:
volumeMounts:
- name: minio-data
mountPath: /data
volumeClaims:
- name: minio-data
claimName: stellaops-minio-data
volumeClaims:
- name: minio-data
claimName: stellaops-minio-data
rustfs:
class: infrastructure
image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge
service:
port: 8080
command:
- serve
- --listen
- 0.0.0.0:8080
- --root
- /data
env:
RUSTFS__LOG__LEVEL: info
RUSTFS__STORAGE__PATH: /data
volumeMounts:
- name: rustfs-data
mountPath: /data
volumeClaims:
- name: rustfs-data
claimName: stellaops-rustfs-data
nats:
class: infrastructure
image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e

View File

@@ -51,7 +51,7 @@ configMaps:
telemetry:
enableRequestLogging: true
minimumLogLevel: Debug
services:
services:
authority:
image: registry.stella-ops.org/stellaops/authority@sha256:a8e8faec44a579aa5714e58be835f25575710430b1ad2ccd1282a018cd9ffcdd
service:
@@ -92,15 +92,16 @@ services:
volumes:
- name: concelier-jobs
emptyDir: {}
scanner-web:
scanner-web:
image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11
service:
port: 8444
env:
SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017"
SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000"
SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops"
SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret"
SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1"
SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222"
SCANNER__EVENTS__ENABLED: "false"
SCANNER__EVENTS__DRIVER: "redis"
@@ -108,13 +109,14 @@ services:
SCANNER__EVENTS__STREAM: "stella.events"
SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5"
SCANNER__EVENTS__MAXSTREAMLENGTH: "10000"
scanner-worker:
scanner-worker:
image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37
env:
SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops:stellaops@stellaops-mongo:27017"
SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000"
SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops"
SCANNER__STORAGE__S3__SECRETACCESSKEY: "dev-minio-secret"
SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1"
SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222"
SCANNER__EVENTS__ENABLED: "false"
SCANNER__EVENTS__DRIVER: "redis"
@@ -161,9 +163,9 @@ services:
volumes:
- name: mongo-data
emptyDir: {}
minio:
class: infrastructure
image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
minio:
class: infrastructure
image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
service:
port: 9000
command:
@@ -177,9 +179,23 @@ services:
volumeMounts:
- name: minio-data
mountPath: /data
volumes:
- name: minio-data
emptyDir: {}
volumes:
- name: minio-data
emptyDir: {}
rustfs:
class: infrastructure
image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge
service:
port: 8080
env:
RUSTFS__LOG__LEVEL: info
RUSTFS__STORAGE__PATH: /data
volumeMounts:
- name: rustfs-data
mountPath: /data
volumes:
- name: rustfs-data
emptyDir: {}
nats:
class: infrastructure
image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e

View File

@@ -92,15 +92,16 @@ services:
volumeClaims:
- name: concelier-jobs
claimName: stellaops-concelier-jobs
scanner-web:
scanner-web:
image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7
service:
port: 8444
env:
SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017"
SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000"
SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage"
SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret"
SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1"
SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222"
SCANNER__EVENTS__ENABLED: "false"
SCANNER__EVENTS__DRIVER: "redis"
@@ -108,14 +109,15 @@ services:
SCANNER__EVENTS__STREAM: "stella.events"
SCANNER__EVENTS__PUBLISHTIMEOUTSECONDS: "5"
SCANNER__EVENTS__MAXSTREAMLENGTH: "10000"
scanner-worker:
scanner-worker:
image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab
replicas: 2
env:
SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://stellaops-stage:stellaops-stage@stellaops-mongo:27017"
SCANNER__STORAGE__S3__ENDPOINT: "http://stellaops-minio:9000"
SCANNER__STORAGE__S3__ACCESSKEYID: "stellaops-stage"
SCANNER__STORAGE__S3__SECRETACCESSKEY: "stage-minio-secret"
SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
SCANNER__ARTIFACTSTORE__ENDPOINT: "http://stellaops-rustfs:8080/api/v1"
SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
SCANNER__QUEUE__BROKER: "nats://stellaops-nats:4222"
SCANNER__EVENTS__ENABLED: "false"
SCANNER__EVENTS__DRIVER: "redis"
@@ -162,11 +164,11 @@ services:
volumeClaims:
- name: mongo-data
claimName: stellaops-mongo-data
minio:
class: infrastructure
image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
service:
port: 9000
minio:
class: infrastructure
image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
service:
port: 9000
command:
- server
- /data
@@ -178,9 +180,29 @@ services:
volumeMounts:
- name: minio-data
mountPath: /data
volumeClaims:
- name: minio-data
claimName: stellaops-minio-data
volumeClaims:
- name: minio-data
claimName: stellaops-minio-data
rustfs:
class: infrastructure
image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge
service:
port: 8080
command:
- serve
- --listen
- 0.0.0.0:8080
- --root
- /data
env:
RUSTFS__LOG__LEVEL: info
RUSTFS__STORAGE__PATH: /data
volumeMounts:
- name: rustfs-data
mountPath: /data
volumeClaims:
- name: rustfs-data
claimName: stellaops-rustfs-data
nats:
class: infrastructure
image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e