feat: Add RustFS artifact object store and migration tool

- Implemented RustFsArtifactObjectStore for managing artifacts in RustFS. - Added unit tests for RustFsArtifactObjectStore functionality. - Created a RustFS migrator tool to transfer objects from S3 to RustFS. - Introduced policy preview and report models for API integration. - Added fixtures and tests for policy preview and report functionality. - Included necessary metadata and scripts for cache_pkg package.
2025-10-23 18:53:18 +03:00
parent aaa5fbfb78
commit f4d7a15a00
117 changed files with 4849 additions and 725 deletions
--- a/deploy/compose/docker-compose.airgap.yaml
+++ b/deploy/compose/docker-compose.airgap.yaml
@@ -7,11 +7,12 @@ networks:
  stellaops:
    driver: bridge

-volumes:
-  mongo-data:
-  minio-data:
-  concelier-jobs:
-  nats-data:
+volumes:
+  mongo-data:
+  minio-data:
+  rustfs-data:
+  concelier-jobs:
+  nats-data:

 services:
  mongo:
@@ -27,8 +28,8 @@ services:
      - stellaops
    labels: *release-labels

-  minio:
-    image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
+  minio:
+    image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
    command: ["server", "/data", "--console-address", ":9001"]
    restart: unless-stopped
    environment:
@@ -40,7 +41,22 @@ services:
      - "${MINIO_CONSOLE_PORT:-29001}:9001"
    networks:
      - stellaops
-    labels: *release-labels
+    labels: *release-labels
+
+  rustfs:
+    image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge
+    command: ["serve", "--listen", "0.0.0.0:8080", "--root", "/data"]
+    restart: unless-stopped
+    environment:
+      RUSTFS__LOG__LEVEL: info
+      RUSTFS__STORAGE__PATH: /data
+    volumes:
+      - rustfs-data:/data
+    ports:
+      - "${RUSTFS_HTTP_PORT:-8080}:8080"
+    networks:
+      - stellaops
+    labels: *release-labels

  nats:
    image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e
@@ -127,18 +143,19 @@ services:
      - stellaops
    labels: *release-labels

-  scanner-web:
+  scanner-web:
    image: registry.stella-ops.org/stellaops/scanner-web@sha256:3df8ca21878126758203c1a0444e39fd97f77ddacf04a69685cda9f1e5e94718
    restart: unless-stopped
-    depends_on:
-      - concelier
-      - minio
-      - nats
-    environment:
-      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
-      SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000"
-      SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}"
-      SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}"
+    depends_on:
+      - concelier
+      - rustfs
+      - nats
+    environment:
+      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
+      SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1"
+      SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
+      SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
      SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-false}"
      SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}"
@@ -152,17 +169,19 @@ services:
      - stellaops
    labels: *release-labels

-  scanner-worker:
+  scanner-worker:
    image: registry.stella-ops.org/stellaops/scanner-worker@sha256:eea5d6cfe7835950c5ec7a735a651f2f0d727d3e470cf9027a4a402ea89c4fb5
    restart: unless-stopped
-    depends_on:
-      - scanner-web
-      - nats
-    environment:
-      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
-      SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000"
-      SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}"
-      SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}"
+    depends_on:
+      - scanner-web
+      - rustfs
+      - nats
+    environment:
+      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
+      SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1"
+      SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
+      SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
    networks:
      - stellaops
--- a/deploy/compose/docker-compose.dev.yaml
+++ b/deploy/compose/docker-compose.dev.yaml
@@ -7,11 +7,12 @@ networks:
  stellaops:
    driver: bridge

-volumes:
-  mongo-data:
-  minio-data:
-  concelier-jobs:
-  nats-data:
+volumes:
+  mongo-data:
+  minio-data:
+  rustfs-data:
+  concelier-jobs:
+  nats-data:

 services:
  mongo:
@@ -27,9 +28,9 @@ services:
      - stellaops
    labels: *release-labels

-  minio:
-    image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
-    command: ["server", "/data", "--console-address", ":9001"]
+  minio:
+    image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
+    command: ["server", "/data", "--console-address", ":9001"]
    restart: unless-stopped
    environment:
      MINIO_ROOT_USER: "${MINIO_ROOT_USER}"
@@ -38,9 +39,24 @@ services:
      - minio-data:/data
    ports:
      - "${MINIO_CONSOLE_PORT:-9001}:9001"
-    networks:
-      - stellaops
-    labels: *release-labels
+    networks:
+      - stellaops
+    labels: *release-labels
+
+  rustfs:
+    image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge
+    command: ["serve", "--listen", "0.0.0.0:8080", "--root", "/data"]
+    restart: unless-stopped
+    environment:
+      RUSTFS__LOG__LEVEL: info
+      RUSTFS__STORAGE__PATH: /data
+    volumes:
+      - rustfs-data:/data
+    ports:
+      - "${RUSTFS_HTTP_PORT:-8080}:8080"
+    networks:
+      - stellaops
+    labels: *release-labels

  nats:
    image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e
@@ -125,18 +141,19 @@ services:
      - stellaops
    labels: *release-labels

-  scanner-web:
+  scanner-web:
    image: registry.stella-ops.org/stellaops/scanner-web@sha256:e0dfdb087e330585a5953029fb4757f5abdf7610820a085bd61b457dbead9a11
    restart: unless-stopped
-    depends_on:
-      - concelier
-      - minio
-      - nats
-    environment:
-      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
-      SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000"
-      SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}"
-      SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}"
+    depends_on:
+      - concelier
+      - rustfs
+      - nats
+    environment:
+      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
+      SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1"
+      SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
+      SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
      SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-false}"
      SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}"
@@ -150,18 +167,20 @@ services:
      - stellaops
    labels: *release-labels

-  scanner-worker:
+  scanner-worker:
    image: registry.stella-ops.org/stellaops/scanner-worker@sha256:92dda42f6f64b2d9522104a5c9ffb61d37b34dd193132b68457a259748008f37
    restart: unless-stopped
-    depends_on:
-      - scanner-web
-      - nats
-    environment:
-      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
-      SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000"
-      SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}"
-      SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}"
-      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
+    depends_on:
+      - scanner-web
+      - rustfs
+      - nats
+    environment:
+      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
+      SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1"
+      SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
+      SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
+      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
    networks:
      - stellaops
    labels: *release-labels
--- a/deploy/compose/docker-compose.stage.yaml
+++ b/deploy/compose/docker-compose.stage.yaml
@@ -7,11 +7,12 @@ networks:
  stellaops:
    driver: bridge

-volumes:
-  mongo-data:
-  minio-data:
-  concelier-jobs:
-  nats-data:
+volumes:
+  mongo-data:
+  minio-data:
+  rustfs-data:
+  concelier-jobs:
+  nats-data:

 services:
  mongo:
@@ -27,8 +28,8 @@ services:
      - stellaops
    labels: *release-labels

-  minio:
-    image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
+  minio:
+    image: docker.io/minio/minio@sha256:14cea493d9a34af32f524e538b8346cf79f3321eff8e708c1e2960462bd8936e
    command: ["server", "/data", "--console-address", ":9001"]
    restart: unless-stopped
    environment:
@@ -40,7 +41,22 @@ services:
      - "${MINIO_CONSOLE_PORT:-9001}:9001"
    networks:
      - stellaops
-    labels: *release-labels
+    labels: *release-labels
+
+  rustfs:
+    image: registry.stella-ops.org/stellaops/rustfs:2025.10.0-edge
+    command: ["serve", "--listen", "0.0.0.0:8080", "--root", "/data"]
+    restart: unless-stopped
+    environment:
+      RUSTFS__LOG__LEVEL: info
+      RUSTFS__STORAGE__PATH: /data
+    volumes:
+      - rustfs-data:/data
+    ports:
+      - "${RUSTFS_HTTP_PORT:-8080}:8080"
+    networks:
+      - stellaops
+    labels: *release-labels

  nats:
    image: docker.io/library/nats@sha256:c82559e4476289481a8a5196e675ebfe67eea81d95e5161e3e78eccfe766608e
@@ -125,18 +141,19 @@ services:
      - stellaops
    labels: *release-labels

-  scanner-web:
+  scanner-web:
    image: registry.stella-ops.org/stellaops/scanner-web@sha256:14b23448c3f9586a9156370b3e8c1991b61907efa666ca37dd3aaed1e79fe3b7
    restart: unless-stopped
-    depends_on:
-      - concelier
-      - minio
-      - nats
-    environment:
-      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
-      SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000"
-      SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}"
-      SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}"
+    depends_on:
+      - concelier
+      - rustfs
+      - nats
+    environment:
+      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
+      SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1"
+      SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
+      SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
      SCANNER__EVENTS__ENABLED: "${SCANNER_EVENTS_ENABLED:-false}"
      SCANNER__EVENTS__DRIVER: "${SCANNER_EVENTS_DRIVER:-redis}"
@@ -150,17 +167,19 @@ services:
      - stellaops
    labels: *release-labels

-  scanner-worker:
+  scanner-worker:
    image: registry.stella-ops.org/stellaops/scanner-worker@sha256:32e25e76386eb9ea8bee0a1ad546775db9a2df989fab61ac877e351881960dab
    restart: unless-stopped
-    depends_on:
-      - scanner-web
-      - nats
-    environment:
-      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
-      SCANNER__STORAGE__S3__ENDPOINT: "http://minio:9000"
-      SCANNER__STORAGE__S3__ACCESSKEYID: "${MINIO_ROOT_USER}"
-      SCANNER__STORAGE__S3__SECRETACCESSKEY: "${MINIO_ROOT_PASSWORD}"
+    depends_on:
+      - scanner-web
+      - rustfs
+      - nats
+    environment:
+      SCANNER__STORAGE__MONGO__CONNECTIONSTRING: "mongodb://${MONGO_INITDB_ROOT_USERNAME}:${MONGO_INITDB_ROOT_PASSWORD}@mongo:27017"
+      SCANNER__ARTIFACTSTORE__DRIVER: "rustfs"
+      SCANNER__ARTIFACTSTORE__ENDPOINT: "http://rustfs:8080/api/v1"
+      SCANNER__ARTIFACTSTORE__BUCKET: "scanner-artifacts"
+      SCANNER__ARTIFACTSTORE__TIMEOUTSECONDS: "30"
      SCANNER__QUEUE__BROKER: "${SCANNER_QUEUE_BROKER}"
    networks:
      - stellaops
--- a/deploy/compose/env/airgap.env.example
+++ b/deploy/compose/env/airgap.env.example
@@ -2,9 +2,10 @@
 MONGO_INITDB_ROOT_USERNAME=stellaops
 MONGO_INITDB_ROOT_PASSWORD=airgap-password
 MINIO_ROOT_USER=stellaops-offline
-MINIO_ROOT_PASSWORD=airgap-minio-secret
-MINIO_CONSOLE_PORT=29001
-AUTHORITY_ISSUER=https://authority.airgap.local
+MINIO_ROOT_PASSWORD=airgap-minio-secret
+MINIO_CONSOLE_PORT=29001
+RUSTFS_HTTP_PORT=8080
+AUTHORITY_ISSUER=https://authority.airgap.local
 AUTHORITY_PORT=8440
 SIGNER_POE_INTROSPECT_URL=file:///offline/poe/introspect.json
 SIGNER_PORT=8441
--- a/deploy/compose/env/dev.env.example
+++ b/deploy/compose/env/dev.env.example
@@ -3,8 +3,9 @@ MONGO_INITDB_ROOT_USERNAME=stellaops
 MONGO_INITDB_ROOT_PASSWORD=dev-password
 MINIO_ROOT_USER=stellaops
 MINIO_ROOT_PASSWORD=dev-minio-secret
-MINIO_CONSOLE_PORT=9001
-AUTHORITY_ISSUER=https://authority.localtest.me
+MINIO_CONSOLE_PORT=9001
+RUSTFS_HTTP_PORT=8080
+AUTHORITY_ISSUER=https://authority.localtest.me
 AUTHORITY_PORT=8440
 SIGNER_POE_INTROSPECT_URL=https://licensing.svc.local/introspect
 SIGNER_PORT=8441
--- a/deploy/compose/env/mirror.env.example
+++ b/deploy/compose/env/mirror.env.example
@@ -3,8 +3,9 @@
 # Core infrastructure credentials
 MONGO_INITDB_ROOT_USERNAME=stellaops_mirror
 MONGO_INITDB_ROOT_PASSWORD=mirror-password
-MINIO_ROOT_USER=stellaops-mirror
-MINIO_ROOT_PASSWORD=mirror-minio-secret
+MINIO_ROOT_USER=stellaops-mirror
+MINIO_ROOT_PASSWORD=mirror-minio-secret
+RUSTFS_HTTP_PORT=8080

 # Mirror HTTP listeners
 MIRROR_GATEWAY_HTTP_PORT=8080
--- a/deploy/compose/env/stage.env.example
+++ b/deploy/compose/env/stage.env.example
@@ -2,9 +2,10 @@
 MONGO_INITDB_ROOT_USERNAME=stellaops
 MONGO_INITDB_ROOT_PASSWORD=stage-password
 MINIO_ROOT_USER=stellaops-stage
-MINIO_ROOT_PASSWORD=stage-minio-secret
-MINIO_CONSOLE_PORT=19001
-AUTHORITY_ISSUER=https://authority.stage.stella-ops.internal
+MINIO_ROOT_PASSWORD=stage-minio-secret
+MINIO_CONSOLE_PORT=19001
+RUSTFS_HTTP_PORT=8080
+AUTHORITY_ISSUER=https://authority.stage.stella-ops.internal
 AUTHORITY_PORT=8440
 SIGNER_POE_INTROSPECT_URL=https://licensing.stage.stella-ops.internal/introspect
 SIGNER_PORT=8441