feat: extend retention policies

artifacthub/Dockerfile

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.24
+ARG GO_VERSION=1.25
 ARG UBUNTU_VERSION=22.04
 ARG BUILDER_IMAGE="golang:${GO_VERSION}"
 ARG RUNNER_IMAGE="ubuntu:${UBUNTU_VERSION}"
@@ -30,13 +30,19 @@ WORKDIR /app
 FROM base AS dev
 
 WORKDIR /tmp
-RUN curl -sL https://github.com/google/protobuf/releases/download/v3.20.0/protoc-3.20.0-linux-x86_64.zip -o protoc && \
+ARG TARGETARCH
+RUN case "${TARGETARCH}" in \
+  amd64) PROTOC_ARCH="linux-x86_64" ;; \
+  arm64) PROTOC_ARCH="linux-aarch_64" ;; \
+  *) echo "unsupported TARGETARCH=${TARGETARCH}" && exit 1 ;; \
+  esac && \
+  curl -sL "https://github.com/google/protobuf/releases/download/v3.20.0/protoc-3.20.0-${PROTOC_ARCH}.zip" -o protoc && \
   unzip protoc && \
   mv bin/protoc /usr/local/bin/protoc
 
 WORKDIR /app
 RUN go install github.com/mgechev/revive@v1.7.0
-RUN go install gotest.tools/gotestsum@v1.12.1
+RUN go install gotest.tools/gotestsum@v1.13.0
 RUN go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.36.10
 RUN go install google.golang.org/grpc/cmd/protoc-gen-go-grpc@v1.5.1
 
@@ -46,7 +52,8 @@ FROM base AS builder
 
 RUN rm -rf build && go build -o build/${APP_NAME}/server cmd/server/main.go && \
   go build -o build/${APP_NAME}/client cmd/client/main.go && \
-  go build -o build/${APP_NAME}/healthcheck cmd/healthcheck/healthcheck.go
+  go build -o build/${APP_NAME}/healthcheck cmd/healthcheck/healthcheck.go && \
+  go build -o build/${APP_NAME}/migrate-lifecycle cmd/migrate-lifecycle/main.go
 
 FROM ${RUNNER_IMAGE} AS runner
 

artifacthub/Makefile

@@ -11,28 +11,28 @@ pb.gen:
 	git clone git@github.com:semaphoreci/api.git tmp/public_api
 	(git clone git@github.com:renderedtext/internal_api.git tmp/internal_api && cd tmp/internal_api && git checkout $(INTERNAL_API_BRANCH) && cd -)
 	chmod -R 777 tmp
-	docker-compose run --rm app bash script/gen-apis
+	docker compose run --rm app bash script/gen-apis
 
 tidy:
-	docker-compose run --rm app go mod tidy
+	docker compose run --rm app go mod tidy
 
 bash:
-	docker-compose run --rm app bash
+	docker compose run --rm app bash
 
 lint:
-	docker-compose run --rm --no-deps app revive -formatter friendly -config lint.toml ./...
+	docker compose run --rm --no-deps app revive -formatter friendly -config lint.toml ./...
 
 test.setup:
-	docker-compose build
-	docker-compose run app go get ./...
+	docker compose build
+	docker compose run app go get ./...
 	-$(MAKE) db.test.create
 	$(MAKE) db.migrate
 
 test:
-	docker-compose run --rm app gotestsum --format short-verbose --junitfile out/test-reports.xml --packages="./..." -- -p 1
+	docker compose run --rm app gotestsum --format short-verbose --junitfile out/test-reports.xml --packages="./..." -- -p 1
 
 test.watch:
-	docker-compose run --rm app gotestsum --watch --format short-verbose --junitfile out/test-reports.xml --packages="./..." -- -p 1
+	docker compose run --rm app gotestsum --watch --format short-verbose --junitfile out/test-reports.xml --packages="./..." -- -p 1
 
 #
 # Database ops
@@ -42,22 +42,22 @@ DB_NAME=artifacthub
 DB_PASSWORD=the-cake-is-a-lie
 
 db.test.create:
-	docker-compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app createdb -h db -p 5432 -U postgres $(DB_NAME)
-	docker-compose run -e PGPASSWORD=$(DB_PASSWORD) app psql -h db -p 5432 -U postgres $(DB_NAME) -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";'
+	docker compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app createdb -h db -p 5432 -U postgres $(DB_NAME)
+	docker compose run -e PGPASSWORD=$(DB_PASSWORD) app psql -h db -p 5432 -U postgres $(DB_NAME) -c 'CREATE EXTENSION IF NOT EXISTS "uuid-ossp";'
 
 db.migration.create:
-	docker-compose run app mkdir -p db/migrations
-	docker-compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app migrate create -ext sql -dir db/migrations $(NAME)
+	docker compose run app mkdir -p db/migrations
+	docker compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app migrate create -ext sql -dir db/migrations $(NAME)
 	ls -lah db/migrations/*$(NAME)*
 
 db.migrate:
 	rm -f db/structure.sql
-	docker-compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app migrate -source file://db/migrations -database postgres://postgres:$(DB_PASSWORD)@db:5432/$(DB_NAME)?sslmode=disable up
-	docker-compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app bash -c "pg_dump --schema-only --no-privileges --no-owner -h db -p 5432 -U postgres -d $(DB_NAME)" > db/structure.sql
-	docker-compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app bash -c "pg_dump --data-only --table schema_migrations -h db -p 5432 -U postgres -d $(DB_NAME)" >> db/structure.sql
+	docker compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app migrate -source file://db/migrations -database postgres://postgres:$(DB_PASSWORD)@db:5432/$(DB_NAME)?sslmode=disable up
+	docker compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app bash -c "pg_dump --schema-only --no-privileges --no-owner -h db -p 5432 -U postgres -d $(DB_NAME)" > db/structure.sql
+	docker compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app bash -c "pg_dump --data-only --table schema_migrations -h db -p 5432 -U postgres -d $(DB_NAME)" >> db/structure.sql
 
 db.test.delete:
-	docker-compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app dropdb -h db -p 5432 -U postgres $(DB_NAME)
+	docker compose run --rm -e PGPASSWORD=$(DB_PASSWORD) app dropdb -h db -p 5432 -U postgres $(DB_NAME)
 
 db.test.shell:
-	docker-compose exec -u postgres db sh -c 'psql artifacthub'
+	docker compose exec -u postgres db sh -c 'psql artifacthub'

artifacthub/cmd/migrate-lifecycle/main.go

@@ -0,0 +1,80 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"os"
+
+	gcsstorage "cloud.google.com/go/storage"
+	"github.com/semaphoreio/semaphore/artifacthub/pkg/db"
+	"github.com/semaphoreio/semaphore/artifacthub/pkg/models"
+	"github.com/semaphoreio/semaphore/artifacthub/pkg/storage"
+)
+
+func main() {
+	if os.Getenv("DB_HOST") == "" {
+		fmt.Println("DB_HOST is not set. Required env vars: DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD")
+		os.Exit(1)
+	}
+
+	sqlDB, err := db.Conn().DB()
+	if err != nil {
+		fmt.Printf("Failed to get database connection: %v\n", err)
+		os.Exit(1)
+	}
+	if err = sqlDB.Ping(); err != nil {
+		fmt.Printf("Failed to connect to database: %v\n", err)
+		os.Exit(1)
+	}
+
+	gcsClient, err := storage.NewGcsClient(os.Getenv("GOOGLE_APPLICATION_CREDENTIALS"))
+	if err != nil {
+		fmt.Printf("Failed to create GCS client: %v\n", err)
+		os.Exit(1)
+	}
+
+	lifecycle := storage.ArtifactLifecycle()
+	if len(lifecycle.Rules) == 0 {
+		fmt.Println("No lifecycle rules configured. Set ARTIFACT_*_RETENTION_DAYS env vars.")
+		os.Exit(1)
+	}
+
+	fmt.Printf("Lifecycle rules to apply (%d rules):\n", len(lifecycle.Rules))
+	for _, r := range lifecycle.Rules {
+		fmt.Printf("  - Delete objects with prefix %v after %d days\n", r.Condition.MatchesPrefix, r.Condition.AgeInDays)
+	}
+	fmt.Println()
+
+	var total, updated, failed int
+
+	err = models.IterAllBuckets(func(bucketName string) {
+		total++
+		ctx := context.Background()
+
+		_, err := gcsClient.Client.Bucket(bucketName).Update(ctx, gcsstorage.BucketAttrsToUpdate{
+			Lifecycle: &lifecycle,
+		})
+
+		if err != nil {
+			failed++
+			fmt.Printf("FAIL %s: %v\n", bucketName, err)
+			return
+		}
+
+		updated++
+		if updated%100 == 0 {
+			fmt.Printf("... updated %d buckets so far\n", updated)
+		}
+	})
+
+	if err != nil {
+		fmt.Printf("Database iteration error: %v\n", err)
+		os.Exit(1)
+	}
+
+	fmt.Printf("\nDone. Total: %d, Updated: %d, Failed: %d\n", total, updated, failed)
+
+	if failed > 0 {
+		os.Exit(1)
+	}
+}

artifacthub/cmd/server/main.go

@@ -15,6 +15,8 @@ import (
 	"github.com/semaphoreio/semaphore/artifacthub/pkg/util/log"
 	"github.com/semaphoreio/semaphore/artifacthub/pkg/workers/bucketcleaner"
 	"github.com/semaphoreio/semaphore/artifacthub/pkg/workers/jobdeletion"
+	"github.com/semaphoreio/semaphore/artifacthub/pkg/workers/pipelinedeletion"
+	"github.com/semaphoreio/semaphore/artifacthub/pkg/workers/workflowdeletion"
 	"go.uber.org/zap"
 )
 
@@ -139,6 +141,26 @@ func jobDeletionWorker(client storage.Client) {
 	}
 }
 
+func pipelineDeletionWorker(client storage.Client) {
+	log.Info("Starting pipeline deletion worker...")
+	worker, err := pipelinedeletion.NewWorker(amqpURL, client)
+	if err != nil {
+		panic(err)
+	}
+
+	worker.Start()
+}
+
+func workflowDeletionWorker(client storage.Client) {
+	log.Info("Starting workflow deletion worker...")
+	worker, err := workflowdeletion.NewWorker(amqpURL, client)
+	if err != nil {
+		panic(err)
+	}
+
+	worker.Start()
+}
+
 func main() {
 	flag.Parse()
 
@@ -183,6 +205,14 @@ func main() {
 		go jobDeletionWorker(storageClient)
 	}
 
+	if os.Getenv("START_PIPELINE_DELETION_WORKER") == "yes" {
+		go pipelineDeletionWorker(storageClient)
+	}
+
+	if os.Getenv("START_WORKFLOW_DELETION_WORKER") == "yes" {
+		go workflowDeletionWorker(storageClient)
+	}
+
 	select {}
 }
 

artifacthub/db/structure.sql

@@ -2,12 +2,15 @@
 -- PostgreSQL database dump
 --
 
+\restrict 2yk8NS74NJDI9lHS5sKknTp3LxCeX71pvR1Lfirp9e6vLPRIlfrcTFsjvlZJliA
+
 -- Dumped from database version 9.6.24
--- Dumped by pg_dump version 15.12 (Debian 15.12-0+deb12u2)
+-- Dumped by pg_dump version 17.9 (Debian 17.9-0+deb13u1)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
+SET transaction_timeout = 0;
 SET client_encoding = 'UTF8';
 SET standard_conforming_strings = on;
 SELECT pg_catalog.set_config('search_path', '', false);
@@ -128,16 +131,21 @@ ALTER TABLE ONLY public.retention_policies
 -- PostgreSQL database dump complete
 --
 
+\unrestrict 2yk8NS74NJDI9lHS5sKknTp3LxCeX71pvR1Lfirp9e6vLPRIlfrcTFsjvlZJliA
+
 --
 -- PostgreSQL database dump
 --
 
+\restrict CA3CH7umkPUddVI7VgCuICCK4icBQ3R5FHkge4PNCxhgiOZxVp4Ne9f9Byf3DJ8
+
 -- Dumped from database version 9.6.24
--- Dumped by pg_dump version 15.12 (Debian 15.12-0+deb12u2)
+-- Dumped by pg_dump version 17.9 (Debian 17.9-0+deb13u1)
 
 SET statement_timeout = 0;
 SET lock_timeout = 0;
 SET idle_in_transaction_session_timeout = 0;
+SET transaction_timeout = 0;
 SET client_encoding = 'UTF8';
 SET standard_conforming_strings = on;
 SELECT pg_catalog.set_config('search_path', '', false);
@@ -159,3 +167,5 @@ COPY public.schema_migrations (version, dirty) FROM stdin;
 -- PostgreSQL database dump complete
 --
 
+\unrestrict CA3CH7umkPUddVI7VgCuICCK4icBQ3R5FHkge4PNCxhgiOZxVp4Ne9f9Byf3DJ8
+

artifacthub/docker-compose.yml

@@ -1,7 +1,6 @@
-version: '3.6'
+version: "3.6"
 
 services:
-
   app:
     build:
       context: .
@@ -58,7 +57,7 @@ services:
 
   gcs:
     image: fsouza/fake-gcs-server
-    container_name: 'gcs'
+    container_name: "gcs"
     volumes:
       - gcs-data:/data
     ports:
@@ -67,7 +66,7 @@ services:
 
   s3:
     image: minio/minio:RELEASE.2021-04-22T15-44-28Z.hotfix.56647434e
-    container_name: 's3'
+    container_name: "s3"
     ports:
       - 9000:9000
     entrypoint: sh

artifacthub/docker-ignore-policy.rego

@@ -0,0 +1,17 @@
+package trivy
+
+default ignore = false
+
+ignore {
+	deny_vulnerability_ids := {
+		#
+		# Trivy complains about this in the /usr/bin/migrate binary.
+		# Nothing we can about it until the maintainer updates that dependency on their end.
+		# See: https://github.com/golang-migrate/migrate/issues/1357
+		#
+		"CVE-2026-33186",
+		"CVE-2025-68121"
+	}
+
+	input.VulnerabilityID = deny_vulnerability_ids[_]
+}

artifacthub/go.mod

@@ -1,6 +1,6 @@
 module github.com/semaphoreio/semaphore/artifacthub
 
-go 1.24.0
+go 1.25
 
 require (
 	cloud.google.com/go/storage v1.36.0
@@ -17,6 +17,7 @@ require (
 	github.com/stretchr/testify v1.11.1
 	go.uber.org/zap v1.25.0
 	google.golang.org/api v0.160.0
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217
 	google.golang.org/grpc v1.79.3
 	google.golang.org/protobuf v1.36.10
 	gorm.io/driver/postgres v1.5.2
@@ -65,7 +66,6 @@ require (
 	golang.org/x/time v0.5.0 // indirect
 	google.golang.org/genproto v0.0.0-20240205150955-31a09d347014 // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20251202230838-ff82c1b0f217 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20251202230838-ff82c1b0f217 // indirect
 	gopkg.in/alexcesaro/statsd.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )