diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fa9dd64c0e..74d4ee432a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,3 +1,4 @@ +--- name: CI on: @@ -10,15 +11,38 @@ on: defaults: run: - # see: https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#using-a-specific-shell shell: bash --noprofile --norc -eo pipefail {0} env: DEBIAN_FRONTEND: noninteractive jobs: + unit-test: + name: "Unit Tests" + runs-on: ubuntu-22.04 + continue-on-error: false + timeout-minutes: 30 + steps: + - name: "Checkout Repository" + uses: actions/checkout@v4 + with: + # fetch the whole repo for `git describe` to work + fetch-depth: 0 + - name: "Docker Image" + run: | + make docker-image + - name: "Unit Tests" + run: | + make docker-unit-test + mkdir -p out/coverage + mv unit.out out/coverage/ + - uses: actions/upload-artifact@v4 + with: + name: unit-tests-coverage + path: out/ + test: - name: "Test (Ubuntu 22.04)" + name: "System Test" runs-on: ubuntu-22.04 continue-on-error: false timeout-minutes: 30 @@ -63,21 +87,10 @@ jobs: with: directory: ${{ runner.temp }} - - name: "Run Unit Tests" - env: - RUN_LONG_TESTS: 'yes' - AZURE_STORAGE_ENDPOINT: "http://127.0.0.1:10000/devstoreaccount1" - AZURE_STORAGE_ACCOUNT: "devstoreaccount1" - AZURE_STORAGE_ACCESS_KEY: "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - run: | - sudo mkdir -p /srv ; sudo chown runner /srv - COVERAGE_DIR=${{ runner.temp }} make test - - name: "Run Benchmark" run: | - COVERAGE_DIR=${{ runner.temp }} make bench + mkdir -p out/coverage + COVERAGE_DIR=$PWD/out/coverage make bench - name: "Run System Tests" env: @@ -89,22 +102,55 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} run: | sudo mkdir -p /srv ; sudo chown runner /srv - COVERAGE_DIR=${{ runner.temp }} make system-test + mkdir -p out/coverage + COVERAGE_DIR=$PWD/out/coverage make system-test + + - uses: actions/upload-artifact@v4 + with: + name: system-tests-coverage + path: out/ + + coverage: + name: "Upload Coverage" + runs-on: ubuntu-22.04 + continue-on-error: false + timeout-minutes: 30 + needs: + - unit-test + - test + steps: + - name: "Checkout Repository" + uses: actions/checkout@v4 + + - name: "Download Unit Test Coverage" + uses: actions/download-artifact@v4 + with: + name: unit-tests-coverage + + - name: "Download System Test Coverage" + uses: actions/download-artifact@v4 + with: + name: system-tests-coverage - name: "Merge Code Coverage" run: | - go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad - ~/go/bin/gocovmerge unit.out ${{ runner.temp }}/*.out > coverage.txt + # go install github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad + # ~/go/bin/gocovmerge coverage/*.out > coverage.txt + awk 'FNR==1 && NR!=1 {next} {print}' coverage/*.out > coverage.txt - name: "Upload Code Coverage" - uses: codecov/codecov-action@v2 + if: github.actor != 'dependabot[bot]' + uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: coverage.txt + fail_ci_if_error: true + ci-debian-build: name: "Build" - needs: test + needs: + - coverage runs-on: ubuntu-latest strategy: fail-fast: false @@ -226,7 +272,8 @@ jobs: ci-binary-build: name: "Build" - needs: test + needs: + - coverage runs-on: ubuntu-latest strategy: matrix: diff --git a/.github/workflows/golangci-lint.yml b/.github/workflows/golangci-lint.yml index dad2bdc5c6..6bc15fbb03 100644 --- a/.github/workflows/golangci-lint.yml +++ b/.github/workflows/golangci-lint.yml @@ -35,7 +35,7 @@ jobs: - name: Install and initialize swagger run: | go install github.com/swaggo/swag/cmd/swag@latest - swag init -q --markdownFiles docs + swag init -q --propertyStrategy pascalcase --markdownFiles docs shell: sh - name: golangci-lint diff --git a/AUTHORS b/AUTHORS index e9bb9f3804..fc983e667d 100644 --- a/AUTHORS +++ b/AUTHORS @@ -69,3 +69,10 @@ List of contributors, in chronological order: * Leigh London (https://github.com/leighlondon) * Gordian Schoenherr (https://github.com/schoenherrg) * Silke Hofstra (https://github.com/silkeh) +* Itay Porezky (https://github.com/itayporezky) +* JupiterRider (https://github.com/JupiterRider) +* Tobias Assarsson (https://github.com/daedaluz) +* Yaksh Bariya (https://github.com/thunder-coding) +* Brian Witt (https://github.com/bwitt) +* Ales Bregar (https://github.com/abregar) +* Tim Foerster (https://github.com/tonobo) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 762cf8a472..005f4dabeb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ Please report unacceptable behavior on [https://github.com/aptly-dev/aptly/discu ### List of Repositories * [aptly-dev/aptly](https://github.com/aptly-dev/aptly) - aptly source code, functional tests, man page -* [apty-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/) +* [aptly-dev/aptly-dev.github.io](https://github.com/aptly-dev/aptly-dev.github.io) - aptly website (https://www.aptly.info/) * [aptly-dev/aptly-fixture-db](https://github.com/aptly-dev/aptly-fixture-db) & [aptly-dev/aptly-fixture-pool](https://github.com/aptly-dev/aptly-fixture-pool) provide fixtures for aptly functional tests @@ -130,14 +130,14 @@ aptly version: 1.5.0+189+g0fc90dff In order to run aptly unit tests, enter the following: ``` -make docker-unit-tests +make docker-unit-test ``` #### Running system tests In order to run aptly system tests, enter the following: ``` -make docker-system-tests +make docker-system-test ``` #### Running golangci-lint diff --git a/Makefile b/Makefile index ffe2e8a3c1..43cb53012b 100644 --- a/Makefile +++ b/Makefile @@ -7,9 +7,23 @@ COVERAGE_DIR?=$(shell mktemp -d) GOOS=$(shell go env GOHOSTOS) GOARCH=$(shell go env GOHOSTARCH) +export PODMAN_USERNS = keep-id +DOCKER_RUN = docker run --security-opt label=disable --user 0:0 --rm -v ${PWD}:/work/src + +# Setting TZ for certificates +export TZ=UTC # Unit Tests and some sysmte tests rely on expired certificates, turn back the time export TEST_FAKETIME := 2025-01-02 03:04:05 +# run with 'COVERAGE_SKIP=1' to skip coverage checks during system tests +ifeq ($(COVERAGE_SKIP),1) +COVERAGE_ARG_BUILD := +COVERAGE_ARG_TEST := --coverage-skip +else +COVERAGE_ARG_BUILD := -coverpkg="./..." +COVERAGE_ARG_TEST := --coverage-dir $(COVERAGE_DIR) +endif + # export CAPUTRE=1 for regenrating test gold files ifeq ($(CAPTURE),1) CAPTURE_ARG := --capture @@ -61,9 +75,9 @@ azurite-start: azurite-stop: @kill `cat ~/.azurite.pid` -swagger: swagger-install +swagger: #swagger-install # Generate swagger docs - @PATH=$(BINPATH)/:$(PATH) swag init --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf + #@PATH=$(BINPATH)/:$(PATH) swag init --propertyStrategy pascalcase --parseDependency --parseInternal --markdownFiles docs --generalInfo docs/swagger.conf etcd-install: # Install etcd @@ -101,13 +115,13 @@ test: prepare swagger etcd-install ## Run unit tests (add TEST=regex to specify system-test: prepare swagger etcd-install ## Run system tests # build coverage binary - go test -v -coverpkg="./..." -c -tags testruncli + go test -v $(COVERAGE_ARG_BUILD) -c -tags testruncli # Download fixture-db, fixture-pool, etcd.db if [ ! -e ~/aptly-fixture-db ]; then git clone https://github.com/aptly-dev/aptly-fixture-db.git ~/aptly-fixture-db/; fi if [ ! -e ~/aptly-fixture-pool ]; then git clone https://github.com/aptly-dev/aptly-fixture-pool.git ~/aptly-fixture-pool/; fi test -f ~/etcd.db || (curl -o ~/etcd.db.xz http://repo.aptly.info/system-tests/etcd.db.xz && xz -d ~/etcd.db.xz) # Run system tests - PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long --coverage-dir $(COVERAGE_DIR) $(CAPTURE_ARG) $(TEST) + PATH=$(BINPATH)/:$(PATH) FORCE_COLOR=1 $(PYTHON) system/run.py --long $(COVERAGE_ARG_TEST) $(CAPTURE_ARG) $(TEST) bench: @echo "\e[33m\e[1mRunning benchmark ...\e[0m" @@ -117,7 +131,8 @@ serve: prepare swagger-install ## Run development server (auto recompiling) test -f $(BINPATH)/air || go install github.com/air-verse/air@v1.52.3 cp debian/aptly.conf ~/.aptly.conf sed -i /enable_swagger_endpoint/s/false/true/ ~/.aptly.conf - PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142 + sed -i /enable_metrics_endpoint/s/false/true/ ~/.aptly.conf + PATH=$(BINPATH):$$PATH air -build.pre_cmd 'swag init -q --propertyStrategy pascalcase --markdownFiles docs --generalInfo docs/swagger.conf' -build.exclude_dir docs,system,debian,pgp/keyrings,pgp/test-bins,completion.d,man,deb/testdata,console,_man,systemd,obj-x86_64-linux-gnu -- api serve -listen 0.0.0.0:3142 dpkg: prepare swagger ## Build debian packages @test -n "$(DEBARCH)" || (echo "please define DEBARCH"; exit 1) @@ -171,16 +186,16 @@ docker-image-no-cache: ## Build aptly-dev docker image (no cache) @docker build --no-cache -f system/Dockerfile . -t aptly-dev docker-build: ## Build aptly in docker container - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper build + @$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper build docker-shell: ## Run aptly and other commands in docker container - @docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper || true + @$(DOCKER_RUN) -it -p 3142:3142 aptly-dev /work/src/system/docker-wrapper || true docker-deb: ## Build debian packages in docker container - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64 + @$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper dpkg DEBARCH=amd64 docker-unit-test: ## Run unit tests in docker container (add TEST=regex to specify which tests to run) - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \ + $(DOCKER_RUN) -t --tmpfs /smallfs:rw,size=1m aptly-dev /work/src/system/docker-wrapper \ azurite-start \ AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ @@ -189,27 +204,27 @@ docker-unit-test: ## Run unit tests in docker container (add TEST=regex to spec azurite-stop docker-system-test: ## Run system tests in docker container (add TEST=t04_mirror or TEST=UpdateMirror26Test to run only specific tests) - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper \ + @$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper \ azurite-start \ AZURE_STORAGE_ENDPOINT=http://127.0.0.1:10000/devstoreaccount1 \ AZURE_STORAGE_ACCOUNT=devstoreaccount1 \ AZURE_STORAGE_ACCESS_KEY="Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" \ AWS_ACCESS_KEY_ID=$(AWS_ACCESS_KEY_ID) \ AWS_SECRET_ACCESS_KEY=$(AWS_SECRET_ACCESS_KEY) \ - system-test TEST=$(TEST) \ + system-test TEST=$(TEST) CAPTURE=$(CAPTURE) COVERAGE_SKIP=$(COVERAGE_SKIP) \ azurite-stop docker-serve: ## Run development server (auto recompiling) on http://localhost:3142 - @docker run -it --rm -p 3142:3142 -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper serve || true + @$(DOCKER_RUN) -it -p 3142:3142 -v /tmp/cache-go-aptly:/var/lib/aptly/.cache/go-build aptly-dev /work/src/system/docker-wrapper serve || true docker-lint: ## Run golangci-lint in docker container - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper lint + @$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper lint docker-binaries: ## Build binary releases (FreeBSD, macOS, Linux generic) in docker container - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper binaries + @$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper binaries docker-man: ## Create man page in docker container - @docker run -it --rm -v ${PWD}:/work/src aptly-dev /work/src/system/docker-wrapper man + @$(DOCKER_RUN) -t aptly-dev /work/src/system/docker-wrapper man mem.png: mem.dat mem.gp gnuplot mem.gp diff --git a/api/api.go b/api/api.go index ab8c8ba50d..b13b40c1ee 100644 --- a/api/api.go +++ b/api/api.go @@ -70,7 +70,7 @@ func apiReady(isReady *atomic.Value) func(*gin.Context) { return } - status := aptlyStatus{Status: "Aptly is ready"} + status := aptlyStatus{Status: "Aptly is ready"} c.JSON(200, status) } } @@ -178,7 +178,7 @@ func truthy(value interface{}) bool { if value == nil { return false } - switch v := value.(type) { + switch v := value.(type) { case string: switch strings.ToLower(v) { case "n", "no", "f", "false", "0", "off": diff --git a/api/files.go b/api/files.go index 7c6ad54d48..e51ea00692 100644 --- a/api/files.go +++ b/api/files.go @@ -13,6 +13,10 @@ import ( "github.com/saracen/walker" ) +// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC). +// In production it calls (*os.File).Sync(). +var syncFile = func(f *os.File) error { return f.Sync() } + func verifyPath(path string) bool { path = filepath.Clean(path) for _, part := range strings.Split(path, string(filepath.Separator)) { @@ -114,34 +118,69 @@ func apiFilesUpload(c *gin.Context) { } stored := []string{} + openFiles := []*os.File{} + // Write all files first for _, files := range c.Request.MultipartForm.File { for _, file := range files { src, err := file.Open() if err != nil { + // Close any files we've opened + for _, f := range openFiles { + _ = f.Close() + } AbortWithJSONError(c, 500, err) return } - defer func() { _ = src.Close() }() destPath := filepath.Join(path, filepath.Base(file.Filename)) dst, err := os.Create(destPath) if err != nil { + _ = src.Close() + // Close any files we've opened + for _, f := range openFiles { + _ = f.Close() + } AbortWithJSONError(c, 500, err) return } - defer func() { _ = dst.Close() }() _, err = io.Copy(dst, src) + _ = src.Close() if err != nil { + _ = dst.Close() + // Close any files we've opened + for _, f := range openFiles { + _ = f.Close() + } AbortWithJSONError(c, 500, err) return } + // Keep file open for batch sync + openFiles = append(openFiles, dst) stored = append(stored, filepath.Join(c.Params.ByName("dir"), filepath.Base(file.Filename))) } } + // Sync all files at once to catch ENOSPC errors + for i, dst := range openFiles { + err := syncFile(dst) + if err != nil { + // Close all files + for _, f := range openFiles { + _ = f.Close() + } + AbortWithJSONError(c, 500, fmt.Errorf("error syncing file %s: %s", stored[i], err)) + return + } + } + + // Close all files + for _, dst := range openFiles { + _ = dst.Close() + } + apiFilesUploadedCounter.WithLabelValues(c.Params.ByName("dir")).Inc() c.JSON(200, stored) } diff --git a/api/files_test.go b/api/files_test.go new file mode 100644 index 0000000000..9083a8d1c1 --- /dev/null +++ b/api/files_test.go @@ -0,0 +1,476 @@ +package api + +import ( + "bytes" + "encoding/json" + "io" + "mime/multipart" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "syscall" + + "github.com/aptly-dev/aptly/aptly" + ctx "github.com/aptly-dev/aptly/context" + "github.com/gin-gonic/gin" + "github.com/smira/flag" + + . "gopkg.in/check.v1" +) + +type FilesUploadDiskFullSuite struct { + aptlyContext *ctx.AptlyContext + flags *flag.FlagSet + configFile *os.File + router http.Handler +} + +var _ = Suite(&FilesUploadDiskFullSuite{}) + +func (s *FilesUploadDiskFullSuite) SetUpTest(c *C) { + aptly.Version = "testVersion" + + file, err := os.CreateTemp("", "aptly") + c.Assert(err, IsNil) + s.configFile = file + + jsonString, err := json.Marshal(gin.H{ + "architectures": []string{}, + "rootDir": c.MkDir(), + }) + c.Assert(err, IsNil) + _, err = file.Write(jsonString) + c.Assert(err, IsNil) + _ = file.Close() + + flags := flag.NewFlagSet("fakeFlags", flag.ContinueOnError) + flags.Bool("no-lock", false, "dummy") + flags.Int("db-open-attempts", 3, "dummy") + flags.String("config", s.configFile.Name(), "dummy") + flags.String("architectures", "", "dummy") + s.flags = flags + + aptlyContext, err := ctx.NewContext(s.flags) + c.Assert(err, IsNil) + + s.aptlyContext = aptlyContext + s.router = Router(aptlyContext) + context = aptlyContext +} + +func (s *FilesUploadDiskFullSuite) TearDownTest(c *C) { + if s.configFile != nil { + _ = os.Remove(s.configFile.Name()) + } + if s.aptlyContext != nil { + s.aptlyContext.Shutdown() + } +} + +func (s *FilesUploadDiskFullSuite) TestUploadSuccessWithSync(c *C) { + testContent := []byte("test file content for upload") + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + part, err := writer.CreateFormFile("file", "testfile.txt") + c.Assert(err, IsNil) + + _, err = part.Write(testContent) + c.Assert(err, IsNil) + + err = writer.Close() + c.Assert(err, IsNil) + + req, err := http.NewRequest("POST", "/api/files/testdir", body) + c.Assert(err, IsNil) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 200) + + uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir", "testfile.txt") + content, err := os.ReadFile(uploadedFile) + c.Assert(err, IsNil) + c.Check(content, DeepEquals, testContent) +} + +func (s *FilesUploadDiskFullSuite) TestUploadVerifiesFileIntegrity(c *C) { + testContent := bytes.Repeat([]byte("A"), 10000) + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + part, err := writer.CreateFormFile("file", "largefile.bin") + c.Assert(err, IsNil) + + _, err = io.Copy(part, bytes.NewReader(testContent)) + c.Assert(err, IsNil) + + err = writer.Close() + c.Assert(err, IsNil) + + req, err := http.NewRequest("POST", "/api/files/testdir2", body) + c.Assert(err, IsNil) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 200) + + uploadedFile := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "testdir2", "largefile.bin") + content, err := os.ReadFile(uploadedFile) + c.Assert(err, IsNil) + c.Check(len(content), Equals, len(testContent)) + c.Check(content, DeepEquals, testContent) +} + +func (s *FilesUploadDiskFullSuite) TestUploadMultipleFilesWithBatchSync(c *C) { + testFiles := map[string][]byte{ + "file1.txt": []byte("content of file 1"), + "file2.txt": bytes.Repeat([]byte("B"), 5000), + "file3.deb": []byte("debian package content"), + } + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + for filename, content := range testFiles { + part, err := writer.CreateFormFile("file", filename) + c.Assert(err, IsNil) + _, err = part.Write(content) + c.Assert(err, IsNil) + } + + err := writer.Close() + c.Assert(err, IsNil) + + req, err := http.NewRequest("POST", "/api/files/multitest", body) + c.Assert(err, IsNil) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 200) + + uploadDir := filepath.Join(s.aptlyContext.Config().GetRootDir(), "upload", "multitest") + for filename, expectedContent := range testFiles { + uploadedFile := filepath.Join(uploadDir, filename) + content, err := os.ReadFile(uploadedFile) + c.Assert(err, IsNil, Commentf("Failed to read %s", filename)) + c.Check(content, DeepEquals, expectedContent, Commentf("Content mismatch for %s", filename)) + } +} + +func (s *FilesUploadDiskFullSuite) TestUploadReturnsErrorOnSyncFailure(c *C) { + oldSyncFile := syncFile + syncFile = func(f *os.File) error { + if filepath.Base(f.Name()) == "syncfail.txt" { + return syscall.ENOSPC + } + return nil + } + defer func() { syncFile = oldSyncFile }() + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + + part1, err := writer.CreateFormFile("file", "ok.txt") + c.Assert(err, IsNil) + _, err = part1.Write([]byte("ok")) + c.Assert(err, IsNil) + + part2, err := writer.CreateFormFile("file", "syncfail.txt") + c.Assert(err, IsNil) + _, err = part2.Write([]byte("will fail on sync")) + c.Assert(err, IsNil) + + err = writer.Close() + c.Assert(err, IsNil) + + req, err := http.NewRequest("POST", "/api/files/syncfaildir", body) + c.Assert(err, IsNil) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 500) + c.Check(bytes.Contains(w.Body.Bytes(), []byte("error syncing file")), Equals, true) +} + +func (s *FilesUploadDiskFullSuite) TestVerifyPath(c *C) { + c.Check(verifyPath("a/b/c"), Equals, true) + c.Check(verifyPath("../x"), Equals, false) + c.Check(verifyPath("./x"), Equals, true) + c.Check(verifyPath(".."), Equals, false) + c.Check(verifyPath("."), Equals, false) +} + +func (s *FilesUploadDiskFullSuite) TestListDirsEmptyWhenUploadMissing(c *C) { + _ = os.RemoveAll(s.aptlyContext.UploadPath()) + + req, err := http.NewRequest("GET", "/api/files", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 200) + c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]") +} + +func (s *FilesUploadDiskFullSuite) TestListDirsReturnsDirectories(c *C) { + uploadRoot := s.aptlyContext.UploadPath() + c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d1"), 0777), IsNil) + c.Assert(os.MkdirAll(filepath.Join(uploadRoot, "d2"), 0777), IsNil) + c.Assert(os.WriteFile(filepath.Join(uploadRoot, "rootfile"), []byte("x"), 0644), IsNil) + + req, err := http.NewRequest("GET", "/api/files", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 200) + body := w.Body.String() + c.Check(strings.Contains(body, "d1"), Equals, true) + c.Check(strings.Contains(body, "d2"), Equals, true) +} + +func (s *FilesUploadDiskFullSuite) TestListFilesNotFound(c *C) { + req, err := http.NewRequest("GET", "/api/files/does-not-exist", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 404) +} + +func (s *FilesUploadDiskFullSuite) TestListFilesReturnsFiles(c *C) { + base := filepath.Join(s.aptlyContext.UploadPath(), "dir") + c.Assert(os.MkdirAll(base, 0777), IsNil) + c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil) + c.Assert(os.WriteFile(filepath.Join(base, "b.txt"), []byte("b"), 0644), IsNil) + + req, err := http.NewRequest("GET", "/api/files/dir", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 200) + body := w.Body.String() + c.Check(strings.Contains(body, "a.txt"), Equals, true) + c.Check(strings.Contains(body, "b.txt"), Equals, true) +} + +func (s *FilesUploadDiskFullSuite) TestDeleteDirRemovesDirectory(c *C) { + base := filepath.Join(s.aptlyContext.UploadPath(), "todel") + c.Assert(os.MkdirAll(base, 0777), IsNil) + c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil) + + req, err := http.NewRequest("DELETE", "/api/files/todel", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 200) + + _, statErr := os.Stat(base) + c.Check(os.IsNotExist(statErr), Equals, true) +} + +func (s *FilesUploadDiskFullSuite) TestDeleteFileRemovesFile(c *C) { + base := filepath.Join(s.aptlyContext.UploadPath(), "todel2") + c.Assert(os.MkdirAll(base, 0777), IsNil) + c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil) + + req, err := http.NewRequest("DELETE", "/api/files/todel2/a.txt", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 200) + + _, statErr := os.Stat(filepath.Join(base, "a.txt")) + c.Check(os.IsNotExist(statErr), Equals, true) +} + +func (s *FilesUploadDiskFullSuite) TestDeleteFileNotFoundStillOk(c *C) { + base := filepath.Join(s.aptlyContext.UploadPath(), "todel3") + c.Assert(os.MkdirAll(base, 0777), IsNil) + + req, err := http.NewRequest("DELETE", "/api/files/todel3/nope.txt", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 200) +} + +func (s *FilesUploadDiskFullSuite) TestRejectsInvalidDir(c *C) { + req, err := http.NewRequest("DELETE", "/api/files/..", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 400) +} + +func (s *FilesUploadDiskFullSuite) TestRejectsInvalidFileName(c *C) { + base := filepath.Join(s.aptlyContext.UploadPath(), "dirx") + c.Assert(os.MkdirAll(base, 0777), IsNil) + + req, err := http.NewRequest("DELETE", "/api/files/dirx/..", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 400) +} + +func (s *FilesUploadDiskFullSuite) TestListDirsEmptyIfUploadPathIsNotDir(c *C) { + _ = os.RemoveAll(s.aptlyContext.UploadPath()) + c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil) + + req, err := http.NewRequest("GET", "/api/files", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 200) + c.Check(strings.TrimSpace(w.Body.String()), Equals, "[]") +} + +func (s *FilesUploadDiskFullSuite) TestListFilesReturns500OnPermissionError(c *C) { + base := filepath.Join(s.aptlyContext.UploadPath(), "noperms") + c.Assert(os.MkdirAll(base, 0777), IsNil) + c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil) + c.Assert(os.Chmod(base, 0), IsNil) + defer func() { _ = os.Chmod(base, 0777) }() + + req, err := http.NewRequest("GET", "/api/files/noperms", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 500) +} + +func (s *FilesUploadDiskFullSuite) TestDeleteFileReturns500OnNonNotExistError(c *C) { + base := filepath.Join(s.aptlyContext.UploadPath(), "dirisfile") + c.Assert(os.MkdirAll(base, 0777), IsNil) + subdir := filepath.Join(base, "subdir") + c.Assert(os.MkdirAll(subdir, 0777), IsNil) + c.Assert(os.WriteFile(filepath.Join(subdir, "x"), []byte("x"), 0644), IsNil) + + req, err := http.NewRequest("DELETE", "/api/files/dirisfile/subdir", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 500) +} + +func (s *FilesUploadDiskFullSuite) TestUploadBadMultipartReturns400(c *C) { + req, err := http.NewRequest("POST", "/api/files/badmultipart", bytes.NewBufferString("not multipart")) + c.Assert(err, IsNil) + req.Header.Set("Content-Type", "multipart/form-data; boundary=missing") + + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + + c.Assert(w.Code, Equals, 400) +} + +func (s *FilesUploadDiskFullSuite) TestUploadRejectsInvalidDir(c *C) { + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("file", "a.txt") + c.Assert(err, IsNil) + _, err = part.Write([]byte("x")) + c.Assert(err, IsNil) + c.Assert(writer.Close(), IsNil) + + req, err := http.NewRequest("POST", "/api/files/..", body) + c.Assert(err, IsNil) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 400) +} + +func (s *FilesUploadDiskFullSuite) TestUploadReturns500IfUploadRootIsNotDir(c *C) { + _ = os.RemoveAll(s.aptlyContext.UploadPath()) + c.Assert(os.WriteFile(s.aptlyContext.UploadPath(), []byte("not a dir"), 0644), IsNil) + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("file", "a.txt") + c.Assert(err, IsNil) + _, err = part.Write([]byte("x")) + c.Assert(err, IsNil) + c.Assert(writer.Close(), IsNil) + + req, err := http.NewRequest("POST", "/api/files/testdir", body) + c.Assert(err, IsNil) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 500) +} + +func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnFileOpenFailure(c *C) { + // Pre-populate MultipartForm to inject a FileHeader that fails on Open(). + form := &multipart.Form{ + File: map[string][]*multipart.FileHeader{ + "file": {{Filename: "broken.bin"}}, + }, + } + + req, err := http.NewRequest("POST", "/api/files/openfaildir", nil) + c.Assert(err, IsNil) + req.MultipartForm = form + + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 500) +} + +func (s *FilesUploadDiskFullSuite) TestUploadReturns500OnCreateFailure(c *C) { + base := filepath.Join(s.aptlyContext.UploadPath(), "readonly") + c.Assert(os.MkdirAll(base, 0777), IsNil) + c.Assert(os.Chmod(base, 0555), IsNil) + defer func() { _ = os.Chmod(base, 0777) }() + + body := &bytes.Buffer{} + writer := multipart.NewWriter(body) + part, err := writer.CreateFormFile("file", "a.txt") + c.Assert(err, IsNil) + _, err = part.Write([]byte("x")) + c.Assert(err, IsNil) + c.Assert(writer.Close(), IsNil) + + req, err := http.NewRequest("POST", "/api/files/readonly", body) + c.Assert(err, IsNil) + req.Header.Set("Content-Type", writer.FormDataContentType()) + + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 500) +} + +func (s *FilesUploadDiskFullSuite) TestDeleteDirReturns500OnRemoveFailure(c *C) { + parent := s.aptlyContext.UploadPath() + base := filepath.Join(parent, "cantremove") + c.Assert(os.MkdirAll(base, 0777), IsNil) + c.Assert(os.WriteFile(filepath.Join(base, "a.txt"), []byte("a"), 0644), IsNil) + + c.Assert(os.Chmod(parent, 0555), IsNil) + defer func() { _ = os.Chmod(parent, 0777) }() + + req, err := http.NewRequest("DELETE", "/api/files/cantremove", nil) + c.Assert(err, IsNil) + w := httptest.NewRecorder() + s.router.ServeHTTP(w, req) + c.Assert(w.Code, Equals, 500) +} diff --git a/api/gpg.go b/api/gpg.go index 47ba1d92f6..f9109e6475 100644 --- a/api/gpg.go +++ b/api/gpg.go @@ -28,7 +28,7 @@ type gpgAddKeyParams struct { // @Summary Add GPG Keys // @Description **Adds GPG keys to aptly keyring** // @Description -// @Description Add GPG public keys for veryfing remote repositories for mirroring. +// @Description Add GPG public keys for verifying remote repositories for mirroring. // @Description // @Description Keys can be added in two ways: // @Description * By providing the ASCII armord key in `GpgKeyArmor` (leave Keyserver and GpgKeyID empty) diff --git a/api/mirror.go b/api/mirror.go index f3dd123ac2..1f3adeec19 100644 --- a/api/mirror.go +++ b/api/mirror.go @@ -175,9 +175,9 @@ func apiMirrorsDrop(c *gin.Context) { name := c.Params.ByName("name") force := c.Request.URL.Query().Get("force") == "1" + // Phase 1: Pre-task validation (shallow load for 404 check only) collectionFactory := context.NewCollectionFactory() mirrorCollection := collectionFactory.RemoteRepoCollection() - snapshotCollection := collectionFactory.SnapshotCollection() repo, err := mirrorCollection.ByName(name) if err != nil { @@ -187,21 +187,34 @@ func apiMirrorsDrop(c *gin.Context) { resources := []string{string(repo.Key())} taskName := fmt.Sprintf("Delete mirror %s", name) + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err := repo.CheckLock() + // Phase 2: Inside task lock - create fresh collections + taskCollectionFactory := context.NewCollectionFactory() + taskMirrorCollection := taskCollectionFactory.RemoteRepoCollection() + taskSnapshotCollection := taskCollectionFactory.SnapshotCollection() + + // Fresh load after lock acquired + repo, err := taskMirrorCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err) + } + + err = repo.CheckLock() if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err) } if !force { - snapshots := snapshotCollection.ByRemoteRepoSource(repo) + // Fresh checks with current collections + snapshots := taskSnapshotCollection.ByRemoteRepoSource(repo) if len(snapshots) > 0 { return &task.ProcessReturnValue{Code: http.StatusForbidden, Value: nil}, fmt.Errorf("won't delete mirror with snapshots, use 'force=1' to override") } } - err = mirrorCollection.Drop(repo) + err = taskMirrorCollection.Drop(repo) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %v", err) } @@ -232,6 +245,7 @@ func apiMirrorsShow(c *gin.Context) { err = collection.LoadComplete(repo) if err != nil { AbortWithJSONError(c, 500, fmt.Errorf("unable to show: %s", err)) + return } c.JSON(200, repo) @@ -333,26 +347,8 @@ func apiMirrorsPackages(c *gin.Context) { type mirrorUpdateParams struct { // Change mirror name to `Name` Name string ` json:"Name" example:"mirror1"` - // Url of the archive to mirror - ArchiveURL string ` json:"ArchiveURL" example:"http://deb.debian.org/debian"` - // Package query that is applied to mirror packages - Filter string ` json:"Filter" example:"xserver-xorg"` - // Limit mirror to those architectures, if not specified aptly would fetch all architectures - Architectures []string ` json:"Architectures" example:"amd64"` - // Components to mirror, if not specified aptly would fetch all components - Components []string ` json:"Components" example:"main"` - // Gpg keyring(s) for verifing Release file + // Gpg keyring(s) for verifying Release file Keyrings []string ` json:"Keyrings" example:"trustedkeys.gpg"` - // Set "true" to include dependencies of matching packages when filtering - FilterWithDeps bool ` json:"FilterWithDeps"` - // Set "true" to mirror source packages - DownloadSources bool ` json:"DownloadSources"` - // Set "true" to mirror udeb files - DownloadUdebs bool ` json:"DownloadUdebs"` - // Set "true" to skip checking if the given components are in the Release file - SkipComponentCheck bool ` json:"SkipComponentCheck"` - // Set "true" to skip checking if the given architectures are in the Release file - SkipArchitectureCheck bool ` json:"SkipArchitectureCheck"` // Set "true" to ignore checksum errors IgnoreChecksums bool ` json:"IgnoreChecksums"` // Set "true" to skip the verification of Release file signatures @@ -387,21 +383,14 @@ func apiMirrorsUpdate(c *gin.Context) { collectionFactory := context.NewCollectionFactory() collection := collectionFactory.RemoteRepoCollection() - remote, err = collection.ByName(c.Params.ByName("name")) + name := c.Params.ByName("name") + remote, err = collection.ByName(name) if err != nil { AbortWithJSONError(c, 404, err) return } b.Name = remote.Name - b.DownloadUdebs = remote.DownloadUdebs - b.DownloadSources = remote.DownloadSources - b.SkipComponentCheck = remote.SkipComponentCheck - b.SkipArchitectureCheck = remote.SkipArchitectureCheck - b.FilterWithDeps = remote.FilterWithDeps - b.Filter = remote.Filter - b.Architectures = remote.Architectures - b.Components = remote.Components b.IgnoreSignatures = context.Config().GpgDisableVerify log.Info().Msgf("%s: Starting mirror update", b.Name) @@ -410,6 +399,7 @@ func apiMirrorsUpdate(c *gin.Context) { return } + // Pre-task validation of new name if provided if b.Name != remote.Name { _, err = collection.ByName(b.Name) if err == nil { @@ -418,27 +408,6 @@ func apiMirrorsUpdate(c *gin.Context) { } } - if b.DownloadUdebs != remote.DownloadUdebs { - if remote.IsFlat() && b.DownloadUdebs { - AbortWithJSONError(c, 400, fmt.Errorf("unable to update: flat mirrors don't support udebs")) - return - } - } - - if b.ArchiveURL != "" { - remote.SetArchiveRoot(b.ArchiveURL) - } - - remote.Name = b.Name - remote.DownloadUdebs = b.DownloadUdebs - remote.DownloadSources = b.DownloadSources - remote.SkipComponentCheck = b.SkipComponentCheck - remote.SkipArchitectureCheck = b.SkipArchitectureCheck - remote.FilterWithDeps = b.FilterWithDeps - remote.Filter = b.Filter - remote.Architectures = b.Architectures - remote.Components = b.Components - verifier, err := getVerifier(b.Keyrings) if err != nil { AbortWithJSONError(c, 400, fmt.Errorf("unable to initialize GPG verifier: %s", err)) @@ -447,9 +416,26 @@ func apiMirrorsUpdate(c *gin.Context) { resources := []string{string(remote.Key())} maybeRunTaskInBackground(c, "Update mirror "+b.Name, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) { + // Phase 2: Inside task lock - create fresh factory + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.RemoteRepoCollection() + + // Fresh load after lock acquired (use captured `name` variable, not gin context) + remote, err := taskCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + // Fresh rename check inside lock (if renaming) + if b.Name != remote.Name { + _, err := taskCollection.ByName(b.Name) + if err == nil { + return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: mirror %s already exists", b.Name) + } + } downloader := context.NewDownloader(out) - err := remote.Fetch(downloader, verifier, b.IgnoreSignatures) + err = remote.Fetch(downloader, verifier, b.IgnoreSignatures) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } @@ -461,7 +447,7 @@ func apiMirrorsUpdate(c *gin.Context) { } } - err = remote.DownloadPackageIndexes(out, downloader, verifier, collectionFactory, b.IgnoreSignatures, b.SkipComponentCheck) + err = remote.DownloadPackageIndexes(out, downloader, verifier, taskCollectionFactory, b.IgnoreSignatures, remote.SkipComponentCheck) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } @@ -480,8 +466,8 @@ func apiMirrorsUpdate(c *gin.Context) { } } - queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), collectionFactory.PackageCollection(), - collectionFactory.ChecksumCollection(nil), b.SkipExistingPackages) + queue, downloadSize, err := remote.BuildDownloadQueue(context.PackagePool(), taskCollectionFactory.PackageCollection(), + taskCollectionFactory.ChecksumCollection(nil), b.SkipExistingPackages) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } @@ -491,12 +477,12 @@ func apiMirrorsUpdate(c *gin.Context) { e := context.ReOpenDatabase() if e == nil { remote.MarkAsIdle() - _ = collection.Update(remote) + _ = taskCollection.Update(remote) } }() remote.MarkAsUpdating() - err = collection.Update(remote) + err = taskCollection.Update(remote) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } @@ -600,7 +586,7 @@ func apiMirrorsUpdate(c *gin.Context) { } // and import it back to the pool - task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, collectionFactory.ChecksumCollection(nil)) + task.File.PoolPath, err = context.PackagePool().Import(task.TempDownPath, task.File.Filename, &task.File.Checksums, true, taskCollectionFactory.ChecksumCollection(nil)) if err != nil { //return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to import file: %s", err) pushError(err) @@ -653,8 +639,8 @@ func apiMirrorsUpdate(c *gin.Context) { } log.Info().Msgf("%s: Finalizing download...", b.Name) - _ = remote.FinalizeDownload(collectionFactory, out) - err = collectionFactory.RemoteRepoCollection().Update(remote) + _ = remote.FinalizeDownload(taskCollectionFactory, out) + err = taskCollection.Update(remote) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } diff --git a/api/publish.go b/api/publish.go index 1a2b5287bb..e13da5358a 100644 --- a/api/publish.go +++ b/api/publish.go @@ -16,8 +16,8 @@ import ( type signingParams struct { // Don't sign published repository Skip bool ` json:"Skip" example:"false"` - // GPG key ID to use when signing the release, if not specified default key is used - GpgKey string ` json:"GpgKey" example:"A0546A43624A8331"` + // GPG key ID(s) to use when signing the release, separated by comma, and if not specified, default configured key(s) are used + GpgKey string ` json:"GpgKey" example:"KEY_ID_a, KEY_ID_b"` // GPG keyring to use (instead of default) Keyring string ` json:"Keyring" example:"trustedkeys.gpg"` // GPG secret keyring to use (instead of default) Note: depreciated with gpg2 @@ -41,7 +41,21 @@ func getSigner(options *signingParams) (pgp.Signer, error) { } signer := context.GetSigner() - signer.SetKey(options.GpgKey) + + var multiGpgKeys []string + // REST params have priority over config + if options.GpgKey != "" { + for _, p := range strings.Split(options.GpgKey, ",") { + if t := strings.TrimSpace(p); t != "" { + multiGpgKeys = append(multiGpgKeys, t) + } + } + } else if len(context.Config().GpgKeys) > 0 { + multiGpgKeys = context.Config().GpgKeys + } + for _, gpgKey := range multiGpgKeys { + signer.SetKey(gpgKey) + } signer.SetKeyRing(options.Keyring, options.SecretKeyring) signer.SetPassphrase(options.Passphrase, options.PassphraseFile) @@ -110,7 +124,7 @@ func apiPublishList(c *gin.Context) { // @Description See also: `aptly publish show` // @Tags Publish // @Produce json -// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambigious in URLs" +// @Param prefix path string true "publishing prefix, use `:.` instead of `.` because it is ambiguous in URLs" // @Param distribution path string true "distribution name" // @Success 200 {object} deb.PublishedRepo // @Failure 404 {object} Error "Published repository not found" @@ -146,10 +160,6 @@ type publishedRepoCreateParams struct { Sources []sourceParams `binding:"required" json:"Sources"` // Distribution name, if missing Aptly would try to guess from sources Distribution string ` json:"Distribution" example:"bookworm"` - // Value of Label: field in published repository stanza - Label string ` json:"Label" example:""` - // Value of Origin: field in published repository stanza - Origin string ` json:"Origin" example:""` // when publishing, overwrite files in pool/ directory without notice ForceOverwrite bool ` json:"ForceOverwrite" example:"false"` // Override list of published architectures @@ -182,7 +192,7 @@ type publishedRepoCreateParams struct { // @Description **Example:** // @Description ``` // @Description $ curl -X POST -H 'Content-Type: application/json' --data '{"Distribution": "wheezy", "Sources": [{"Name": "aptly-repo"}]}' http://localhost:8080/api/publish//repos -// @Description {"Architectures":["i386"],"Distribution":"wheezy","Label":"","Origin":"","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""} +// @Description {"Architectures":["i386"],"Distribution":"wheezy","Prefix":".","SourceKind":"local","Sources":[{"Component":"main","Name":"aptly-repo"}],"Storage":""} // @Description ``` // @Description // @Description See also: `aptly publish create` @@ -249,7 +259,7 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { return } - resources = append(resources, string(snapshot.ResourceKey())) + resources = append(resources, string(snapshot.Key())) sources = append(sources, snapshot) } } else if b.SourceKind == deb.SourceLocalRepo { @@ -280,11 +290,24 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { multiDist = *b.MultiDist } - collection := collectionFactory.PublishedRepoCollection() + // Non-MultiDist publishes share a single pool/ directory under the + // prefix. Lock at the prefix level so that concurrent publish/drop + // operations on sibling distributions cannot race during cleanup. + if !multiDist { + storagePrefix := prefix + if storage != "" { + storagePrefix = storage + ":" + prefix + } + + resources = append(resources, deb.PrefixPoolLockKey(storagePrefix)) + } taskName := fmt.Sprintf("Publish %s repository %s/%s with components \"%s\" and sources \"%s\"", b.SourceKind, param, b.Distribution, strings.Join(components, `", "`), strings.Join(names, `", "`)) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, detail *task.Detail) (*task.ProcessReturnValue, error) { + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.PublishedRepoCollection() + taskDetail := task.PublishDetail{ Detail: detail, } @@ -296,10 +319,10 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { for _, source := range sources { switch s := source.(type) { case *deb.Snapshot: - snapshotCollection := collectionFactory.SnapshotCollection() + snapshotCollection := taskCollectionFactory.SnapshotCollection() err = snapshotCollection.LoadComplete(s) case *deb.LocalRepo: - localCollection := collectionFactory.LocalRepoCollection() + localCollection := taskCollectionFactory.LocalRepoCollection() err = localCollection.LoadComplete(s) default: err = fmt.Errorf("unexpected type for source: %T", source) @@ -309,23 +332,17 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { } } - published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, collectionFactory, multiDist) + published, err := deb.NewPublishedRepo(storage, prefix, b.Distribution, b.Architectures, components, sources, taskCollectionFactory, multiDist) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err) } - resources = append(resources, string(published.Key())) - - if b.Origin != "" { - published.Origin = b.Origin - } if b.NotAutomatic != "" { published.NotAutomatic = b.NotAutomatic } if b.ButAutomaticUpgrades != "" { published.ButAutomaticUpgrades = b.ButAutomaticUpgrades } - published.Label = b.Label published.SkipContents = context.Config().SkipContentsPublishing if b.SkipContents != nil { @@ -341,18 +358,18 @@ func apiPublishRepoOrSnapshot(c *gin.Context) { published.AcquireByHash = *b.AcquireByHash } - duplicate := collection.CheckDuplicate(published) + duplicate := taskCollection.CheckDuplicate(published) if duplicate != nil { - _ = collectionFactory.PublishedRepoCollection().LoadComplete(duplicate, collectionFactory) + _ = taskCollectionFactory.PublishedRepoCollection().LoadComplete(duplicate, taskCollectionFactory) return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("prefix/distribution already used by another published repo: %s", duplicate) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath()) + err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, publishOutput, b.ForceOverwrite, context.SkelPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to publish: %s", err) } - err = collection.Add(published) + err = taskCollection.Add(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } @@ -393,7 +410,6 @@ type publishedRepoUpdateSwitchParams struct { // @Description // @Description See also: `aptly publish update` / `aptly publish switch` // @Tags Publish -// @Produce json // @Param prefix path string true "publishing prefix" // @Param distribution path string true "distribution name" // @Param _async query bool false "Run in background and return task object" @@ -425,6 +441,7 @@ func apiPublishUpdateSwitch(c *gin.Context) { collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() snapshotCollection := collectionFactory.SnapshotCollection() + localRepoCollection := collectionFactory.LocalRepoCollection() published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { @@ -432,48 +449,76 @@ func apiPublishUpdateSwitch(c *gin.Context) { return } + resources := []string{string(published.Key())} + if published.SourceKind == deb.SourceLocalRepo { if len(b.Snapshots) > 0 { AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("snapshots shouldn't be given when updating local repo")) return } + for _, uuid := range published.Sources { + repo, err2 := localRepoCollection.ByUUID(uuid) + if err2 != nil { + AbortWithJSONError(c, http.StatusNotFound, err2) + return + } + resources = append(resources, string(repo.Key())) + } } else if published.SourceKind == deb.SourceSnapshot { for _, snapshotInfo := range b.Snapshots { - _, err2 := snapshotCollection.ByName(snapshotInfo.Name) + snapshot, err2 := snapshotCollection.ByName(snapshotInfo.Name) if err2 != nil { AbortWithJSONError(c, http.StatusNotFound, err2) return } + resources = append(resources, string(snapshot.Key())) } } else { AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unknown published repository type")) return } - if b.SkipContents != nil { - published.SkipContents = *b.SkipContents - } - - if b.SkipBz2 != nil { - published.SkipBz2 = *b.SkipBz2 - } - - if b.AcquireByHash != nil { - published.AcquireByHash = *b.AcquireByHash - } - - if b.MultiDist != nil { - published.MultiDist = *b.MultiDist + // Non-MultiDist distributions share a single pool/ directory under the + // prefix. Acquire the prefix-level pool lock so that concurrent updates + // on sibling distributions are serialised and cannot race during cleanup. + if !published.MultiDist { + resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix())) } - resources := []string{string(published.Key())} + // Field mutations and fresh DB load are deferred to inside the task so + // they always operate on a consistent state after the lock is held. taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.LoadComplete(published, collectionFactory) + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.PublishedRepoCollection() + + published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } + err = taskCollection.LoadComplete(published, taskCollectionFactory) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + // Capture MultiDist before mutations to detect a false→true transition. + prevMultiDist := published.MultiDist + + // Apply field mutations on the freshly loaded object. + if b.SkipContents != nil { + published.SkipContents = *b.SkipContents + } + if b.SkipBz2 != nil { + published.SkipBz2 = *b.SkipBz2 + } + if b.AcquireByHash != nil { + published.AcquireByHash = *b.AcquireByHash + } + if b.MultiDist != nil { + published.MultiDist = *b.MultiDist + } + revision := published.ObtainRevision() sources := revision.Sources @@ -485,17 +530,17 @@ func apiPublishUpdateSwitch(c *gin.Context) { } } - result, err := published.Update(collectionFactory, out) + result, err := published.Update(taskCollectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath()) + err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - err = collection.Update(published) + err = taskCollection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } @@ -503,10 +548,19 @@ func apiPublishUpdateSwitch(c *gin.Context) { if b.SkipCleanup == nil || !*b.SkipCleanup { cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources)) cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...) - err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out) + err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } + // When MultiDist is toggled, the old pool layout still has files that + // CleanupPrefixComponentFiles won't touch (it only scans the new layout). + // Run a second pass over the previous layout to remove stale files. + if prevMultiDist != published.MultiDist { + err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err) + } + } } return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil @@ -551,10 +605,19 @@ func apiPublishDrop(c *gin.Context) { } resources := []string{string(published.Key())} + // Non-MultiDist distributions share a single pool/ directory under the + // prefix. Acquire the prefix-level pool lock so that a drop cannot race + // with a concurrent update or drop of a sibling distribution during cleanup. + if !published.MultiDist { + resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix())) + } taskName := fmt.Sprintf("Delete published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err := collection.Remove(context, storage, prefix, distribution, - collectionFactory, out, force, skipCleanup) + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.PublishedRepoCollection() + + err := taskCollection.Remove(context, storage, prefix, distribution, + taskCollectionFactory, out, force, skipCleanup) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to drop: %s", err) } @@ -590,43 +653,52 @@ func apiPublishAddSource(c *gin.Context) { storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) + if c.Bind(&b) != nil { + return + } + collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() + // Load shallowly (no LoadComplete) to verify existence and obtain the + // resource key and task name. The actual mutation is performed inside + // the task on a freshly loaded copy to prevent lost-update races. published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to create: %s", err)) return } - err = collection.LoadComplete(published, collectionFactory) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to create: %s", err)) - return - } + resources := []string{string(published.Key())} + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.PublishedRepoCollection() - if c.Bind(&b) != nil { - return - } + published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to create: %s", err) + } - revision := published.ObtainRevision() - sources := revision.Sources + err = taskCollection.LoadComplete(published, taskCollectionFactory) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create: %s", err) + } - component := b.Component - name := b.Name + revision := published.ObtainRevision() + sources := revision.Sources - _, exists := sources[component] - if exists { - AbortWithJSONError(c, http.StatusBadRequest, fmt.Errorf("unable to create: Component '%s' already exists", component)) - return - } + component := b.Component + name := b.Name - sources[component] = name + _, exists := sources[component] + if exists { + return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("unable to create: Component '%s' already exists", component) + } - resources := []string{string(published.Key())} - taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.Update(published) + sources[component] = name + + err = taskCollection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } @@ -708,39 +780,48 @@ func apiPublishSetSources(c *gin.Context) { storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) + if c.Bind(&b) != nil { + return + } + collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() + // Load shallowly for 404 check, resource key, and task name. + // Full load and mutation happen inside the task. published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err)) return } - err = collection.LoadComplete(published, collectionFactory) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err)) - return - } + resources := []string{string(published.Key())} + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.PublishedRepoCollection() - if c.Bind(&b) != nil { - return - } + published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err) + } - revision := published.ObtainRevision() - sources := make(map[string]string, len(b)) - revision.Sources = sources + err = taskCollection.LoadComplete(published, taskCollectionFactory) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } - for _, source := range b { - component := source.Component - name := source.Name - sources[component] = name - } + revision := published.ObtainRevision() + sources := make(map[string]string, len(b)) + revision.Sources = sources - resources := []string{string(published.Key())} - taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.Update(published) + for _, source := range b { + component := source.Component + name := source.Name + sources[component] = name + } + + err = taskCollection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } @@ -773,24 +854,33 @@ func apiPublishDropChanges(c *gin.Context) { collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() + // Load shallowly for 404 check, resource key, and task name. + // Full load and DropRevision happen inside the task. published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err)) return } - err = collection.LoadComplete(published, collectionFactory) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err)) - return - } - - published.DropRevision() - resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.Update(published) + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.PublishedRepoCollection() + + published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err) + } + + err = taskCollection.LoadComplete(published, taskCollectionFactory) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err) + } + + published.DropRevision() + + err = taskCollection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } @@ -826,51 +916,58 @@ func apiPublishUpdateSource(c *gin.Context) { param := slashEscape(c.Params.ByName("prefix")) storage, prefix := deb.ParsePrefix(param) distribution := slashEscape(c.Params.ByName("distribution")) - component := slashEscape(c.Params.ByName("component")) + urlComponent := slashEscape(c.Params.ByName("component")) + + // Default component to the URL path segment; the body may rename it. + b.Component = urlComponent + if c.Bind(&b) != nil { + return + } collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() + // Load shallowly for 404 check, resource key, and task name. + // Full load and mutation happen inside the task. published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err)) return } - err = collection.LoadComplete(published, collectionFactory) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err)) - return - } + resources := []string{string(published.Key())} + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.PublishedRepoCollection() - revision := published.ObtainRevision() - sources := revision.Sources + published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: %s", err) + } - _, exists := sources[component] - if !exists { - AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: Component '%s' does not exist", component)) - return - } + err = taskCollection.LoadComplete(published, taskCollectionFactory) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } - b.Component = component - b.Name = revision.Sources[component] + revision := published.ObtainRevision() + sources := revision.Sources - if c.Bind(&b) != nil { - return - } + _, exists := sources[urlComponent] + if !exists { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to update: Component '%s' does not exist", urlComponent) + } - if b.Component != component { - delete(sources, component) - } + if b.Component != urlComponent { + delete(sources, urlComponent) + } - component = b.Component - name := b.Name - sources[component] = name + newComponent := b.Component + name := b.Name + sources[newComponent] = name - resources := []string{string(published.Key())} - taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.Update(published) + err = taskCollection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } @@ -907,33 +1004,41 @@ func apiPublishRemoveSource(c *gin.Context) { collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() + // Load shallowly for 404 check, resource key, and task name. + // Full load and mutation happen inside the task. published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: %s", err)) return } - err = collection.LoadComplete(published, collectionFactory) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to delete: %s", err)) - return - } + resources := []string{string(published.Key())} + taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.PublishedRepoCollection() - revision := published.ObtainRevision() - sources := revision.Sources + published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: %s", err) + } - _, exists := sources[component] - if !exists { - AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to delete: Component '%s' does not exist", component)) - return - } + err = taskCollection.LoadComplete(published, taskCollectionFactory) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to delete: %s", err) + } - delete(sources, component) + revision := published.ObtainRevision() + sources := revision.Sources - resources := []string{string(published.Key())} - taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) - maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.Update(published) + _, exists := sources[component] + if !exists { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("unable to delete: Component '%s' does not exist", component) + } + + delete(sources, component) + + err = taskCollection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } @@ -997,48 +1102,92 @@ func apiPublishUpdate(c *gin.Context) { collectionFactory := context.NewCollectionFactory() collection := collectionFactory.PublishedRepoCollection() + // Load shallowly for 404 check, resource key, and task name. + // Full load and field mutations happen inside the task. published, err := collection.ByStoragePrefixDistribution(storage, prefix, distribution) if err != nil { AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("unable to update: %s", err)) return } - err = collection.LoadComplete(published, collectionFactory) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to update: %s", err)) - return - } - - if b.SkipContents != nil { - published.SkipContents = *b.SkipContents - } + resources := []string{string(published.Key())} - if b.SkipBz2 != nil { - published.SkipBz2 = *b.SkipBz2 + // Non-MultiDist distributions share a single pool/ directory under the + // prefix. Acquire the prefix-level pool lock so that concurrent updates + // on sibling distributions are serialised and cannot race during cleanup. + if !published.MultiDist { + resources = append(resources, deb.PrefixPoolLockKey(published.StoragePrefix())) } - if b.AcquireByHash != nil { - published.AcquireByHash = *b.AcquireByHash - } + // Lock source repos / snapshots the same way apiPublishUpdateSwitch does, + // because published.Update() reads from them and concurrent modification + // would produce an inconsistent view. + snapshotCollection := collectionFactory.SnapshotCollection() + localRepoCollection := collectionFactory.LocalRepoCollection() - if b.MultiDist != nil { - published.MultiDist = *b.MultiDist + if published.SourceKind == deb.SourceLocalRepo { + for _, uuid := range published.Sources { + repo, err2 := localRepoCollection.ByUUID(uuid) + if err2 != nil { + AbortWithJSONError(c, http.StatusNotFound, err2) + return + } + resources = append(resources, string(repo.Key())) + } + } else if published.SourceKind == deb.SourceSnapshot { + for _, uuid := range published.Sources { + snapshot, err2 := snapshotCollection.ByUUID(uuid) + if err2 != nil { + AbortWithJSONError(c, http.StatusNotFound, err2) + return + } + resources = append(resources, string(snapshot.Key())) + } } - resources := []string{string(published.Key())} taskName := fmt.Sprintf("Update published %s repository %s/%s", published.SourceKind, published.StoragePrefix(), published.Distribution) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - result, err := published.Update(collectionFactory, out) + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.PublishedRepoCollection() + + published, err := taskCollection.ByStoragePrefixDistribution(storage, prefix, distribution) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + err = taskCollection.LoadComplete(published, taskCollectionFactory) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) + } + + // Capture MultiDist before mutations to detect a false→true transition. + prevMultiDist := published.MultiDist + + // Apply field mutations on the freshly loaded object. + if b.SkipContents != nil { + published.SkipContents = *b.SkipContents + } + if b.SkipBz2 != nil { + published.SkipBz2 = *b.SkipBz2 + } + if b.AcquireByHash != nil { + published.AcquireByHash = *b.AcquireByHash + } + if b.MultiDist != nil { + published.MultiDist = *b.MultiDist + } + + result, err := published.Update(taskCollectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - err = published.Publish(context.PackagePool(), context, collectionFactory, signer, out, b.ForceOverwrite, context.SkelPath()) + err = published.Publish(context.PackagePool(), context, taskCollectionFactory, signer, out, b.ForceOverwrite, context.SkelPath()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } - err = collection.Update(published) + err = taskCollection.Update(published) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save to DB: %s", err) } @@ -1046,10 +1195,19 @@ func apiPublishUpdate(c *gin.Context) { if b.SkipCleanup == nil || !*b.SkipCleanup { cleanComponents := make([]string, 0, len(result.UpdatedSources)+len(result.RemovedSources)) cleanComponents = append(append(cleanComponents, result.UpdatedComponents()...), result.RemovedComponents()...) - err = collection.CleanupPrefixComponentFiles(context, published, cleanComponents, collectionFactory, out) + err = taskCollection.CleanupPrefixComponentFiles(context, published, cleanComponents, taskCollectionFactory, out) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to update: %s", err) } + // When MultiDist is toggled, the old pool layout still has files that + // CleanupPrefixComponentFiles won't touch (it only scans the new layout). + // Run a second pass over the previous layout to remove stale files. + if prevMultiDist != published.MultiDist { + err = taskCollection.CleanupAfterMultiDistToggle(context, published, prevMultiDist, cleanComponents, taskCollectionFactory, out) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to clean legacy pool: %s", err) + } + } } return &task.ProcessReturnValue{Code: http.StatusOK, Value: published}, nil diff --git a/api/published_file_missing_test.go b/api/published_file_missing_test.go new file mode 100644 index 0000000000..e25beee594 --- /dev/null +++ b/api/published_file_missing_test.go @@ -0,0 +1,737 @@ +package api + +import ( + "bytes" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "sync" + "time" + + "github.com/aptly-dev/aptly/aptly" + ctx "github.com/aptly-dev/aptly/context" + "github.com/aptly-dev/aptly/deb" + "github.com/gin-gonic/gin" + "github.com/smira/flag" + + . "gopkg.in/check.v1" +) + +// PublishedFileMissingSuite reproduces the exact bug where: +// - Package import succeeds +// - Metadata is updated (Packages.gz shows the package) +// - Publish reports success +// - BUT the .deb file is missing from the published pool directory +// - Result: apt-get returns 404 when trying to download the package +type PublishedFileMissingSuite struct { + context *ctx.AptlyContext + flags *flag.FlagSet + configFile *os.File + router http.Handler + tempDir string + poolPath string + publicPath string +} + +var _ = Suite(&PublishedFileMissingSuite{}) + +func (s *PublishedFileMissingSuite) SetUpSuite(c *C) { + aptly.Version = "publishedFileMissingTest" + + tempDir, err := os.MkdirTemp("", "aptly-published-missing-test") + c.Assert(err, IsNil) + s.tempDir = tempDir + s.poolPath = filepath.Join(tempDir, "pool") + s.publicPath = filepath.Join(tempDir, "public") + + file, err := os.CreateTemp("", "aptly-published-missing-config") + c.Assert(err, IsNil) + s.configFile = file + + config := gin.H{ + "rootDir": tempDir, + "downloadDir": filepath.Join(tempDir, "download"), + "architectures": []string{"amd64"}, + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "gpgDisableSign": true, + "gpgDisableVerify": true, + "gpgProvider": "internal", + "skipLegacyPool": true, + "enableMetricsEndpoint": false, + } + + jsonString, err := json.Marshal(config) + c.Assert(err, IsNil) + _, err = file.Write(jsonString) + c.Assert(err, IsNil) + + flags := flag.NewFlagSet("publishedFileMissingTestFlags", flag.ContinueOnError) + flags.Bool("no-lock", true, "disable database locking for test") + flags.Int("db-open-attempts", 3, "dummy") + flags.String("config", s.configFile.Name(), "config file") + flags.String("architectures", "", "dummy") + s.flags = flags + + context, err := ctx.NewContext(s.flags) + c.Assert(err, IsNil) + + s.context = context + s.router = Router(context) +} + +func (s *PublishedFileMissingSuite) TearDownSuite(c *C) { + if s.configFile != nil { + _ = os.Remove(s.configFile.Name()) + } + if s.context != nil { + s.context.Shutdown() + } + if s.tempDir != "" { + _ = os.RemoveAll(s.tempDir) + } +} + +func (s *PublishedFileMissingSuite) SetUpTest(c *C) { + collectionFactory := s.context.NewCollectionFactory() + + localRepoCollection := collectionFactory.LocalRepoCollection() + _ = localRepoCollection.ForEach(func(repo *deb.LocalRepo) error { + _ = localRepoCollection.Drop(repo) + return nil + }) + + publishedCollection := collectionFactory.PublishedRepoCollection() + _ = publishedCollection.ForEach(func(published *deb.PublishedRepo) error { + _ = publishedCollection.Remove(s.context, published.Storage, published.Prefix, + published.Distribution, collectionFactory, nil, true, true) + return nil + }) +} + +func (s *PublishedFileMissingSuite) TearDownTest(c *C) { + s.SetUpTest(c) +} + +func (s *PublishedFileMissingSuite) httpRequest(c *C, method string, url string, body []byte) *httptest.ResponseRecorder { + w := httptest.NewRecorder() + var req *http.Request + var err error + + if body != nil { + req, err = http.NewRequest(method, url, bytes.NewReader(body)) + } else { + req, err = http.NewRequest(method, url, nil) + } + c.Assert(err, IsNil) + req.Header.Add("Content-Type", "application/json") + s.router.ServeHTTP(w, req) + return w +} + +func (s *PublishedFileMissingSuite) createDebPackage(c *C, uploadID, packageName, version string) { + uploadPath := s.context.UploadPath() + uploadDir := filepath.Join(uploadPath, uploadID) + err := os.MkdirAll(uploadDir, 0755) + c.Assert(err, IsNil) + + tempDir, err := os.MkdirTemp("", "deb-build") + c.Assert(err, IsNil) + defer func() { _ = os.RemoveAll(tempDir) }() + + debianDir := filepath.Join(tempDir, "DEBIAN") + err = os.MkdirAll(debianDir, 0755) + c.Assert(err, IsNil) + + controlContent := fmt.Sprintf(`Package: %s +Version: %s +Section: libs +Priority: optional +Architecture: amd64 +Maintainer: Test +Description: Test package + Test package for published file missing bug. +`, packageName, version) + + err = os.WriteFile(filepath.Join(debianDir, "control"), []byte(controlContent), 0644) + c.Assert(err, IsNil) + + usrDir := filepath.Join(tempDir, "usr", "lib") + err = os.MkdirAll(usrDir, 0755) + c.Assert(err, IsNil) + err = os.WriteFile(filepath.Join(usrDir, "lib.so"), []byte("library"), 0644) + c.Assert(err, IsNil) + + debFile := filepath.Join(uploadDir, fmt.Sprintf("%s_%s_amd64.deb", packageName, version)) + cmd := exec.Command("dpkg-deb", "--build", tempDir, debFile) + err = cmd.Run() + c.Assert(err, IsNil) +} + +// TestPublishedFileGoMissing reproduces the exact production bug +func (s *PublishedFileMissingSuite) TestPublishedFileGoMissing(c *C) { + c.Log("=== Reproducing: Package in metadata but 404 on download ===") + + // Create and publish a repository + repoName := "test-repo" + distribution := "bullseye" + + createBody, _ := json.Marshal(gin.H{ + "Name": repoName, + "DefaultDistribution": distribution, + "DefaultComponent": "main", + }) + resp := s.httpRequest(c, "POST", "/api/repos", createBody) + c.Assert(resp.Code, Equals, 201, Commentf("Failed to create repo: %s", resp.Body.String())) + + publishBody, _ := json.Marshal(gin.H{ + "SourceKind": "local", + "Distribution": distribution, + "Architectures": []string{"amd64"}, + "Sources": []gin.H{ + {"Component": "main", "Name": repoName}, + }, + "Signing": gin.H{"Skip": true}, + }) + resp = s.httpRequest(c, "POST", "/api/publish/hrt", publishBody) + c.Assert(resp.Code, Equals, 201, Commentf("Failed to publish: %s", resp.Body.String())) + + // Create package + packageName := "hrt-libblobbyclient1" + version := "20250926.152427+hrtdeb11" + uploadID := "test-upload-1" + + s.createDebPackage(c, uploadID, packageName, version) + + // Add package + resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoName, uploadID), nil) + c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package: %s", resp.Body.String())) + + // Update publish + updateBody, _ := json.Marshal(gin.H{ + "Signing": gin.H{"Skip": true}, + "ForceOverwrite": true, + }) + resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/hrt/%s", distribution), updateBody) + c.Assert(resp.Code, Equals, 200, Commentf("Failed to update publish: %s", resp.Body.String())) + + // Now check if the file is actually accessible in the published location + publishedStorage, err := s.context.GetPublishedStorage("") + c.Assert(err, IsNil) + publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() + + // Expected file path: hrt/pool/main/h/hrt-libblobbyclient1/hrt-libblobbyclient1_20250926.152427+hrtdeb11_amd64.deb + expectedPath := filepath.Join(publicPath, "hrt", "pool", "main", "h", packageName, + fmt.Sprintf("%s_%s_amd64.deb", packageName, version)) + + c.Logf("Checking for published file at: %s", expectedPath) + + fileInfo, err := os.Stat(expectedPath) + fileExists := err == nil + + c.Logf("File exists: %v", fileExists) + if fileExists { + c.Logf("File size: %d bytes", fileInfo.Size()) + } + + // Check metadata + resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repoName), nil) + var packages []string + err = json.Unmarshal(resp.Body.Bytes(), &packages) + c.Assert(err, IsNil) + c.Logf("Packages in metadata: %d", len(packages)) + + // THE BUG: Metadata says package exists, but file is missing from published location + if len(packages) > 0 && !fileExists { + c.Logf("★★★ BUG REPRODUCED! ★★★") + c.Logf("Metadata shows %d package(s) but file is missing at: %s", len(packages), expectedPath) + c.Logf("This is exactly what causes: 404 Not Found [IP: 10.20.72.62 3142]") + + c.Fatal("BUG CONFIRMED: Package in metadata but missing from published directory!") + } + + c.Assert(fileExists, Equals, true, Commentf( + "Published file should exist at %s when package is in metadata", expectedPath)) +} + +// TestConcurrentPublishRace tries to trigger the race with concurrent publishes +func (s *PublishedFileMissingSuite) TestConcurrentPublishRace(c *C) { + c.Log("=== Testing concurrent publish race condition ===") + + const numIterations = 4 + + for iteration := 0; iteration < numIterations; iteration++ { + c.Logf("--- Iteration %d/%d ---", iteration+1, numIterations) + + // Create repo + repoName := fmt.Sprintf("race-repo-%d", iteration) + distribution := fmt.Sprintf("dist-%d", iteration) + + createBody, _ := json.Marshal(gin.H{ + "Name": repoName, + "DefaultDistribution": distribution, + "DefaultComponent": "main", + }) + resp := s.httpRequest(c, "POST", "/api/repos", createBody) + c.Assert(resp.Code, Equals, 201) + + publishBody, _ := json.Marshal(gin.H{ + "SourceKind": "local", + "Distribution": distribution, + "Architectures": []string{"amd64"}, + "Sources": []gin.H{ + {"Component": "main", "Name": repoName}, + }, + "Signing": gin.H{"Skip": true}, + }) + resp = s.httpRequest(c, "POST", "/api/publish/concurrent", publishBody) + c.Assert(resp.Code, Equals, 201) + + // Create multiple packages + var wg sync.WaitGroup + numPackages := 5 + + for i := 0; i < numPackages; i++ { + wg.Add(1) + go func(idx int) { + defer wg.Done() + + packageName := fmt.Sprintf("pkg-%d-%d", iteration, idx) + version := "1.0.0" + uploadID := fmt.Sprintf("upload-%d-%d", iteration, idx) + + s.createDebPackage(c, uploadID, packageName, version) + + // Add package + resp := s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoName, uploadID), nil) + c.Logf("Package %d add: %d", idx, resp.Code) + + // Small delay + time.Sleep(time.Duration(5+idx*2) * time.Millisecond) + + // Publish + updateBody, _ := json.Marshal(gin.H{ + "Signing": gin.H{"Skip": true}, + "ForceOverwrite": true, + }) + resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/concurrent/%s", distribution), updateBody) + c.Logf("Publish %d: %d", idx, resp.Code) + }(i) + } + + wg.Wait() + time.Sleep(100 * time.Millisecond) + + // Check all packages + resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repoName), nil) + var packages []string + err := json.Unmarshal(resp.Body.Bytes(), &packages) + c.Assert(err, IsNil) + + // Check published files + publishedStorage, err := s.context.GetPublishedStorage("") + c.Assert(err, IsNil) + publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() + + missingFiles := []string{} + for i := 0; i < numPackages; i++ { + packageName := fmt.Sprintf("pkg-%d-%d", iteration, i) + version := "1.0.0" + + // Calculate pool path + poolSubdir := string(packageName[0]) + expectedPath := filepath.Join(publicPath, "concurrent", "pool", "main", poolSubdir, packageName, + fmt.Sprintf("%s_%s_amd64.deb", packageName, version)) + + if _, err := os.Stat(expectedPath); os.IsNotExist(err) { + missingFiles = append(missingFiles, expectedPath) + } + } + + if len(missingFiles) > 0 { + c.Logf("★★★ BUG DETECTED in iteration %d/%d! ★★★", iteration+1, numIterations) + c.Logf("Metadata shows %d packages, but %d files are MISSING:", len(packages), len(missingFiles)) + for i, f := range missingFiles { + c.Logf(" [iter %d] File MISSING %d/%d: %s", iteration+1, i+1, len(missingFiles), f) + } + + c.Fatalf("BUG REPRODUCED in iteration %d/%d: %d published files missing", iteration+1, numIterations, len(missingFiles)) + } else { + c.Logf("[iter %d/%d] All %d files present - OK", iteration+1, numIterations, numPackages) + } + } + + c.Logf("All %d iterations passed - bug not reproduced with current timing", numIterations) +} + +// TestIdenticalPackageRace tests the specific case of identical SHA256 packages +func (s *PublishedFileMissingSuite) TestIdenticalPackageRace(c *C) { + c.Log("=== AGGRESSIVE test: identical package (same SHA256) race ===") + + const numIterations = 4 + packageName := "shared-package" + + for iter := 0; iter < numIterations; iter++ { + c.Logf("Iteration %d/%d", iter+1, numIterations) + + // Create two repos that will get the SAME package (unique per iteration) + repos := []string{fmt.Sprintf("identical-a-%d", iter), fmt.Sprintf("identical-b-%d", iter)} + dists := []string{fmt.Sprintf("dist-a-%d", iter), fmt.Sprintf("dist-b-%d", iter)} + + for i := range repos { + createBody, _ := json.Marshal(gin.H{ + "Name": repos[i], + "DefaultDistribution": dists[i], + "DefaultComponent": "main", + }) + resp := s.httpRequest(c, "POST", "/api/repos", createBody) + c.Assert(resp.Code, Equals, 201) + + publishBody, _ := json.Marshal(gin.H{ + "SourceKind": "local", + "Distribution": dists[i], + "Architectures": []string{"amd64"}, + "Sources": []gin.H{ + {"Component": "main", "Name": repos[i]}, + }, + "Signing": gin.H{"Skip": true}, + "SkipBz2": true, + }) + resp = s.httpRequest(c, "POST", "/api/publish/identical", publishBody) + c.Assert(resp.Code, Equals, 201) + } + + // Create IDENTICAL package file with UNIQUE VERSION per iteration + version := fmt.Sprintf("1.0.%d", iter) + uploadID1 := fmt.Sprintf("identical-upload-1-%d", iter) + uploadID2 := fmt.Sprintf("identical-upload-2-%d", iter) + + s.createDebPackage(c, uploadID1, packageName, version) + + // Copy to second upload (same SHA256) + uploadPath := s.context.UploadPath() + src := filepath.Join(uploadPath, uploadID1, fmt.Sprintf("%s_%s_amd64.deb", packageName, version)) + destDir := filepath.Join(uploadPath, uploadID2) + err := os.MkdirAll(destDir, 0755) + c.Assert(err, IsNil) + dest := filepath.Join(destDir, fmt.Sprintf("%s_%s_amd64.deb", packageName, version)) + + srcData, readErr := os.ReadFile(src) + c.Assert(readErr, IsNil) + err = os.WriteFile(dest, srcData, 0644) + c.Assert(err, IsNil) + + // Race: add and publish both simultaneously + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repos[0], uploadID1), nil) + updateBody, _ := json.Marshal(gin.H{"Signing": gin.H{"Skip": true}, "ForceOverwrite": true, "SkipBz2": true}) + s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/identical/%s", dists[0]), updateBody) + }() + + go func() { + defer wg.Done() + s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repos[1], uploadID2), nil) + updateBody, _ := json.Marshal(gin.H{"Signing": gin.H{"Skip": true}, "ForceOverwrite": true, "SkipBz2": true}) + s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/identical/%s", dists[1]), updateBody) + }() + + wg.Wait() + time.Sleep(200 * time.Millisecond) + c.Logf("[iter %d] All operations complete", iter) + + // Check the shared pool location + publishedStorage, err := s.context.GetPublishedStorage("") + c.Assert(err, IsNil) + publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() + + poolSubdir := string(packageName[0]) + sharedPoolPath := filepath.Join(publicPath, "identical", "pool", "main", poolSubdir, packageName, + fmt.Sprintf("%s_%s_amd64.deb", packageName, version)) + + fileInfo, err := os.Stat(sharedPoolPath) + fileExists := err == nil + + if fileExists { + c.Logf("[iter %d] File EXISTS at %s (size: %d)", iter, sharedPoolPath, fileInfo.Size()) + } else { + c.Logf("[iter %d] File MISSING at %s (error: %v)", iter, sharedPoolPath, err) + } + + // Check metadata + var packagesA, packagesB []string + resp := s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repos[0]), nil) + err = json.Unmarshal(resp.Body.Bytes(), &packagesA) + c.Assert(err, IsNil) + resp = s.httpRequest(c, "GET", fmt.Sprintf("/api/repos/%s/packages", repos[1]), nil) + err = json.Unmarshal(resp.Body.Bytes(), &packagesB) + c.Assert(err, IsNil) + + c.Logf("[iter %d] Packages in metadata: A=%d, B=%d", iter, len(packagesA), len(packagesB)) + + // THE BUG: Both repos show packages in metadata, but the shared pool file is missing + if (len(packagesA) > 0 || len(packagesB) > 0) && !fileExists { + c.Logf("★★★ BUG REPRODUCED in iteration %d! ★★★", iter+1) + c.Logf("Packages in metadata A: %d, B: %d", len(packagesA), len(packagesB)) + c.Logf("Shared pool file exists: %v", fileExists) + c.Logf("Pool path: %s", sharedPoolPath) + + // List what files ARE in the pool directory + poolDir := filepath.Dir(sharedPoolPath) + if entries, err := os.ReadDir(poolDir); err == nil { + c.Logf("Files in pool directory %s:", poolDir) + for _, entry := range entries { + c.Logf(" - %s", entry.Name()) + } + } + + c.Fatalf("Metadata shows packages but shared pool file is missing (iteration %d)", iter+1) + } + } + + c.Logf("All %d iterations passed - bug not reproduced", numIterations) +} + +// TestConcurrentSnapshotPublishToSamePrefix reproduces the EXACT production bug: +// Multiple snapshots are published concurrently to the SAME prefix but different distributions. +// Example from production logs: +// - trixie-pgdg published to "external/postgres-auto/trixie" +// - bullseye-pgdg published to "external/postgres-auto/bullseye" +// Both share the same pool directory, causing cleanup race conditions. +func (s *PublishedFileMissingSuite) TestConcurrentSnapshotPublishToSamePrefix(c *C) { + const numIterations = 4 + + for iter := 0; iter < numIterations; iter++ { + c.Logf("--- Iteration %d/%d ---", iter+1, numIterations) + + // Create two repos with different packages (simulating trixie-pgdg and bullseye-pgdg) + repoTrixie := fmt.Sprintf("trixie-pgdg-%d", iter) + repoBullseye := fmt.Sprintf("bullseye-pgdg-%d", iter) + + // Create trixie repo + createBody, _ := json.Marshal(gin.H{ + "Name": repoTrixie, + "DefaultDistribution": "trixie", + "DefaultComponent": "main", + }) + resp := s.httpRequest(c, "POST", "/api/repos", createBody) + c.Assert(resp.Code, Equals, 201, Commentf("Failed to create trixie repo")) + + // Create bullseye repo + createBody, _ = json.Marshal(gin.H{ + "Name": repoBullseye, + "DefaultDistribution": "bullseye", + "DefaultComponent": "main", + }) + resp = s.httpRequest(c, "POST", "/api/repos", createBody) + c.Assert(resp.Code, Equals, 201, Commentf("Failed to create bullseye repo")) + + // Add packages to both repos + numPackages := 3 + + // Add packages to trixie repo + for i := 0; i < numPackages; i++ { + packageName := fmt.Sprintf("postgresql-17-trixie-pkg%d", i) + version := fmt.Sprintf("17.0.%d", iter) + uploadID := fmt.Sprintf("trixie-upload-%d-%d", iter, i) + + s.createDebPackage(c, uploadID, packageName, version) + resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoTrixie, uploadID), nil) + c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package to trixie")) + } + + // Add packages to bullseye repo + for i := 0; i < numPackages; i++ { + packageName := fmt.Sprintf("postgresql-17-bullseye-pkg%d", i) + version := fmt.Sprintf("17.0.%d", iter) + uploadID := fmt.Sprintf("bullseye-upload-%d-%d", iter, i) + + s.createDebPackage(c, uploadID, packageName, version) + resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/file/%s?noRemove=0", repoBullseye, uploadID), nil) + c.Assert(resp.Code, Equals, 200, Commentf("Failed to add package to bullseye")) + } + + // Create snapshots from both repos + snapshotTrixie := fmt.Sprintf("%s-snap", repoTrixie) + snapshotBullseye := fmt.Sprintf("%s-snap", repoBullseye) + + createSnapshotBody, _ := json.Marshal(gin.H{"Name": snapshotTrixie}) + resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/snapshots", repoTrixie), createSnapshotBody) + c.Assert(resp.Code, Equals, 201, Commentf("Failed to create trixie snapshot")) + + createSnapshotBody, _ = json.Marshal(gin.H{"Name": snapshotBullseye}) + resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/repos/%s/snapshots", repoBullseye), createSnapshotBody) + c.Assert(resp.Code, Equals, 201, Commentf("Failed to create bullseye snapshot")) + + // Publish both snapshots CONCURRENTLY to the SAME prefix + // This mimics production where both are published to "external/postgres-auto" + // Use the SAME prefix across all iterations to trigger the race more aggressively + sharedPrefix := "postgres-auto" + + var wg sync.WaitGroup + var trixiePublishCode, bullseyePublishCode int + + wg.Add(2) + + // Publish or update trixie snapshot + go func() { + defer wg.Done() + + var resp *httptest.ResponseRecorder + if iter == 0 { + // First iteration: CREATE + publishBody, _ := json.Marshal(gin.H{ + "SourceKind": "snapshot", + "Distribution": "trixie", + "Architectures": []string{"amd64"}, + "Sources": []gin.H{ + {"Name": snapshotTrixie}, + }, + "Signing": gin.H{"Skip": true}, + "SkipBz2": true, + "ForceOverwrite": true, + "SkipCleanup": false, // Force cleanup to run + }) + resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/publish/%s", sharedPrefix), publishBody) + } else { + // Subsequent iterations: UPDATE (this is what happens in production) + updateBody, _ := json.Marshal(gin.H{ + "Snapshots": []gin.H{ + {"Component": "main", "Name": snapshotTrixie}, + }, + "Signing": gin.H{"Skip": true}, + "SkipBz2": true, + "ForceOverwrite": true, + "SkipCleanup": false, + }) + resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/%s/trixie", sharedPrefix), updateBody) + } + trixiePublishCode = resp.Code + c.Logf("[iter %d] Trixie publish/update completed: %d", iter, resp.Code) + }() + + // Publish or update bullseye snapshot + go func() { + defer wg.Done() + + var resp *httptest.ResponseRecorder + if iter == 0 { + // First iteration: CREATE + publishBody, _ := json.Marshal(gin.H{ + "SourceKind": "snapshot", + "Distribution": "bullseye", + "Architectures": []string{"amd64"}, + "Sources": []gin.H{ + {"Name": snapshotBullseye}, + }, + "Signing": gin.H{"Skip": true}, + "SkipBz2": true, + "ForceOverwrite": true, + "SkipCleanup": false, + }) + resp = s.httpRequest(c, "POST", fmt.Sprintf("/api/publish/%s", sharedPrefix), publishBody) + } else { + // Subsequent iterations: UPDATE + updateBody, _ := json.Marshal(gin.H{ + "Snapshots": []gin.H{ + {"Component": "main", "Name": snapshotBullseye}, + }, + "Signing": gin.H{"Skip": true}, + "SkipBz2": true, + "ForceOverwrite": true, + "SkipCleanup": false, + }) + resp = s.httpRequest(c, "PUT", fmt.Sprintf("/api/publish/%s/bullseye", sharedPrefix), updateBody) + } + bullseyePublishCode = resp.Code + c.Logf("[iter %d] Bullseye publish/update completed: %d", iter, resp.Code) + }() + + wg.Wait() + time.Sleep(50 * time.Millisecond) + + // Verify publishes succeeded (201 for create, 200 for update) + expectedCode := 201 + if iter > 0 { + expectedCode = 200 + } + c.Assert(trixiePublishCode, Equals, expectedCode, Commentf("Trixie publish/update should succeed")) + c.Assert(bullseyePublishCode, Equals, expectedCode, Commentf("Bullseye publish/update should succeed")) + + // Verify ALL package files exist in the published pool + publishedStorage, err := s.context.GetPublishedStorage("") + c.Assert(err, IsNil) + publicPath := publishedStorage.(aptly.FileSystemPublishedStorage).PublicPath() + + missingFiles := []string{} + expectedFiles := []string{} + + // Check trixie packages + for i := 0; i < numPackages; i++ { + packageName := fmt.Sprintf("postgresql-17-trixie-pkg%d", i) + version := fmt.Sprintf("17.0.%d", iter) + + poolSubdir := string(packageName[0]) + expectedPath := filepath.Join(publicPath, sharedPrefix, "pool", "main", poolSubdir, packageName, + fmt.Sprintf("%s_%s_amd64.deb", packageName, version)) + + expectedFiles = append(expectedFiles, expectedPath) + if _, err := os.Stat(expectedPath); os.IsNotExist(err) { + missingFiles = append(missingFiles, fmt.Sprintf("TRIXIE: %s", filepath.Base(expectedPath))) + } + } + + // Check bullseye packages + for i := 0; i < numPackages; i++ { + packageName := fmt.Sprintf("postgresql-17-bullseye-pkg%d", i) + version := fmt.Sprintf("17.0.%d", iter) + + poolSubdir := string(packageName[0]) + expectedPath := filepath.Join(publicPath, sharedPrefix, "pool", "main", poolSubdir, packageName, + fmt.Sprintf("%s_%s_amd64.deb", packageName, version)) + + expectedFiles = append(expectedFiles, expectedPath) + if _, err := os.Stat(expectedPath); os.IsNotExist(err) { + missingFiles = append(missingFiles, fmt.Sprintf("BULLSEYE: %s", filepath.Base(expectedPath))) + } + } + + // BUG: Files from one distribution are deleted by the other's cleanup + if len(missingFiles) > 0 { + c.Logf("★★★ BUG REPRODUCED in iteration %d/%d! ★★★", iter+1, numIterations) + c.Logf("Both publishes to prefix '%s' succeeded, but %d files are MISSING:", sharedPrefix, len(missingFiles)) + for i, f := range missingFiles { + c.Logf(" Missing file %d/%d: %s", i+1, len(missingFiles), f) + } + + c.Logf("\nThis reproduces the exact production bug where:") + c.Logf(" 1. Mirror updates complete successfully") + c.Logf(" 2. Snapshots are created") + c.Logf(" 3. Both snapshots publish to same prefix (different distributions)") + c.Logf(" 4. Cleanup from one publish DELETES files from the other") + c.Logf(" 5. Result: apt-get returns 404 when downloading packages") + + // List what's actually in the pool + poolDir := filepath.Join(publicPath, sharedPrefix, "pool", "main") + if entries, err := os.ReadDir(poolDir); err == nil { + c.Logf("\nActual pool directory contents (%s):", poolDir) + for _, entry := range entries { + c.Logf(" - %s/", entry.Name()) + } + } + + c.Fatalf("BUG CONFIRMED (iteration %d/%d): %d files missing from shared pool", + iter+1, numIterations, len(missingFiles)) + } else { + c.Logf("[iter %d/%d] All %d files present - OK", iter+1, numIterations, len(expectedFiles)) + } + } + c.Logf("✓ All %d iterations passed - no files missing", numIterations) +} diff --git a/api/repos.go b/api/repos.go index 0ced5efd6f..c3bcfbe309 100644 --- a/api/repos.go +++ b/api/repos.go @@ -24,7 +24,7 @@ import ( // @Tags Repos // @Produce html // @Success 200 {object} string "HTML" -// @Router /api/repos [get] +// @Router /repos [get] func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.HandlerFunc { return func(c *gin.Context) { c.Writer.Header().Set("Content-Type", "text/html; charset=utf-8") @@ -49,7 +49,7 @@ func reposListInAPIMode(localRepos map[string]utils.FileSystemPublishRoot) gin.H // @Param pkgPath path string true "Package Path" allowReserved=true // @Produce json // @Success 200 "" -// @Router /api/{storage}/{pkgPath} [get] +// @Router /repos/{storage}/{pkgPath} [get] func reposServeInAPIMode(c *gin.Context) { pkgpath := c.Param("pkgPath") @@ -60,7 +60,12 @@ func reposServeInAPIMode(c *gin.Context) { storage = "filesystem:" + storage } - publicPath := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage).PublicPath() + ps, err := context.GetPublishedStorage(storage) + if err != nil { + AbortWithJSONError(c, http.StatusNotFound, err) + return + } + publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath() c.FileFromFS(pkgpath, http.Dir(publicPath)) } @@ -93,7 +98,7 @@ type repoCreateParams struct { DefaultDistribution string ` json:"DefaultDistribution" example:"stable"` // Default component when publishing from this local repo DefaultComponent string ` json:"DefaultComponent" example:"main"` - // Snapshot name to create repoitory from (optional) + // Snapshot name to create repository from (optional) FromSnapshot string ` json:"FromSnapshot" example:""` } @@ -122,63 +127,79 @@ func apiReposCreate(c *gin.Context) { return } - repo := deb.NewLocalRepo(b.Name, b.Comment) - repo.DefaultComponent = b.DefaultComponent - repo.DefaultDistribution = b.DefaultDistribution - + // Handler: Pre-task validations (shallow) collectionFactory := context.NewCollectionFactory() + var resources []string if b.FromSnapshot != "" { - var snapshot *deb.Snapshot - - snapshotCollection := collectionFactory.SnapshotCollection() - - snapshot, err := snapshotCollection.ByName(b.FromSnapshot) + snapshot, err := collectionFactory.SnapshotCollection().ByName(b.FromSnapshot) if err != nil { AbortWithJSONError(c, http.StatusNotFound, fmt.Errorf("source snapshot not found: %s", err)) return } + resources = append(resources, string(snapshot.Key())) + } - err = snapshotCollection.LoadComplete(snapshot) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, fmt.Errorf("unable to load source snapshot: %s", err)) - return + taskName := fmt.Sprintf("Create repository %s", b.Name) + + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + // Task: Create fresh collection and check/create ATOMIC inside task + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.LocalRepoCollection() + + // Check duplicate inside lock + if _, err := taskCollection.ByName(b.Name); err == nil { + return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, + fmt.Errorf("local repo with name %s already exists", b.Name) } - repo.UpdateRefList(snapshot.RefList()) - } + // Create repo + repo := deb.NewLocalRepo(b.Name, b.Comment) + repo.DefaultComponent = b.DefaultComponent + repo.DefaultDistribution = b.DefaultDistribution - localRepoCollection := collectionFactory.LocalRepoCollection() + if b.FromSnapshot != "" { + snapshotCollection := taskCollectionFactory.SnapshotCollection() - if _, err := localRepoCollection.ByName(b.Name); err == nil { - AbortWithJSONError(c, http.StatusConflict, fmt.Errorf("local repo with name %s already exists", b.Name)) - return - } + snapshot, err := snapshotCollection.ByName(b.FromSnapshot) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, + fmt.Errorf("source snapshot not found: %s", err) + } - err := localRepoCollection.Add(repo) - if err != nil { - AbortWithJSONError(c, http.StatusInternalServerError, err) - return - } + err = snapshotCollection.LoadComplete(snapshot) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, + fmt.Errorf("unable to load source snapshot: %s", err) + } + + repo.UpdateRefList(snapshot.RefList()) + } + + err := taskCollection.Add(repo) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } - c.JSON(http.StatusCreated, repo) + return &task.ProcessReturnValue{Code: http.StatusCreated, Value: repo}, nil + }) } type reposEditParams struct { // Name of repository to modify - Name *string `binding:"required" json:"Name" example:"repo1"` + Name *string ` json:"Name" example:"new-repo-name"` // Change Comment of repository Comment *string ` json:"Comment" example:"example repo"` // Change Default Distribution for publishing DefaultDistribution *string ` json:"DefaultDistribution" example:""` - // Change Devault Component for publishing + // Change Default Component for publishing DefaultComponent *string ` json:"DefaultComponent" example:""` } // @Summary Update Repository // @Description **Update local repository meta information** // @Tags Repos -// @Param name path string true "Repository name" +// @Param name path string true "Repository name to modify" // @Consume json // @Param request body reposEditParams true "Parameters" // @Produce json @@ -191,42 +212,66 @@ func apiReposEdit(c *gin.Context) { if c.Bind(&b) != nil { return } - + // Load shallowly for 404 check and resource key. + // Mutation and duplicate check happen inside the task for atomicity. collectionFactory := context.NewCollectionFactory() collection := collectionFactory.LocalRepoCollection() - repo, err := collection.ByName(c.Params.ByName("name")) + name := c.Params.ByName("name") + repo, err := collection.ByName(name) if err != nil { AbortWithJSONError(c, 404, err) return } - if b.Name != nil { - _, err := collection.ByName(*b.Name) - if err == nil { - // already exists - AbortWithJSONError(c, 404, err) + if b.Name != nil && *b.Name != name { + if _, err = collection.ByName(*b.Name); err == nil { + AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: local repo %q already exists", *b.Name)) return } - repo.Name = *b.Name - } - if b.Comment != nil { - repo.Comment = *b.Comment - } - if b.DefaultDistribution != nil { - repo.DefaultDistribution = *b.DefaultDistribution - } - if b.DefaultComponent != nil { - repo.DefaultComponent = *b.DefaultComponent } - err = collection.Update(repo) - if err != nil { - AbortWithJSONError(c, 500, err) - return - } + resources := []string{string(repo.Key())} + taskName := fmt.Sprintf("Edit repository %s", name) - c.JSON(200, repo) + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + // Task: Create fresh collection inside task after lock + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.LocalRepoCollection() + + // Fresh load after lock acquired + repo, err := taskCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err + } + + // Check and update ATOMIC (inside lock) + if b.Name != nil && *b.Name != name { + _, err := taskCollection.ByName(*b.Name) + if err == nil { + // already exists + return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, + fmt.Errorf("local repo with name %q already exists", *b.Name) + } + repo.Name = *b.Name + } + if b.Comment != nil { + repo.Comment = *b.Comment + } + if b.DefaultDistribution != nil { + repo.DefaultDistribution = *b.DefaultDistribution + } + if b.DefaultComponent != nil { + repo.DefaultComponent = *b.DefaultComponent + } + + err = taskCollection.Update(repo) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + + return &task.ProcessReturnValue{Code: http.StatusOK, Value: repo}, nil + }) } // GET /api/repos/:name @@ -268,10 +313,10 @@ func apiReposDrop(c *gin.Context) { force := c.Request.URL.Query().Get("force") == "1" name := c.Params.ByName("name") + // Load shallowly for 404 check, resource key, and task name. + // Full checks (published/snapshots) happen inside the task. collectionFactory := context.NewCollectionFactory() collection := collectionFactory.LocalRepoCollection() - snapshotCollection := collectionFactory.SnapshotCollection() - publishedCollection := collectionFactory.PublishedRepoCollection() repo, err := collection.ByName(name) if err != nil { @@ -282,19 +327,32 @@ func apiReposDrop(c *gin.Context) { resources := []string{string(repo.Key())} taskName := fmt.Sprintf("Delete repo %s", name) maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - published := publishedCollection.ByLocalRepo(repo) + // Task: Create fresh collections inside task after lock acquired + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.LocalRepoCollection() + taskSnapshotCollection := taskCollectionFactory.SnapshotCollection() + taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection() + + // Re-read repo with fresh collection after lock + repo, err := taskCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: %s", err) + } + + // Check with fresh collections + published := taskPublishedCollection.ByLocalRepo(repo) if len(published) > 0 { return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo is published") } if !force { - snapshots := snapshotCollection.ByLocalRepoSource(repo) + snapshots := taskSnapshotCollection.ByLocalRepoSource(repo) if len(snapshots) > 0 { return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop, local repo has snapshots, use ?force=1 to override") } } - return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, collection.Drop(repo) + return &task.ProcessReturnValue{Code: http.StatusOK, Value: gin.H{}}, taskCollection.Drop(repo) }) } @@ -351,10 +409,13 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li return } + // Load shallowly for 404 check and resource key. + // Full load and mutations happen inside the task. collectionFactory := context.NewCollectionFactory() collection := collectionFactory.LocalRepoCollection() - repo, err := collection.ByName(c.Params.ByName("name")) + name := c.Params.ByName("name") + repo, err := collection.ByName(name) if err != nil { AbortWithJSONError(c, 404, err) return @@ -363,13 +424,23 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li resources := []string{string(repo.Key())} maybeRunTaskInBackground(c, taskNamePrefix+repo.Name, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.LoadComplete(repo) + // Task: Create fresh factory and collection inside task after lock + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.LocalRepoCollection() + + // Fresh load after lock acquired (use captured `name` variable, not gin context) + repo, err := taskCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, err + } + + err = taskCollection.LoadComplete(repo) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } out.Printf("Loading packages...\n") - list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil) + list, err := deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } @@ -378,7 +449,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li for _, ref := range b.PackageRefs { var p *deb.Package - p, err = collectionFactory.PackageCollection().ByKey([]byte(ref)) + p, err = taskCollectionFactory.PackageCollection().ByKey([]byte(ref)) if err != nil { if err == database.ErrNotFound { return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("packages %s: %s", ref, err) @@ -394,7 +465,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list)) - err = collectionFactory.LocalRepoCollection().Update(repo) + err = taskCollection.Update(repo) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err) } @@ -410,6 +481,7 @@ func apiReposPackagesAddDelete(c *gin.Context, taskNamePrefix string, cb func(li // @Description API verifies that packages actually exist in aptly database and checks constraint that conflicting packages can’t be part of the same local repository. // @Tags Repos // @Param name path string true "Repository name" +// @Consume json // @Param request body reposPackagesAddDeleteParams true "Parameters" // @Param _async query bool false "Run in background and return task object" // @Produce json @@ -455,7 +527,7 @@ func apiReposPackagesDelete(c *gin.Context) { // @Tags Repos // @Param name path string true "Repository name" // @Param dir path string true "Directory of packages" -// @Param file path string false "Filename (optional)" +// @Param file path string true "Filename" // @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 200 {string} string "OK" @@ -500,6 +572,8 @@ func apiReposPackageFromDir(c *gin.Context) { return } + // Load shallowly for 404 check and resource key. + // Full load and mutations happen inside the task. collectionFactory := context.NewCollectionFactory() collection := collectionFactory.LocalRepoCollection() @@ -523,7 +597,17 @@ func apiReposPackageFromDir(c *gin.Context) { resources := []string{string(repo.Key())} resources = append(resources, sources...) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collection.LoadComplete(repo) + // Task: Create fresh factory and collection inside task after lock + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.LocalRepoCollection() + + // Fresh load after lock acquired + repo, err := taskCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + + err = taskCollection.LoadComplete(repo) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } @@ -544,13 +628,13 @@ func apiReposPackageFromDir(c *gin.Context) { packageFiles, otherFiles, failedFiles = deb.CollectPackageFiles(sources, reporter) - list, err := deb.NewPackageListFromRefList(repo.RefList(), collectionFactory.PackageCollection(), nil) + list, err = deb.NewPackageListFromRefList(repo.RefList(), taskCollectionFactory.PackageCollection(), nil) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages: %s", err) } processedFiles, failedFiles2, err = deb.ImportPackageFiles(list, packageFiles, forceReplace, verifier, context.PackagePool(), - collectionFactory.PackageCollection(), reporter, nil, collectionFactory.ChecksumCollection) + taskCollectionFactory.PackageCollection(), reporter, nil, taskCollectionFactory.ChecksumCollection) failedFiles = append(failedFiles, failedFiles2...) processedFiles = append(processedFiles, otherFiles...) @@ -560,7 +644,7 @@ func apiReposPackageFromDir(c *gin.Context) { repo.UpdateRefList(deb.NewPackageRefListFromPackageList(list)) - err = collectionFactory.LocalRepoCollection().Update(repo) + err = taskCollection.Update(repo) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err) } @@ -613,11 +697,11 @@ type reposCopyPackageParams struct { // @Summary Copy Package // @Description Copies a package from a source to destination repository // @Tags Repos -// @Produce json // @Param name path string true "Destination repo" // @Param src path string true "Source repo" // @Param file path string true "File/packages to copy" // @Param _async query bool false "Run in background and return task object" +// @Produce json // @Success 200 {object} task.ProcessReturnValue "msg" // @Failure 400 {object} Error "Bad Request" // @Failure 404 {object} Error "Not Found" @@ -639,6 +723,8 @@ func apiReposCopyPackage(c *gin.Context) { return } + // Load shallowly for 404 check and resource keys. + // Full load and mutations happen inside the task. collectionFactory := context.NewCollectionFactory() dstRepo, err := collectionFactory.LocalRepoCollection().ByName(dstRepoName) if err != nil { @@ -662,12 +748,26 @@ func apiReposCopyPackage(c *gin.Context) { resources := []string{string(dstRepo.Key()), string(srcRepo.Key())} maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collectionFactory.LocalRepoCollection().LoadComplete(dstRepo) + // Task: Create fresh factory and collections inside task after lock + taskCollectionFactory := context.NewCollectionFactory() + + // Fresh load of both repos after lock acquired + dstRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(dstRepoName) if err != nil { return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err) } - err = collectionFactory.LocalRepoCollection().LoadComplete(srcRepo) + srcRepo, err := taskCollectionFactory.LocalRepoCollection().ByName(srcRepoName) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err) + } + + err = taskCollectionFactory.LocalRepoCollection().LoadComplete(dstRepo) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("dest repo error: %s", err) + } + + err = taskCollectionFactory.LocalRepoCollection().LoadComplete(srcRepo) if err != nil { return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, fmt.Errorf("src repo error: %s", err) } @@ -680,12 +780,12 @@ func apiReposCopyPackage(c *gin.Context) { RemovedLines: []string{}, } - dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), collectionFactory.PackageCollection(), context.Progress()) + dstList, err := deb.NewPackageListFromRefList(dstRepo.RefList(), taskCollectionFactory.PackageCollection(), context.Progress()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in dest: %s", err) } - srcList, err := deb.NewPackageListFromRefList(srcRefList, collectionFactory.PackageCollection(), context.Progress()) + srcList, err := deb.NewPackageListFromRefList(srcRefList, taskCollectionFactory.PackageCollection(), context.Progress()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to load packages in src: %s", err) } @@ -753,7 +853,7 @@ func apiReposCopyPackage(c *gin.Context) { } else { dstRepo.UpdateRefList(deb.NewPackageRefListFromPackageList(dstList)) - err = collectionFactory.LocalRepoCollection().Update(dstRepo) + err = taskCollectionFactory.LocalRepoCollection().Update(dstRepo) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to save: %s", err) } @@ -856,6 +956,9 @@ func apiReposIncludePackageFromDir(c *gin.Context) { resources = append(resources, sources...) maybeRunTaskInBackground(c, taskName, resources, func(out aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { + // Task: Create fresh factory and collection inside task after lock + taskCollectionFactory := context.NewCollectionFactory() + var ( err error verifier = context.GetVerifier() @@ -871,8 +974,8 @@ func apiReposIncludePackageFromDir(c *gin.Context) { changesFiles, failedFiles = deb.CollectChangesFiles(sources, reporter) _, failedFiles2, err = deb.ImportChangesFiles( changesFiles, reporter, acceptUnsigned, ignoreSignature, forceReplace, noRemoveFiles, verifier, - repoTemplate, context.Progress(), collectionFactory.LocalRepoCollection(), collectionFactory.PackageCollection(), - context.PackagePool(), collectionFactory.ChecksumCollection, nil, query.Parse) + repoTemplate, context.Progress(), taskCollectionFactory.LocalRepoCollection(), taskCollectionFactory.PackageCollection(), + context.PackagePool(), taskCollectionFactory.ChecksumCollection, nil, query.Parse) failedFiles = append(failedFiles, failedFiles2...) if err != nil { @@ -901,10 +1004,10 @@ func apiReposIncludePackageFromDir(c *gin.Context) { out.Printf("Failed files: %s\n", strings.Join(failedFiles, ", ")) } - ret := reposIncludePackageFromDirResponse{ + ret := reposIncludePackageFromDirResponse{ Report: reporter, FailedFiles: failedFiles, - } + } return &task.ProcessReturnValue{Code: http.StatusOK, Value: ret}, nil }) } diff --git a/api/router.go b/api/router.go index 3cd7d42710..38de11abac 100644 --- a/api/router.go +++ b/api/router.go @@ -11,13 +11,19 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog/log" - "github.com/aptly-dev/aptly/docs" - swaggerFiles "github.com/swaggo/files" - ginSwagger "github.com/swaggo/gin-swagger" + // _ "github.com/aptly-dev/aptly/docs" // import docs + // swaggerFiles "github.com/swaggo/files" + // ginSwagger "github.com/swaggo/gin-swagger" ) var context *ctx.AptlyContext +// @Summary Get Metrics +// @Description **Get Prometheus Metrics** +// @Tags Status +// @Produce text/plain +// @Success 200 {string} string Metrics +// @Router /api/metrics [get] func apiMetricsGet() gin.HandlerFunc { return func(c *gin.Context) { countPackagesByRepos() @@ -25,21 +31,21 @@ func apiMetricsGet() gin.HandlerFunc { } } -func redirectSwagger(c *gin.Context) { - if c.Request.URL.Path == "/docs/index.html" { - c.Redirect(http.StatusMovedPermanently, "/docs.html") - return - } - if c.Request.URL.Path == "/docs/" { - c.Redirect(http.StatusMovedPermanently, "/docs.html") - return - } - if c.Request.URL.Path == "/docs" { - c.Redirect(http.StatusMovedPermanently, "/docs.html") - return - } - c.Next() -} +// func redirectSwagger(c *gin.Context) { +// if c.Request.URL.Path == "/docs/index.html" { +// c.Redirect(http.StatusMovedPermanently, "/docs.html") +// return +// } +// if c.Request.URL.Path == "/docs/" { +// c.Redirect(http.StatusMovedPermanently, "/docs.html") +// return +// } +// if c.Request.URL.Path == "/docs" { +// c.Redirect(http.StatusMovedPermanently, "/docs.html") +// return +// } +// c.Next() +// } // Router returns prebuilt with routes http.Handler func Router(c *ctx.AptlyContext) http.Handler { @@ -63,14 +69,14 @@ func Router(c *ctx.AptlyContext) http.Handler { router.Use(gin.Recovery(), gin.ErrorLogger()) - if c.Config().EnableSwaggerEndpoint { - router.GET("docs.html", func(c *gin.Context) { - c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML) - }) - router.Use(redirectSwagger) - url := ginSwagger.URL("/docs/doc.json") - router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url)) - } + // if c.Config().EnableSwaggerEndpoint { + // router.GET("docs.html", func(c *gin.Context) { + // c.Data(http.StatusOK, "text/html; charset=utf-8", docs.DocsHTML) + // }) + // router.Use(redirectSwagger) + // url := ginSwagger.URL("/docs/doc.json") + // router.GET("/docs/*any", ginSwagger.WrapHandler(swaggerFiles.Handler, url)) + // } if c.Config().EnableMetricsEndpoint { MetricsCollectorRegistrar.Register(router) diff --git a/api/s3.go b/api/s3.go index f38b084706..be68e09d98 100644 --- a/api/s3.go +++ b/api/s3.go @@ -14,7 +14,8 @@ import ( // @Router /api/s3 [get] func apiS3List(c *gin.Context) { keys := []string{} - for k := range context.Config().S3PublishRoots { + s3Roots := context.Config().S3PublishRoots + for k := range s3Roots { keys = append(keys, k) } c.JSON(200, keys) diff --git a/api/snapshot.go b/api/snapshot.go index be7297177e..9cd3670553 100644 --- a/api/snapshot.go +++ b/api/snapshot.go @@ -74,26 +74,33 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { } collectionFactory := context.NewCollectionFactory() - collection := collectionFactory.RemoteRepoCollection() - snapshotCollection := collectionFactory.SnapshotCollection() name := c.Params.ByName("name") - repo, err = collection.ByName(name) + repo, err = collectionFactory.RemoteRepoCollection().ByName(name) if err != nil { AbortWithJSONError(c, 404, err) return } // including snapshot resource key - resources := []string{string(repo.Key()), "S" + b.Name} + resources := []string{string(repo.Key())} taskName := fmt.Sprintf("Create snapshot of mirror %s", name) maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err := repo.CheckLock() + taskCollectionFactory := context.NewCollectionFactory() + taskMirrorCollection := taskCollectionFactory.RemoteRepoCollection() + taskSnapshotCollection := taskCollectionFactory.SnapshotCollection() + + repo, err := taskMirrorCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + + err = repo.CheckLock() if err != nil { return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, err } - err = collection.LoadComplete(repo) + err = taskMirrorCollection.LoadComplete(repo) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } @@ -107,7 +114,7 @@ func apiSnapshotsCreateFromMirror(c *gin.Context) { snapshot.Description = b.Description } - err = snapshotCollection.Add(snapshot) + err = taskSnapshotCollection.Add(snapshot) if err != nil { return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err } @@ -156,6 +163,7 @@ func apiSnapshotsCreate(c *gin.Context) { } } + // Phase 1: Pre-task validation (shallow load for 404 checks only) collectionFactory := context.NewCollectionFactory() snapshotCollection := collectionFactory.SnapshotCollection() var resources []string @@ -169,37 +177,62 @@ func apiSnapshotsCreate(c *gin.Context) { return } - resources = append(resources, string(sources[i].ResourceKey())) + resources = append(resources, string(sources[i].Key())) } maybeRunTaskInBackground(c, "Create snapshot "+b.Name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - for i := range sources { - err = snapshotCollection.LoadComplete(sources[i]) + // Phase 2: Inside task lock - create fresh factory + taskCollectionFactory := context.NewCollectionFactory() + taskSnapshotCollection := taskCollectionFactory.SnapshotCollection() + taskPackageCollection := taskCollectionFactory.PackageCollection() + + // Fresh load of all sources after lock acquired + freshSources := make([]*deb.Snapshot, len(b.SourceSnapshots)) + for i := range b.SourceSnapshots { + freshSources[i], err = taskSnapshotCollection.ByName(b.SourceSnapshots[i]) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } - } - - list := deb.NewPackageList() - - // verify package refs and build package list - for _, ref := range b.PackageRefs { - p, err := collectionFactory.PackageCollection().ByKey([]byte(ref)) + // LoadComplete on fresh copy + err = taskSnapshotCollection.LoadComplete(freshSources[i]) if err != nil { - if err == database.ErrNotFound { - return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err) - } return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } - err = list.Add(p) - if err != nil { - return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err + } + + // Merge packages from all source snapshots + var refList *deb.PackageRefList + if len(freshSources) > 0 { + refList = freshSources[0].RefList() + for i := 1; i < len(freshSources); i++ { + refList = refList.Merge(freshSources[i].RefList(), true, false) } + } else { + refList = deb.NewPackageRefList() + } + + // Add any explicitly specified package refs on top + if len(b.PackageRefs) > 0 { + list := deb.NewPackageList() + for _, ref := range b.PackageRefs { + p, err := taskPackageCollection.ByKey([]byte(ref)) + if err != nil { + if err == database.ErrNotFound { + return &task.ProcessReturnValue{Code: http.StatusNotFound, Value: nil}, fmt.Errorf("package %s: %s", ref, err) + } + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + err = list.Add(p) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err + } + } + refList = refList.Merge(deb.NewPackageRefListFromPackageList(list), true, false) } - snapshot = deb.NewSnapshotFromRefList(b.Name, sources, deb.NewPackageRefListFromPackageList(list), b.Description) + snapshot = deb.NewSnapshotFromRefList(b.Name, freshSources, refList, b.Description) - err = snapshotCollection.Add(snapshot) + err = taskSnapshotCollection.Add(snapshot) if err != nil { return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err } @@ -217,10 +250,9 @@ type snapshotsCreateFromRepositoryParams struct { // @Summary Snapshot Repository // @Description **Create a snapshot of a repository by name** // @Tags Snapshots -// @Param name path string true "Repository name" // @Consume json // @Param request body snapshotsCreateFromRepositoryParams true "Parameters" -// @Param name path string true "Name of the snapshot" +// @Param name path string true "Repository name" // @Param _async query bool false "Run in background and return task object" // @Produce json // @Success 201 {object} deb.Snapshot "Created snapshot object" @@ -241,21 +273,28 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { } collectionFactory := context.NewCollectionFactory() - collection := collectionFactory.LocalRepoCollection() - snapshotCollection := collectionFactory.SnapshotCollection() name := c.Params.ByName("name") - repo, err = collection.ByName(name) + repo, err = collectionFactory.LocalRepoCollection().ByName(name) if err != nil { AbortWithJSONError(c, 404, err) return } // including snapshot resource key - resources := []string{string(repo.Key()), "S" + b.Name} + resources := []string{string(repo.Key())} taskName := fmt.Sprintf("Create snapshot of repo %s", name) maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err := collection.LoadComplete(repo) + taskCollectionFactory := context.NewCollectionFactory() + taskRepoCollection := taskCollectionFactory.LocalRepoCollection() + taskSnapshotCollection := taskCollectionFactory.SnapshotCollection() + + repo, err := taskRepoCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + + err = taskRepoCollection.LoadComplete(repo) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } @@ -269,7 +308,7 @@ func apiSnapshotsCreateFromRepository(c *gin.Context) { snapshot.Description = b.Description } - err = snapshotCollection.Add(snapshot) + err = taskSnapshotCollection.Add(snapshot) if err != nil { return &task.ProcessReturnValue{Code: http.StatusBadRequest, Value: nil}, err } @@ -307,6 +346,7 @@ func apiSnapshotsUpdate(c *gin.Context) { return } + // Phase 1: Pre-task validation (shallow load for 404 check only) collectionFactory := context.NewCollectionFactory() collection := collectionFactory.SnapshotCollection() name := c.Params.ByName("name") @@ -317,14 +357,38 @@ func apiSnapshotsUpdate(c *gin.Context) { return } - resources := []string{string(snapshot.ResourceKey()), "S" + b.Name} + // Pre-task validation of new name if provided (skip if renaming to same name) + if b.Name != "" && b.Name != name { + _, err = collection.ByName(b.Name) + if err == nil { + AbortWithJSONError(c, 409, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name)) + return + } + } + + resources := []string{string(snapshot.Key())} taskName := fmt.Sprintf("Update snapshot %s", name) + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - _, err := collection.ByName(b.Name) - if err == nil { - return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name) + // Phase 2: Inside task lock - create fresh factory + taskCollectionFactory := context.NewCollectionFactory() + taskCollection := taskCollectionFactory.SnapshotCollection() + + // Fresh load after lock acquired + snapshot, err = taskCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } + // Fresh duplicate check inside lock + if b.Name != "" { + _, err := taskCollection.ByName(b.Name) + if err == nil { + return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to rename: snapshot %s already exists", b.Name) + } + } + + // Update fresh copy if b.Name != "" { snapshot.Name = b.Name } @@ -333,7 +397,7 @@ func apiSnapshotsUpdate(c *gin.Context) { snapshot.Description = b.Description } - err = collectionFactory.SnapshotCollection().Update(snapshot) + err = taskCollection.Update(snapshot) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } @@ -387,9 +451,9 @@ func apiSnapshotsDrop(c *gin.Context) { name := c.Params.ByName("name") force := c.Request.URL.Query().Get("force") == "1" + // Phase 1: Pre-task validation (shallow load for 404 check only) collectionFactory := context.NewCollectionFactory() snapshotCollection := collectionFactory.SnapshotCollection() - publishedCollection := collectionFactory.PublishedRepoCollection() snapshot, err := snapshotCollection.ByName(name) if err != nil { @@ -397,23 +461,37 @@ func apiSnapshotsDrop(c *gin.Context) { return } - resources := []string{string(snapshot.ResourceKey())} + resources := []string{string(snapshot.Key())} taskName := fmt.Sprintf("Delete snapshot %s", name) + maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - published := publishedCollection.BySnapshot(snapshot) + // Phase 2: Inside task lock - create fresh collections + taskCollectionFactory := context.NewCollectionFactory() + taskSnapshotCollection := taskCollectionFactory.SnapshotCollection() + taskPublishedCollection := taskCollectionFactory.PublishedRepoCollection() + + // Fresh load after lock acquired + snapshot, err := taskSnapshotCollection.ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + + // Fresh checks with current collections + published := taskPublishedCollection.BySnapshot(snapshot) if len(published) > 0 { return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("unable to drop: snapshot is published") } if !force { - snapshots := snapshotCollection.BySnapshotSource(snapshot) + // Using fresh collection for dependency check + snapshots := taskSnapshotCollection.BySnapshotSource(snapshot) if len(snapshots) > 0 { return &task.ProcessReturnValue{Code: http.StatusConflict, Value: nil}, fmt.Errorf("won't delete snapshot that was used as source for other snapshots, use ?force=1 to override") } } - err = snapshotCollection.Drop(snapshot) + err = taskSnapshotCollection.Drop(snapshot) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } @@ -568,6 +646,7 @@ func apiSnapshotsMerge(c *gin.Context) { return } + // Phase 1: Pre-task validation (shallow load for 404 checks only) collectionFactory := context.NewCollectionFactory() snapshotCollection := collectionFactory.SnapshotCollection() @@ -580,36 +659,47 @@ func apiSnapshotsMerge(c *gin.Context) { return } - resources[i] = string(sources[i].ResourceKey()) + resources[i] = string(sources[i].Key()) } maybeRunTaskInBackground(c, "Merge snapshot "+name, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = snapshotCollection.LoadComplete(sources[0]) - if err != nil { - return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err - } - result := sources[0].RefList() - for i := 1; i < len(sources); i++ { - err = snapshotCollection.LoadComplete(sources[i]) + // Phase 2: Inside task lock - create fresh factory + taskCollectionFactory := context.NewCollectionFactory() + taskSnapshotCollection := taskCollectionFactory.SnapshotCollection() + + // Fresh load of all sources inside task + freshSources := make([]*deb.Snapshot, len(body.Sources)) + for i := range body.Sources { + freshSources[i], err = taskSnapshotCollection.ByName(body.Sources[i]) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + // LoadComplete on fresh copy + err = taskSnapshotCollection.LoadComplete(freshSources[i]) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } - result = result.Merge(sources[i].RefList(), overrideMatching, false) + } + + // Merge using fresh sources + result := freshSources[0].RefList() + for i := 1; i < len(freshSources); i++ { + result = result.Merge(freshSources[i].RefList(), overrideMatching, false) } if latest { result.FilterLatestRefs() } - sourceDescription := make([]string, len(sources)) - for i, s := range sources { + sourceDescription := make([]string, len(freshSources)) + for i, s := range freshSources { sourceDescription[i] = fmt.Sprintf("'%s'", s.Name) } - snapshot = deb.NewSnapshotFromRefList(name, sources, result, + snapshot = deb.NewSnapshotFromRefList(name, freshSources, result, fmt.Sprintf("Merged from sources: %s", strings.Join(sourceDescription, ", "))) - err = collectionFactory.SnapshotCollection().Add(snapshot) + err = taskCollectionFactory.SnapshotCollection().Add(snapshot) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, fmt.Errorf("unable to create snapshot: %s", err) } @@ -690,24 +780,32 @@ func apiSnapshotsPull(c *gin.Context) { return } - resources := []string{string(sourceSnapshot.ResourceKey()), string(toSnapshot.ResourceKey())} + resources := []string{string(sourceSnapshot.Key()), string(toSnapshot.Key())} taskName := fmt.Sprintf("Pull snapshot %s into %s and save as %s", body.Source, name, body.Destination) maybeRunTaskInBackground(c, taskName, resources, func(_ aptly.Progress, _ *task.Detail) (*task.ProcessReturnValue, error) { - err = collectionFactory.SnapshotCollection().LoadComplete(toSnapshot) + // Phase 2: Inside task lock - create fresh factory + taskCollectionFactory := context.NewCollectionFactory() + + // Fresh load of snapshots after lock acquired + freshToSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(name) + if err != nil { + return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err + } + freshSourceSnapshot, err := taskCollectionFactory.SnapshotCollection().ByName(body.Source) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } - err = collectionFactory.SnapshotCollection().LoadComplete(sourceSnapshot) + err = taskCollectionFactory.SnapshotCollection().LoadComplete(freshSourceSnapshot) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } // convert snapshots to package list - toPackageList, err := deb.NewPackageListFromRefList(toSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress()) + toPackageList, err := deb.NewPackageListFromRefList(freshToSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } - sourcePackageList, err := deb.NewPackageListFromRefList(sourceSnapshot.RefList(), collectionFactory.PackageCollection(), context.Progress()) + sourcePackageList, err := deb.NewPackageListFromRefList(freshSourceSnapshot.RefList(), taskCollectionFactory.PackageCollection(), context.Progress()) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } @@ -804,10 +902,10 @@ func apiSnapshotsPull(c *gin.Context) { } // Create snapshot - destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{toSnapshot, sourceSnapshot}, toPackageList, - fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", toSnapshot.Name, sourceSnapshot.Name, strings.Join(body.Queries, ", "))) + destinationSnapshot = deb.NewSnapshotFromPackageList(body.Destination, []*deb.Snapshot{freshToSnapshot, freshSourceSnapshot}, toPackageList, + fmt.Sprintf("Pulled into '%s' with '%s' as source, pull request was: '%s'", freshToSnapshot.Name, freshSourceSnapshot.Name, strings.Join(body.Queries, ", "))) - err = collectionFactory.SnapshotCollection().Add(destinationSnapshot) + err = taskCollectionFactory.SnapshotCollection().Add(destinationSnapshot) if err != nil { return &task.ProcessReturnValue{Code: http.StatusInternalServerError, Value: nil}, err } diff --git a/aptly/interfaces.go b/aptly/interfaces.go index 412daecd8a..1412c719bf 100644 --- a/aptly/interfaces.go +++ b/aptly/interfaces.go @@ -95,8 +95,8 @@ type FileSystemPublishedStorage interface { // PublishedStorageProvider is a thing that returns PublishedStorage by name type PublishedStorageProvider interface { - // GetPublishedStorage returns PublishedStorage by name - GetPublishedStorage(name string) PublishedStorage + // GetPublishedStorage returns PublishedStorage by name, or an error if the storage is not configured + GetPublishedStorage(name string) (PublishedStorage, error) } // BarType used to differentiate between different progress bars diff --git a/azure/azure.go b/azure/azure.go index 3f12678b1b..b313f90479 100644 --- a/azure/azure.go +++ b/azure/azure.go @@ -5,35 +5,28 @@ package azure import ( "context" "encoding/hex" - "errors" "fmt" "io" - "os" + "net/url" "path/filepath" "time" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-storage-blob-go/azblob" "github.com/aptly-dev/aptly/aptly" ) func isBlobNotFound(err error) bool { - var respErr *azcore.ResponseError - if errors.As(err, &respErr) { - return respErr.StatusCode == 404 // BlobNotFound - } - return false + storageError, ok := err.(azblob.StorageError) + return ok && storageError.ServiceCode() == azblob.ServiceCodeBlobNotFound } type azContext struct { - client *azblob.Client - container string + container azblob.ContainerURL prefix string } func newAzContext(accountName, accountKey, container, prefix, endpoint string) (*azContext, error) { - cred, err := azblob.NewSharedKeyCredential(accountName, accountKey) + credential, err := azblob.NewSharedKeyCredential(accountName, accountKey) if err != nil { return nil, err } @@ -42,14 +35,15 @@ func newAzContext(accountName, accountKey, container, prefix, endpoint string) ( endpoint = fmt.Sprintf("https://%s.blob.core.windows.net", accountName) } - serviceClient, err := azblob.NewClientWithSharedKeyCredential(endpoint, cred, nil) + url, err := url.Parse(fmt.Sprintf("%s/%s", endpoint, container)) if err != nil { return nil, err } + containerURL := azblob.NewContainerURL(*url, azblob.NewPipeline(credential, azblob.PipelineOptions{})) + result := &azContext{ - client: serviceClient, - container: container, + container: containerURL, prefix: prefix, } @@ -60,6 +54,10 @@ func (az *azContext) blobPath(path string) string { return filepath.Join(az.prefix, path) } +func (az *azContext) blobURL(path string) azblob.BlobURL { + return az.container.NewBlobURL(az.blobPath(path)) +} + func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (paths []string, md5s []string, err error) { const delimiter = "/" paths = make([]string, 0, 1024) @@ -69,33 +67,27 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p prefix += delimiter } - ctx := context.Background() - maxResults := int32(1) - pager := az.client.NewListBlobsFlatPager(az.container, &azblob.ListBlobsFlatOptions{ - Prefix: &prefix, - MaxResults: &maxResults, - Include: azblob.ListBlobsInclude{Metadata: true}, - }) - - // Iterate over each page - for pager.More() { - page, err := pager.NextPage(ctx) + for marker := (azblob.Marker{}); marker.NotDone(); { + listBlob, err := az.container.ListBlobsFlatSegment( + context.Background(), marker, azblob.ListBlobsSegmentOptions{ + Prefix: prefix, + MaxResults: 1, + Details: azblob.BlobListingDetails{Metadata: true}}) if err != nil { return nil, nil, fmt.Errorf("error listing under prefix %s in %s: %s", prefix, az, err) } - for _, blob := range page.Segment.BlobItems { + marker = listBlob.NextMarker + + for _, blob := range listBlob.Segment.BlobItems { if prefix == "" { - paths = append(paths, *blob.Name) + paths = append(paths, blob.Name) } else { - name := *blob.Name - paths = append(paths, name[len(prefix):]) + paths = append(paths, blob.Name[len(prefix):]) } - b := *blob - md5 := b.Properties.ContentMD5 - md5s = append(md5s, fmt.Sprintf("%x", md5)) - + md5s = append(md5s, fmt.Sprintf("%x", blob.Properties.ContentMD5)) } + if progress != nil { time.Sleep(time.Duration(500) * time.Millisecond) progress.AddBar(1) @@ -105,27 +97,28 @@ func (az *azContext) internalFilelist(prefix string, progress aptly.Progress) (p return paths, md5s, nil } -func (az *azContext) putFile(blobName string, source io.Reader, sourceMD5 string) error { - uploadOptions := &azblob.UploadFileOptions{ - BlockSize: 4 * 1024 * 1024, - Concurrency: 8, +func (az *azContext) putFile(blob azblob.BlobURL, source io.Reader, sourceMD5 string) error { + uploadOptions := azblob.UploadStreamToBlockBlobOptions{ + BufferSize: 4 * 1024 * 1024, + MaxBuffers: 8, } - path := az.blobPath(blobName) if len(sourceMD5) > 0 { decodedMD5, err := hex.DecodeString(sourceMD5) if err != nil { return err } - uploadOptions.HTTPHeaders = &blob.HTTPHeaders{ - BlobContentMD5: decodedMD5, + uploadOptions.BlobHTTPHeaders = azblob.BlobHTTPHeaders{ + ContentMD5: decodedMD5, } } - var err error - if file, ok := source.(*os.File); ok { - _, err = az.client.UploadFile(context.TODO(), az.container, path, file, uploadOptions) - } + _, err := azblob.UploadStreamToBlockBlob( + context.Background(), + source, + blob.ToBlockBlobURL(), + uploadOptions, + ) return err } diff --git a/azure/package_pool.go b/azure/package_pool.go index 97be8e63c1..eea67b2a99 100644 --- a/azure/package_pool.go +++ b/azure/package_pool.go @@ -5,6 +5,7 @@ import ( "os" "path/filepath" + "github.com/Azure/azure-storage-blob-go/azblob" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" "github.com/pkg/errors" @@ -40,7 +41,10 @@ func (pool *PackagePool) buildPoolPath(filename string, checksums *utils.Checksu return filepath.Join(hash[0:2], hash[2:4], hash[4:32]+"_"+filename) } -func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly.ChecksumStorage) (*utils.ChecksumInfo, error) { +func (pool *PackagePool) ensureChecksums( + poolPath string, + checksumStorage aptly.ChecksumStorage, +) (*utils.ChecksumInfo, error) { targetChecksums, err := checksumStorage.Get(poolPath) if err != nil { return nil, err @@ -48,7 +52,8 @@ func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly. if targetChecksums == nil { // we don't have checksums stored yet for this file - download, err := pool.az.client.DownloadStream(context.Background(), pool.az.container, poolPath, nil) + blob := pool.az.blobURL(poolPath) + download, err := blob.Download(context.Background(), 0, 0, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) if err != nil { if isBlobNotFound(err) { return nil, nil @@ -58,7 +63,7 @@ func (pool *PackagePool) ensureChecksums(poolPath string, checksumStorage aptly. } targetChecksums = &utils.ChecksumInfo{} - *targetChecksums, err = utils.ChecksumsForReader(download.Body) + *targetChecksums, err = utils.ChecksumsForReader(download.Body(azblob.RetryReaderOptions{})) if err != nil { return nil, errors.Wrapf(err, "error checksumming blob at %s", poolPath) } @@ -87,49 +92,45 @@ func (pool *PackagePool) LegacyPath(_ string, _ *utils.ChecksumInfo) (string, er } func (pool *PackagePool) Size(path string) (int64, error) { - serviceClient := pool.az.client.ServiceClient() - containerClient := serviceClient.NewContainerClient(pool.az.container) - blobClient := containerClient.NewBlobClient(path) - - props, err := blobClient.GetProperties(context.TODO(), nil) + blob := pool.az.blobURL(path) + props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) if err != nil { return 0, errors.Wrapf(err, "error examining %s from %s", path, pool) } - return *props.ContentLength, nil + return props.ContentLength(), nil } func (pool *PackagePool) Open(path string) (aptly.ReadSeekerCloser, error) { + blob := pool.az.blobURL(path) + temp, err := os.CreateTemp("", "blob-download") if err != nil { - return nil, errors.Wrapf(err, "error creating tempfile for %s", path) + return nil, errors.Wrap(err, "error creating temporary file for blob download") } - defer func () { _ = os.Remove(temp.Name()) }() + defer func() { _ = os.Remove(temp.Name()) }() - _, err = pool.az.client.DownloadFile(context.TODO(), pool.az.container, path, temp, nil) + err = azblob.DownloadBlobToFile(context.Background(), blob, 0, 0, temp, azblob.DownloadFromBlobOptions{}) if err != nil { - return nil, errors.Wrapf(err, "error downloading blob %s", path) + return nil, errors.Wrapf(err, "error downloading blob at %s", path) } return temp, nil } func (pool *PackagePool) Remove(path string) (int64, error) { - serviceClient := pool.az.client.ServiceClient() - containerClient := serviceClient.NewContainerClient(pool.az.container) - blobClient := containerClient.NewBlobClient(path) - - props, err := blobClient.GetProperties(context.TODO(), nil) + blob := pool.az.blobURL(path) + props, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) if err != nil { - return 0, errors.Wrapf(err, "error examining %s from %s", path, pool) + return 0, errors.Wrapf(err, "error getting props of %s from %s", path, pool) } - _, err = pool.az.client.DeleteBlob(context.Background(), pool.az.container, path, nil) + _, err = blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) if err != nil { return 0, errors.Wrapf(err, "error deleting %s from %s", path, pool) } - return *props.ContentLength, nil + return props.ContentLength(), nil } func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, _ bool, checksumStorage aptly.ChecksumStorage) (string, error) { @@ -143,6 +144,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check } path := pool.buildPoolPath(basename, checksums) + blob := pool.az.blobURL(path) targetChecksums, err := pool.ensureChecksums(path, checksumStorage) if err != nil { return "", err @@ -158,7 +160,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check } defer func() { _ = source.Close() }() - err = pool.az.putFile(path, source, checksums.MD5) + err = pool.az.putFile(blob, source, checksums.MD5) if err != nil { return "", err } diff --git a/azure/package_pool_test.go b/azure/package_pool_test.go index ef562cb33a..6b1341d44b 100644 --- a/azure/package_pool_test.go +++ b/azure/package_pool_test.go @@ -7,7 +7,7 @@ import ( "path/filepath" "runtime" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-storage-blob-go/azblob" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/files" "github.com/aptly-dev/aptly/utils" @@ -50,10 +50,8 @@ func (s *PackagePoolSuite) SetUpTest(c *C) { s.pool, err = NewPackagePool(s.accountName, s.accountKey, container, "", s.endpoint) c.Assert(err, IsNil) - publicAccessType := azblob.PublicAccessTypeContainer - _, err = s.pool.az.client.CreateContainer(context.TODO(), s.pool.az.container, &azblob.CreateContainerOptions{ - Access: &publicAccessType, - }) + cnt := s.pool.az.container + _, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer) c.Assert(err, IsNil) s.prefixedPool, err = NewPackagePool(s.accountName, s.accountKey, container, prefix, s.endpoint) diff --git a/azure/public.go b/azure/public.go index 6775e14c51..057e469e7e 100644 --- a/azure/public.go +++ b/azure/public.go @@ -3,22 +3,19 @@ package azure import ( "context" "fmt" + "net/http" "os" "path/filepath" "time" - "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/lease" + "github.com/Azure/azure-storage-blob-go/azblob" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" - "github.com/google/uuid" "github.com/pkg/errors" ) // PublishedStorage abstract file system with published files (actually hosted on Azure) type PublishedStorage struct { - // FIXME: unused ???? prefix string az *azContext pathCache map[string]map[string]string } @@ -67,7 +64,7 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err } defer func() { _ = source.Close() }() - err = storage.az.putFile(path, source, sourceMD5) + err = storage.az.putFile(storage.az.blobURL(path), source, sourceMD5) if err != nil { err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s", sourceFilename, storage)) } @@ -77,15 +74,14 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err // RemoveDirs removes directory structure under public path func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error { - path = storage.az.blobPath(path) filelist, err := storage.Filelist(path) if err != nil { return err } for _, filename := range filelist { - blob := filepath.Join(path, filename) - _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, blob, nil) + blob := storage.az.blobURL(filepath.Join(path, filename)) + _, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) if err != nil { return fmt.Errorf("error deleting path %s from %s: %s", filename, storage, err) } @@ -96,8 +92,8 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error // Remove removes single file under public path func (storage *PublishedStorage) Remove(path string) error { - path = storage.az.blobPath(path) - _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, path, nil) + blob := storage.az.blobURL(path) + _, err := blob.Delete(context.Background(), azblob.DeleteSnapshotsOptionNone, azblob.BlobAccessConditions{}) if err != nil { err = errors.Wrap(err, fmt.Sprintf("error deleting %s from %s: %s", path, storage, err)) } @@ -116,8 +112,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, sourcePath string, sourceChecksums utils.ChecksumInfo, force bool) error { relFilePath := filepath.Join(publishedRelPath, fileName) - prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath) - poolPath := storage.az.blobPath(prefixRelFilePath) + // prefixRelFilePath := filepath.Join(publishedPrefix, relFilePath) + // FIXME: check how to integrate publishedPrefix: + poolPath := storage.az.blobPath(fileName) if storage.pathCache == nil { storage.pathCache = make(map[string]map[string]string) @@ -160,7 +157,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, } defer func() { _ = source.Close() }() - err = storage.az.putFile(relFilePath, source, sourceMD5) + err = storage.az.putFile(storage.az.blobURL(relFilePath), source, sourceMD5) if err == nil { pathCache[relFilePath] = sourceMD5 } else { @@ -177,60 +174,57 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { } // Internal copy or move implementation -func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata map[string]*string, move bool) error { +func (storage *PublishedStorage) internalCopyOrMoveBlob(src, dst string, metadata azblob.Metadata, move bool) error { const leaseDuration = 30 - leaseID := uuid.NewString() - serviceClient := storage.az.client.ServiceClient() - containerClient := serviceClient.NewContainerClient(storage.az.container) - srcBlobClient := containerClient.NewBlobClient(src) - blobLeaseClient, err := lease.NewBlobClient(srcBlobClient, &lease.BlobClientOptions{LeaseID: to.Ptr(leaseID)}) - if err != nil { - return fmt.Errorf("error acquiring lease on source blob %s", src) - } - - _, err = blobLeaseClient.AcquireLease(context.Background(), leaseDuration, nil) - if err != nil { - return fmt.Errorf("error acquiring lease on source blob %s", src) + dstBlobURL := storage.az.blobURL(dst) + srcBlobURL := storage.az.blobURL(src) + leaseResp, err := srcBlobURL.AcquireLease(context.Background(), "", leaseDuration, azblob.ModifiedAccessConditions{}) + if err != nil || leaseResp.StatusCode() != http.StatusCreated { + return fmt.Errorf("error acquiring lease on source blob %s", srcBlobURL) } - defer func() { - _, _ = blobLeaseClient.BreakLease(context.Background(), &lease.BlobBreakOptions{BreakPeriod: to.Ptr(int32(60))}) - }() - - dstBlobClient := containerClient.NewBlobClient(dst) - copyResp, err := dstBlobClient.StartCopyFromURL(context.Background(), srcBlobClient.URL(), &blob.StartCopyFromURLOptions{ - Metadata: metadata, - }) - + defer func() { _, _ = srcBlobURL.BreakLease(context.Background(), azblob.LeaseBreakNaturally, azblob.ModifiedAccessConditions{}) }() + srcBlobLeaseID := leaseResp.LeaseID() + + copyResp, err := dstBlobURL.StartCopyFromURL( + context.Background(), + srcBlobURL.URL(), + metadata, + azblob.ModifiedAccessConditions{}, + azblob.BlobAccessConditions{}, + azblob.DefaultAccessTier, + nil) if err != nil { return fmt.Errorf("error copying %s -> %s in %s: %s", src, dst, storage, err) } - copyStatus := *copyResp.CopyStatus + copyStatus := copyResp.CopyStatus() for { - if copyStatus == blob.CopyStatusTypeSuccess { + if copyStatus == azblob.CopyStatusSuccess { if move { - _, err := storage.az.client.DeleteBlob(context.Background(), storage.az.container, src, &blob.DeleteOptions{ - AccessConditions: &blob.AccessConditions{ - LeaseAccessConditions: &blob.LeaseAccessConditions{ - LeaseID: &leaseID, - }, - }, - }) + _, err = srcBlobURL.Delete( + context.Background(), + azblob.DeleteSnapshotsOptionNone, + azblob.BlobAccessConditions{ + LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}, + }) return err } return nil - } else if copyStatus == blob.CopyStatusTypePending { + } else if copyStatus == azblob.CopyStatusPending { time.Sleep(1 * time.Second) - getMetadata, err := dstBlobClient.GetProperties(context.TODO(), nil) + blobPropsResp, err := dstBlobURL.GetProperties( + context.Background(), + azblob.BlobAccessConditions{LeaseAccessConditions: azblob.LeaseAccessConditions{LeaseID: srcBlobLeaseID}}, + azblob.ClientProvidedKeyOptions{}) if err != nil { - return fmt.Errorf("error getting copy progress %s", dst) + return fmt.Errorf("error getting destination blob properties %s", dstBlobURL) } - copyStatus = *getMetadata.CopyStatus + copyStatus = blobPropsResp.CopyStatus() - _, err = blobLeaseClient.RenewLease(context.Background(), nil) + _, err = srcBlobURL.RenewLease(context.Background(), srcBlobLeaseID, azblob.ModifiedAccessConditions{}) if err != nil { - return fmt.Errorf("error renewing source blob lease %s", src) + return fmt.Errorf("error renewing source blob lease %s", srcBlobURL) } } else { return fmt.Errorf("error copying %s -> %s in %s: %s", dst, src, storage, copyStatus) @@ -245,9 +239,7 @@ func (storage *PublishedStorage) RenameFile(oldName, newName string) error { // SymLink creates a copy of src file and adds link information as meta data func (storage *PublishedStorage) SymLink(src string, dst string) error { - metadata := make(map[string]*string) - metadata["SymLink"] = &src - return storage.internalCopyOrMoveBlob(src, dst, metadata, false /* do not remove src */) + return storage.internalCopyOrMoveBlob(src, dst, azblob.Metadata{"SymLink": src}, false /* move */) } // HardLink using symlink functionality as hard links do not exist @@ -257,33 +249,28 @@ func (storage *PublishedStorage) HardLink(src string, dst string) error { // FileExists returns true if path exists func (storage *PublishedStorage) FileExists(path string) (bool, error) { - serviceClient := storage.az.client.ServiceClient() - containerClient := serviceClient.NewContainerClient(storage.az.container) - blobClient := containerClient.NewBlobClient(path) - _, err := blobClient.GetProperties(context.Background(), nil) + blob := storage.az.blobURL(path) + resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) if err != nil { if isBlobNotFound(err) { return false, nil } - return false, fmt.Errorf("error checking if blob %s exists: %v", path, err) + return false, err + } else if resp.StatusCode() == http.StatusOK { + return true, nil } - return true, nil + return false, fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode()) } // ReadLink returns the symbolic link pointed to by path. // This simply reads text file created with SymLink func (storage *PublishedStorage) ReadLink(path string) (string, error) { - serviceClient := storage.az.client.ServiceClient() - containerClient := serviceClient.NewContainerClient(storage.az.container) - blobClient := containerClient.NewBlobClient(path) - props, err := blobClient.GetProperties(context.Background(), nil) + blob := storage.az.blobURL(path) + resp, err := blob.GetProperties(context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) if err != nil { - return "", fmt.Errorf("failed to get blob properties: %v", err) - } - - metadata := props.Metadata - if originalBlob, exists := metadata["original_blob"]; exists { - return *originalBlob, nil + return "", err + } else if resp.StatusCode() != http.StatusOK { + return "", fmt.Errorf("error checking if blob %s exists %d", blob, resp.StatusCode()) } - return "", fmt.Errorf("error reading link %s: %v", path, err) + return resp.NewMetadata()["SymLink"], nil } diff --git a/azure/public_test.go b/azure/public_test.go index 5c912c5119..f58ad51f21 100644 --- a/azure/public_test.go +++ b/azure/public_test.go @@ -7,11 +7,8 @@ import ( "io" "os" "path/filepath" - "bytes" - "github.com/Azure/azure-sdk-for-go/sdk/azcore" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" - "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/Azure/azure-storage-blob-go/azblob" "github.com/aptly-dev/aptly/files" "github.com/aptly-dev/aptly/utils" . "gopkg.in/check.v1" @@ -69,10 +66,8 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) { s.storage, err = NewPublishedStorage(s.accountName, s.accountKey, container, "", s.endpoint) c.Assert(err, IsNil) - publicAccessType := azblob.PublicAccessTypeContainer - _, err = s.storage.az.client.CreateContainer(context.Background(), s.storage.az.container, &azblob.CreateContainerOptions{ - Access: &publicAccessType, - }) + cnt := s.storage.az.container + _, err = cnt.Create(context.Background(), azblob.Metadata{}, azblob.PublicAccessContainer) c.Assert(err, IsNil) s.prefixedStorage, err = NewPublishedStorage(s.accountName, s.accountKey, container, prefix, s.endpoint) @@ -80,39 +75,41 @@ func (s *PublishedStorageSuite) SetUpTest(c *C) { } func (s *PublishedStorageSuite) TearDownTest(c *C) { - _, err := s.storage.az.client.DeleteContainer(context.Background(), s.storage.az.container, nil) + cnt := s.storage.az.container + _, err := cnt.Delete(context.Background(), azblob.ContainerAccessConditions{}) c.Assert(err, IsNil) } func (s *PublishedStorageSuite) GetFile(c *C, path string) []byte { - resp, err := s.storage.az.client.DownloadStream(context.Background(), s.storage.az.container, path, nil) + blob := s.storage.az.container.NewBlobURL(path) + resp, err := blob.Download(context.Background(), 0, azblob.CountToEnd, azblob.BlobAccessConditions{}, false, azblob.ClientProvidedKeyOptions{}) c.Assert(err, IsNil) - data, err := io.ReadAll(resp.Body) + body := resp.Body(azblob.RetryReaderOptions{MaxRetryRequests: 3}) + data, err := io.ReadAll(body) c.Assert(err, IsNil) return data } func (s *PublishedStorageSuite) AssertNoFile(c *C, path string) { - serviceClient := s.storage.az.client.ServiceClient() - containerClient := serviceClient.NewContainerClient(s.storage.az.container) - blobClient := containerClient.NewBlobClient(path) - _, err := blobClient.GetProperties(context.Background(), nil) + _, err := s.storage.az.container.NewBlobURL(path).GetProperties( + context.Background(), azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) c.Assert(err, NotNil) - - storageError, ok := err.(*azcore.ResponseError) + storageError, ok := err.(azblob.StorageError) c.Assert(ok, Equals, true) - c.Assert(storageError.StatusCode, Equals, 404) + c.Assert(string(storageError.ServiceCode()), Equals, string(string(azblob.StorageErrorCodeBlobNotFound))) } func (s *PublishedStorageSuite) PutFile(c *C, path string, data []byte) { hash := md5.Sum(data) - uploadOptions := &azblob.UploadStreamOptions{ - HTTPHeaders: &blob.HTTPHeaders{ - BlobContentMD5: hash[:], - }, - } - reader := bytes.NewReader(data) - _, err := s.storage.az.client.UploadStream(context.Background(), s.storage.az.container, path, reader, uploadOptions) + _, err := azblob.UploadBufferToBlockBlob( + context.Background(), + data, + s.storage.az.container.NewBlockBlobURL(path), + azblob.UploadToBlockBlobOptions{ + BlobHTTPHeaders: azblob.BlobHTTPHeaders{ + ContentMD5: hash[:], + }, + }) c.Assert(err, IsNil) } @@ -333,7 +330,7 @@ func (s *PublishedStorageSuite) TestLinkFromPool(c *C) { // 2nd link from pool, providing wrong path for source file // - // this test should check that file already exists in Azure and skip upload (which would fail if not skipped) + // this test should check that file already exists in S3 and skip upload (which would fail if not skipped) s.prefixedStorage.pathCache = nil err = s.prefixedStorage.LinkFromPool("", filepath.Join("pool", "main", "m/mars-invaders"), "mars-invaders_1.03.deb", pool, "wrong-looks-like-pathcache-doesnt-work", cksum1, false) c.Check(err, IsNil) diff --git a/cmd/publish.go b/cmd/publish.go index 4217ff87a6..6f4904ada4 100644 --- a/cmd/publish.go +++ b/cmd/publish.go @@ -1,6 +1,8 @@ package cmd import ( + "strings" + "github.com/aptly-dev/aptly/pgp" "github.com/smira/commander" "github.com/smira/flag" @@ -12,7 +14,20 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) { } signer := context.GetSigner() - signer.SetKey(flags.Lookup("gpg-key").Value.String()) + + var gpgKeys []string + + // CLI args have priority over config + cliKeys := flags.Lookup("gpg-key").Value.Get().([]string) + if len(cliKeys) > 0 { + gpgKeys = cliKeys + } else if len(context.Config().GpgKeys) > 0 { + gpgKeys = context.Config().GpgKeys + } + + for _, gpgKey := range gpgKeys { + signer.SetKey(gpgKey) + } signer.SetKeyRing(flags.Lookup("keyring").Value.String(), flags.Lookup("secret-keyring").Value.String()) signer.SetPassphrase(flags.Lookup("passphrase").Value.String(), flags.Lookup("passphrase-file").Value.String()) signer.SetBatch(flags.Lookup("batch").Value.Get().(bool)) @@ -26,6 +41,23 @@ func getSigner(flags *flag.FlagSet) (pgp.Signer, error) { } +type gpgKeyFlag struct { + gpgKeys []string +} + +func (k *gpgKeyFlag) Set(value string) error { + k.gpgKeys = append(k.gpgKeys, value) + return nil +} + +func (k *gpgKeyFlag) Get() interface{} { + return k.gpgKeys +} + +func (k *gpgKeyFlag) String() string { + return strings.Join(k.gpgKeys, ",") +} + func makeCmdPublish() *commander.Command { return &commander.Command{ UsageLine: "publish", diff --git a/cmd/publish_repo.go b/cmd/publish_repo.go index 919710226d..9e8457f25f 100644 --- a/cmd/publish_repo.go +++ b/cmd/publish_repo.go @@ -34,7 +34,7 @@ Example: } cmd.Flag.String("distribution", "", "distribution name to publish") cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)") - cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release") + cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)") cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") diff --git a/cmd/publish_snapshot.go b/cmd/publish_snapshot.go index 7e0d8452ea..63f33a910d 100644 --- a/cmd/publish_snapshot.go +++ b/cmd/publish_snapshot.go @@ -190,9 +190,11 @@ func aptlyPublishSnapshotOrRepo(cmd *commander.Command, args []string) error { context.Progress().Printf("\n%s been successfully published.\n", message) - if localStorage, ok := context.GetPublishedStorage(storage).(aptly.FileSystemPublishedStorage); ok { - context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n", - localStorage.PublicPath()) + if ps, err := context.GetPublishedStorage(storage); err == nil { + if localStorage, ok := ps.(aptly.FileSystemPublishedStorage); ok { + context.Progress().Printf("Please setup your webserver to serve directory '%s' with autoindexing.\n", + localStorage.PublicPath()) + } } context.Progress().Printf("Now you can add following line to apt sources:\n") @@ -230,7 +232,7 @@ Example: } cmd.Flag.String("distribution", "", "distribution name to publish") cmd.Flag.String("component", "", "component name to publish (for multi-component publishing, separate components with commas)") - cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release") + cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)") cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") diff --git a/cmd/publish_switch.go b/cmd/publish_switch.go index f39269a16d..fbd8719cec 100644 --- a/cmd/publish_switch.go +++ b/cmd/publish_switch.go @@ -151,7 +151,7 @@ This command would switch published repository (with one component) named ppa/wh `, Flag: *flag.NewFlagSet("aptly-publish-switch", flag.ExitOnError), } - cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release") + cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)") cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") diff --git a/cmd/publish_update.go b/cmd/publish_update.go index 6ea638d41f..1148196be4 100644 --- a/cmd/publish_update.go +++ b/cmd/publish_update.go @@ -115,7 +115,7 @@ Example: `, Flag: *flag.NewFlagSet("aptly-publish-update", flag.ExitOnError), } - cmd.Flag.String("gpg-key", "", "GPG key ID to use when signing the release") + cmd.Flag.Var(&gpgKeyFlag{}, "gpg-key", "GPG key ID to use when signing the release (flag is repeatable, can be specified multiple times)") cmd.Flag.Var(&keyRingsFlag{}, "keyring", "GPG keyring to use (instead of default)") cmd.Flag.String("secret-keyring", "", "GPG secret keyring to use (instead of default)") cmd.Flag.String("passphrase", "", "GPG passphrase for the key (warning: could be insecure)") diff --git a/cmd/serve.go b/cmd/serve.go index be974b007e..16951c5182 100644 --- a/cmd/serve.go +++ b/cmd/serve.go @@ -97,7 +97,11 @@ func aptlyServe(cmd *commander.Command, args []string) error { } } - publicPath := context.GetPublishedStorage("").(aptly.FileSystemPublishedStorage).PublicPath() + ps, err := context.GetPublishedStorage("") + if err != nil { + return err + } + publicPath := ps.(aptly.FileSystemPublishedStorage).PublicPath() ShutdownContext() fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen) diff --git a/console/progress_test.go b/console/progress_test.go index 2b412ecadb..9278012281 100644 --- a/console/progress_test.go +++ b/console/progress_test.go @@ -11,7 +11,7 @@ func Test(t *testing.T) { TestingT(t) } -type ProgressSuite struct {} +type ProgressSuite struct{} var _ = Suite(&ProgressSuite{}) diff --git a/context/context.go b/context/context.go index 0ffc3f7222..febe7f89a7 100644 --- a/context/context.go +++ b/context/context.go @@ -100,6 +100,7 @@ func (context *AptlyContext) config() *utils.ConfigStructure { configLocations := []string{homeLocation, "/usr/local/etc/aptly.conf", "/etc/aptly.conf"} for _, configLocation := range configLocations { + // FIXME: check if exists, check if readable err = utils.LoadConfig(configLocation, &utils.Config) if os.IsPermission(err) || os.IsNotExist(err) { continue @@ -115,7 +116,12 @@ func (context *AptlyContext) config() *utils.ConfigStructure { if err != nil { fmt.Fprintf(os.Stderr, "Config file not found, creating default config at %s\n\n", homeLocation) - _ = utils.SaveConfigRaw(homeLocation, aptly.AptlyConf) + defaultConfig := aptly.AptlyConf + if len(defaultConfig) == 0 { + defaultConfig = []byte("root_dir: \"\"") + } + + _ = utils.SaveConfigRaw(homeLocation, defaultConfig) err = utils.LoadConfig(homeLocation, &utils.Config) if err != nil { Fatal(fmt.Errorf("error loading config file %s: %s", homeLocation, err)) @@ -406,8 +412,8 @@ func (context *AptlyContext) PackagePool() aptly.PackagePool { return context.packagePool } -// GetPublishedStorage returns instance of PublishedStorage -func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedStorage { +// GetPublishedStorage returns instance of PublishedStorage, or an error if the storage is not configured +func (context *AptlyContext) GetPublishedStorage(name string) (aptly.PublishedStorage, error) { context.Lock() defer context.Unlock() @@ -418,14 +424,14 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto } else if strings.HasPrefix(name, "filesystem:") { params, ok := context.config().FileSystemPublishRoots[name[11:]] if !ok { - Fatal(fmt.Errorf("published local storage %v not configured", name[11:])) + return nil, fmt.Errorf("published local storage %v not configured", name[11:]) } publishedStorage = files.NewPublishedStorage(params.RootDir, params.LinkMethod, params.VerifyMethod) } else if strings.HasPrefix(name, "s3:") { params, ok := context.config().S3PublishRoots[name[3:]] if !ok { - Fatal(fmt.Errorf("published S3 storage %v not configured", name[3:])) + return nil, fmt.Errorf("published S3 storage %v not configured", name[3:]) } var err error @@ -435,39 +441,39 @@ func (context *AptlyContext) GetPublishedStorage(name string) aptly.PublishedSto params.EncryptionMethod, params.PlusWorkaround, params.DisableMultiDel, params.ForceSigV2, params.ForceVirtualHostedStyle, params.Debug) if err != nil { - Fatal(err) + return nil, err } } else if strings.HasPrefix(name, "swift:") { params, ok := context.config().SwiftPublishRoots[name[6:]] if !ok { - Fatal(fmt.Errorf("published Swift storage %v not configured", name[6:])) + return nil, fmt.Errorf("published Swift storage %v not configured", name[6:]) } var err error publishedStorage, err = swift.NewPublishedStorage(params.UserName, params.Password, params.AuthURL, params.Tenant, params.TenantID, params.Domain, params.DomainID, params.TenantDomain, params.TenantDomainID, params.Container, params.Prefix) if err != nil { - Fatal(err) + return nil, err } } else if strings.HasPrefix(name, "azure:") { params, ok := context.config().AzurePublishRoots[name[6:]] if !ok { - Fatal(fmt.Errorf("published Azure storage %v not configured", name[6:])) + return nil, fmt.Errorf("published Azure storage %v not configured", name[6:]) } var err error publishedStorage, err = azure.NewPublishedStorage( params.AccountName, params.AccountKey, params.Container, params.Prefix, params.Endpoint) if err != nil { - Fatal(err) + return nil, err } } else { - Fatal(fmt.Errorf("unknown published storage format: %v", name)) + return nil, fmt.Errorf("unknown published storage format: %v", name) } context.publishedStorages[name] = publishedStorage } - return publishedStorage + return publishedStorage, nil } // UploadPath builds path to upload storage diff --git a/context/context_test.go b/context/context_test.go index 16ecbb2b46..db8eae6735 100644 --- a/context/context_test.go +++ b/context/context_test.go @@ -1,8 +1,6 @@ package context import ( - "fmt" - "os" "reflect" "testing" @@ -80,10 +78,9 @@ func (s *AptlyContextSuite) SetUpTest(c *C) { func (s *AptlyContextSuite) TestGetPublishedStorageBadFS(c *C) { // https://github.com/aptly-dev/aptly/issues/711 - // This will fail on account of us not having a config, so the - // storage never exists. - c.Assert(func() { s.context.GetPublishedStorage("filesystem:fuji") }, - FatalErrorPanicMatches, - &FatalError{ReturnCode: 1, Message: fmt.Sprintf("error loading config file %s/.aptly.conf: invalid yaml (EOF) or json (EOF)", - os.Getenv("HOME"))}) + // https://github.com/aptly-dev/aptly/issues/1477 + // GetPublishedStorage must return an error (not panic) when the + // requested storage is not configured. + _, err := s.context.GetPublishedStorage("filesystem:fuji") + c.Assert(err, NotNil) } diff --git a/database/etcddb/database_test.go b/database/etcddb/database_test.go index ce88209c21..c22faa1709 100644 --- a/database/etcddb/database_test.go +++ b/database/etcddb/database_test.go @@ -14,7 +14,7 @@ func Test(t *testing.T) { } type EtcDDBSuite struct { - db database.Storage + db database.Storage } var _ = Suite(&EtcDDBSuite{}) @@ -133,7 +133,7 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) { v, err := s.db.Get(key) c.Assert(err, IsNil) c.Check(v, DeepEquals, value) - err = transaction.Delete(key) + err = transaction.Delete(key) c.Assert(err, IsNil) _, err = transaction.Get(key2) @@ -156,4 +156,3 @@ func (s *EtcDDBSuite) TestTransactionCommit(c *C) { _, err = transaction.Get(key) c.Assert(err, NotNil) } - diff --git a/deb/format.go b/deb/format.go index 95febe35c5..2696c8903a 100644 --- a/deb/format.go +++ b/deb/format.go @@ -288,8 +288,11 @@ func (c *ControlFileReader) ReadStanza() (Stanza, error) { lastField = canonicalCase(parts[0]) lastFieldMultiline = isMultilineField(lastField, c.isRelease) if lastFieldMultiline { - stanza[lastField] = parts[1] - if parts[1] != "" { + // Trim trailing whitespace from the inline value so that + // "Package-List: " does not add empty line + inlineVal := strings.TrimRight(parts[1], " \t") + stanza[lastField] = inlineVal + if inlineVal != "" { stanza[lastField] += "\n" } } else { diff --git a/deb/format_test.go b/deb/format_test.go index 7228566b45..b9d3b0a6dc 100644 --- a/deb/format_test.go +++ b/deb/format_test.go @@ -128,6 +128,35 @@ func (s *ControlFileSuite) TestReadWriteStanza(c *C) { c.Assert(strings.HasPrefix(str, "Package: "), Equals, true) } +// Sources may contain "Package-List: " with a trailing space. +// That trailing space must not be preserved and re-emitted +// as a spurious blank continuation line when the stanza is written back out. +func (s *ControlFileSuite) TestPackageListTrailingSpace(c *C) { + input := "Package-List: \n" + + " bash deb shells required arch=any\n" + + " bash-doc deb doc optional arch=all\n" + + r := NewControlFileReader(bytes.NewBufferString(input), false, false) + stanza, err := r.ReadStanza() + c.Assert(err, IsNil) + + c.Check(stanza["Package-List"], Equals, + " bash deb shells required arch=any\n"+ + " bash-doc deb doc optional arch=all\n") + + buf := &bytes.Buffer{} + w := bufio.NewWriter(buf) + err = stanza.Copy().WriteTo(w, true, false, false) + c.Assert(err, IsNil) + c.Assert(w.Flush(), IsNil) + + written := buf.String() + c.Assert(strings.Contains(written, "Package-List:\n \n"), Equals, false, + Commentf("spurious blank continuation line found in written output:\n%s", written)) + c.Assert(strings.Contains(written, "Package-List:\n bash"), Equals, true, + Commentf("expected Package-List entries not found in written output:\n%s", written)) +} + func (s *ControlFileSuite) TestReadWriteInstallerStanza(c *C) { s.reader = bytes.NewBufferString(installerFile) r := NewControlFileReader(s.reader, false, true) diff --git a/deb/list.go b/deb/list.go index 25a2d2830b..9eda528cba 100644 --- a/deb/list.go +++ b/deb/list.go @@ -598,6 +598,7 @@ func (l *PackageList) Filter(options FilterOptions) (*PackageList, error) { // // when follow-all-variants is enabled, we need to try to expand anyway, // as even if dependency is satisfied now, there might be other ways to satisfy dependency + // FIXME: do not search twice if result.Search(dep, false, true) != nil { if options.DependencyOptions&DepVerboseResolve == DepVerboseResolve && options.Progress != nil { options.Progress.ColoredPrintf("@{y}Already satisfied dependency@|: %s with %s", &dep, result.Search(dep, true, true)) diff --git a/deb/local.go b/deb/local.go index 4be4607697..e0da96f1fc 100644 --- a/deb/local.go +++ b/deb/local.go @@ -168,6 +168,8 @@ func (collection *LocalRepoCollection) Update(repo *LocalRepo) error { // LoadComplete loads additional information for local repo func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error { + repo.packageRefs = &PackageRefList{} + encoded, err := collection.db.Get(repo.RefKey()) if err == database.ErrNotFound { return nil @@ -176,7 +178,6 @@ func (collection *LocalRepoCollection) LoadComplete(repo *LocalRepo) error { return err } - repo.packageRefs = &PackageRefList{} return repo.packageRefs.Decode(encoded) } diff --git a/deb/local_test.go b/deb/local_test.go index 07f818c3d5..4a40f6bdd9 100644 --- a/deb/local_test.go +++ b/deb/local_test.go @@ -133,6 +133,18 @@ func (s *LocalRepoCollectionSuite) TestByUUID(c *C) { c.Assert(r.String(), Equals, repo.String()) } +func (s *LocalRepoCollectionSuite) TestLoadCompleteNoRefKey(c *C) { + repo := NewLocalRepo("local1", "Comment 1") + c.Assert(s.collection.Update(repo), IsNil) + + r, err := s.collection.ByName("local1") + c.Assert(err, IsNil) + + c.Assert(s.collection.LoadComplete(r), IsNil) + c.Assert(r.packageRefs, NotNil) + c.Assert(r.NumPackages(), Equals, 0) +} + func (s *LocalRepoCollectionSuite) TestUpdateLoadComplete(c *C) { repo := NewLocalRepo("local1", "Comment 1") c.Assert(s.collection.Update(repo), IsNil) diff --git a/deb/ppa.go b/deb/ppa.go index 9668468cae..e03860c2d9 100644 --- a/deb/ppa.go +++ b/deb/ppa.go @@ -28,6 +28,11 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib } } + baseurl := config.PpaBaseURL + if baseurl == "" { + baseurl = "http://ppa.launchpad.net" + } + codename := config.PpaCodename if codename == "" { codename, err = getCodename() @@ -39,7 +44,7 @@ func ParsePPA(ppaURL string, config *utils.ConfigStructure) (url string, distrib distribution = codename components = []string{"main"} - url = fmt.Sprintf("http://ppa.launchpad.net/%s/%s/%s", matches[1], matches[2], distributorID) + url = fmt.Sprintf("%s/%s/%s/%s", baseurl, matches[1], matches[2], distributorID) return } diff --git a/deb/publish.go b/deb/publish.go index 46c9557f28..9e4c5ac3d5 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "sort" + "strconv" "strings" "time" @@ -603,6 +604,15 @@ func (p *PublishedRepo) Key() []byte { return []byte("U" + p.StoragePrefix() + ">>" + p.Distribution) } +// PrefixPoolLockKey returns the task-queue resource key that serialises all +// publish operations sharing the same pool directory under storagePrefix. +// It must be held whenever a non-MultiDist publish may read or clean the +// shared pool, to prevent concurrent cleanup runs from deleting each other's +// files. See docs/Resource-Locking.md for the full key-namespace table. +func PrefixPoolLockKey(storagePrefix string) string { + return "P" + storagePrefix +} + // RefKey is a unique id for package reference list func (p *PublishedRepo) RefKey(component string) []byte { return []byte("E" + p.UUID + component) @@ -814,9 +824,12 @@ func (p *PublishedRepo) GetSkelFiles(skelDir string, component string) (map[stri // Publish publishes snapshot (repository) contents, links package files, generates Packages & Release files, signs them func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageProvider aptly.PublishedStorageProvider, collectionFactory *CollectionFactory, signer pgp.Signer, progress aptly.Progress, forceOverwrite bool, skelDir string) error { - publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) + publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage) + if err != nil { + return err + } - err := publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) + err = publishedStorage.MkDir(filepath.Join(p.Prefix, "pool")) if err != nil { return err } @@ -1126,7 +1139,15 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP release["Label"] = p.GetLabel() release["Suite"] = p.GetSuite() release["Codename"] = p.GetCodename() - release["Date"] = time.Now().UTC().Format("Mon, 2 Jan 2006 15:04:05 MST") + datetimeFormat := "Mon, 2 Jan 2006 15:04:05 MST" + + publishDate := time.Now().UTC() + if epoch := os.Getenv("SOURCE_DATE_EPOCH"); epoch != "" { + if sec, err := strconv.ParseInt(epoch, 10, 64); err == nil { + publishDate = time.Unix(sec, 0).UTC() + } + } + release["Date"] = publishDate.Format(datetimeFormat) release["Architectures"] = strings.Join(utils.StrSlicesSubstract(p.Architectures, []string{ArchitectureSource}), " ") if p.AcquireByHash { release["Acquire-By-Hash"] = "yes" @@ -1182,7 +1203,10 @@ func (p *PublishedRepo) Publish(packagePool aptly.PackagePool, publishedStorageP // It can remove prefix fully, and part of pool (for specific component) func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStorageProvider, removePrefix bool, removePoolComponents []string, progress aptly.Progress) error { - publishedStorage := publishedStorageProvider.GetPublishedStorage(p.Storage) + publishedStorage, err := publishedStorageProvider.GetPublishedStorage(p.Storage) + if err != nil { + return err + } // I. Easy: remove whole prefix (meta+packages) if removePrefix { @@ -1195,7 +1219,7 @@ func (p *PublishedRepo) RemoveFiles(publishedStorageProvider aptly.PublishedStor } // II. Medium: remove metadata, it can't be shared as prefix/distribution as unique - err := publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress) + err = publishedStorage.RemoveDirs(filepath.Join(p.Prefix, "dists", p.Distribution), progress) if err != nil { return err } @@ -1522,6 +1546,55 @@ func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix return referencedFiles, nil } +// CleanupAfterMultiDistToggle cleans up stale pool files left behind when the +// MultiDist flag is toggled on a published repository. +// +// - false→true: Publish() wrote packages into pool/// +// but the old flat pool// files were not removed because +// CleanupPrefixComponentFiles only scans the new MultiDist tree. +// A second pass with MultiDist=false cleans the legacy flat layout by +// reusing the existing orphan-detection logic (the repo is now MultiDist=true +// so it is excluded from the referenced-files scan, making its old pool +// entries appear orphaned). +// +// - true→false: Publish() wrote packages into pool// but the old +// per-distribution pool/// directories were not +// removed. The orphan-detection approach cannot be used here because the +// repo's RefList still contains all packages (they just moved locations). +// Instead we directly remove each pool/// directory. +// This is safe because per-distribution pool dirs are exclusive to a single +// prefix+distribution combination — no other published repo can share them. +func (collection *PublishedRepoCollection) CleanupAfterMultiDistToggle(publishedStorageProvider aptly.PublishedStorageProvider, + published *PublishedRepo, prevMultiDist bool, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error { + if prevMultiDist == published.MultiDist { + return nil + } + + if !prevMultiDist && published.MultiDist { + // false→true: use orphan-detection via the existing cleanup, but with + // MultiDist temporarily set to false so it scans the flat pool layout. + legacy := *published + legacy.MultiDist = false + return collection.CleanupPrefixComponentFiles(publishedStorageProvider, &legacy, cleanComponents, collectionFactory, progress) + } + + // true→false: directly remove the per-distribution pool directories. + publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage) + if err != nil { + return err + } + for _, component := range cleanComponents { + poolDir := filepath.Join(published.Prefix, "pool", published.Distribution, component) + if err := publishedStorage.RemoveDirs(poolDir, progress); err != nil { + return err + } + } + // Remove the distribution-level pool dir if it is now empty. + distPoolDir := filepath.Join(published.Prefix, "pool", published.Distribution) + _ = publishedStorage.RemoveDirs(distPoolDir, progress) + return nil +} + // CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(publishedStorageProvider aptly.PublishedStorageProvider, published *PublishedRepo, cleanComponents []string, collectionFactory *CollectionFactory, progress aptly.Progress) error { @@ -1535,7 +1608,10 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(published distribution := published.Distribution rootPath := filepath.Join(prefix, "dists", distribution) - publishedStorage := publishedStorageProvider.GetPublishedStorage(published.Storage) + publishedStorage, err := publishedStorageProvider.GetPublishedStorage(published.Storage) + if err != nil { + return err + } sort.Strings(cleanComponents) publishedComponents := published.Components() diff --git a/deb/publish_test.go b/deb/publish_test.go index 0bfa08d7cc..7ad63e4d83 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -61,12 +61,12 @@ type FakeStorageProvider struct { storages map[string]aptly.PublishedStorage } -func (p *FakeStorageProvider) GetPublishedStorage(name string) aptly.PublishedStorage { +func (p *FakeStorageProvider) GetPublishedStorage(name string) (aptly.PublishedStorage, error) { storage, ok := p.storages[name] if !ok { - panic(fmt.Sprintf("unknown storage: %#v", name)) + return nil, fmt.Errorf("unknown storage: %#v", name) } - return storage + return storage, nil } type PublishedRepoSuite struct { @@ -433,6 +433,47 @@ func (s *PublishedRepoSuite) TestPublishNoSigner(c *C) { c.Check(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/main/binary-i386/Release"), PathExists) } +func (s *PublishedRepoSuite) TestPublishSourceDateEpoch(c *C) { + // Test with SOURCE_DATE_EPOCH set + _ = os.Setenv("SOURCE_DATE_EPOCH", "1234567890") + defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }() + + err := s.repo.Publish(s.packagePool, s.provider, s.factory, &NullSigner{}, nil, false, "") + c.Assert(err, IsNil) + + rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/squeeze/Release")) + c.Assert(err, IsNil) + defer func() { _ = rf.Close() }() + + cfr := NewControlFileReader(rf, true, false) + st, err := cfr.ReadStanza() + c.Assert(err, IsNil) + + // Expected date for Unix timestamp 1234567890: Fri, 13 Feb 2009 23:31:30 UTC + c.Check(st["Date"], Equals, "Fri, 13 Feb 2009 23:31:30 UTC") +} + +func (s *PublishedRepoSuite) TestPublishSourceDateEpochInvalid(c *C) { + // Test with invalid SOURCE_DATE_EPOCH (should fallback to current time) + _ = os.Setenv("SOURCE_DATE_EPOCH", "invalid") + defer func() { _ = os.Unsetenv("SOURCE_DATE_EPOCH") }() + + err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") + c.Assert(err, IsNil) + + rf, err := os.Open(filepath.Join(s.publishedStorage.PublicPath(), "ppa/dists/maverick/Release")) + c.Assert(err, IsNil) + defer func() { _ = rf.Close() }() + + cfr := NewControlFileReader(rf, true, false) + st, err := cfr.ReadStanza() + c.Assert(err, IsNil) + + // Should have a valid Date field (not empty, not the fixed date from SOURCE_DATE_EPOCH) + c.Check(st["Date"], Not(Equals), "") + c.Check(st["Date"], Not(Equals), "Fri, 13 Feb 2009 23:31:30 UTC") +} + func (s *PublishedRepoSuite) TestPublishLocalRepo(c *C) { err := s.repo2.Publish(s.packagePool, s.provider, s.factory, nil, nil, false, "") c.Assert(err, IsNil) @@ -756,7 +797,10 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) { snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3") _ = s.snapshotCollection.Add(snap3) - // Ensure that adding a second publish point with matching files doesn't give duplicate results. + // When a second publish point references the same package (snap3 is a clone of snap2, + // both containing p3/lonely-strangers), listReferencedFilesByComponent deduplicates by + // package ref so the file appears only once. StrSlicesSubstract handles a single entry + // correctly, so no duplicate is needed for cleanup safety. repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory, false) c.Check(err, IsNil) c.Check(s.collection.Add(repo3), IsNil) @@ -771,7 +815,9 @@ func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) { "a/alien-arena/alien-arena-common_7.40-2_i386.deb", "a/alien-arena/mars-invaders_7.40-2_i386.deb", }, - "main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"}, + "main": { + "a/alien-arena/lonely-strangers_7.40-2_i386.deb", + }, }) } diff --git a/deb/reflist.go b/deb/reflist.go index e559cad4fc..0a5c1831e0 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -79,6 +79,9 @@ func (l *PackageRefList) Decode(input []byte) error { // ForEach calls handler for each package ref in list func (l *PackageRefList) ForEach(handler func([]byte) error) error { + if l == nil { + return nil + } var err error for _, p := range l.Refs { err = handler(p) diff --git a/deb/reflist_test.go b/deb/reflist_test.go index a3f035dbaa..3a7297ea78 100644 --- a/deb/reflist_test.go +++ b/deb/reflist_test.go @@ -65,7 +65,7 @@ func (s *PackageRefListSuite) TestNewPackageListFromRefList(c *C) { list, err := NewPackageListFromRefList(reflist, coll, nil) c.Assert(err, IsNil) c.Check(list.Len(), Equals, 4) - c.Check(list.Add(s.p4), ErrorMatches, "package already exists and is different: .*") + c.Check(list.Add(s.p4), ErrorMatches, "package already exists and is different: .*") list, err = NewPackageListFromRefList(nil, coll, nil) c.Assert(err, IsNil) @@ -130,6 +130,17 @@ func (s *PackageRefListSuite) TestPackageRefListForeach(c *C) { c.Check(err, Equals, e) } +func (s *PackageRefListSuite) TestForEachNilList(c *C) { + var l *PackageRefList + called := false + err := l.ForEach(func([]byte) error { + called = true + return nil + }) + c.Assert(err, IsNil) + c.Assert(called, Equals, false) +} + func (s *PackageRefListSuite) TestHas(c *C) { _ = s.list.Add(s.p1) _ = s.list.Add(s.p3) diff --git a/deb/remote.go b/deb/remote.go index efba26869f..c6fe595c2f 100644 --- a/deb/remote.go +++ b/deb/remote.go @@ -574,7 +574,7 @@ func (repo *RemoteRepo) DownloadPackageIndexes(progress aptly.Progress, d aptly. if progress != nil { progress.ColoredPrintf("@y[!]@| @!skipping package %s: duplicate in packages index@|", p) } - } else if err != nil { + } else { return err } } diff --git a/deb/snapshot.go b/deb/snapshot.go index 74f8865db3..77b853a156 100644 --- a/deb/snapshot.go +++ b/deb/snapshot.go @@ -125,12 +125,6 @@ func (s *Snapshot) Key() []byte { return []byte("S" + s.UUID) } -// ResourceKey is a unique identifier of the resource -// this snapshot uses. Instead of uuid it uses name -// which needs to be unique as well. -func (s *Snapshot) ResourceKey() []byte { - return []byte("S" + s.Name) -} // RefKey is a unique id for package reference list func (s *Snapshot) RefKey() []byte { diff --git a/deb/snapshot_bench_test.go b/deb/snapshot_bench_test.go index c7137eb110..afa835ba52 100644 --- a/deb/snapshot_bench_test.go +++ b/deb/snapshot_bench_test.go @@ -31,8 +31,7 @@ func BenchmarkSnapshotCollectionForEach(b *testing.B) { for i := 0; i < b.N; i++ { collection = NewSnapshotCollection(db) - - _ = collection.ForEach(func(s *Snapshot) error { + _ = collection.ForEach(func(s *Snapshot) error { return nil }) } diff --git a/deb/version.go b/deb/version.go index a88ccd5407..4b9150389a 100644 --- a/deb/version.go +++ b/deb/version.go @@ -30,14 +30,14 @@ func CompareVersions(ver1, ver2 string) int { // parseVersions breaks down full version to components (possibly empty) func parseVersion(ver string) (epoch, upstream, debian string) { - i := strings.LastIndex(ver, "-") + i := strings.Index(ver, ":") if i != -1 { - debian, ver = ver[i+1:], ver[:i] + epoch, ver = ver[:i], ver[i+1:] } - i = strings.Index(ver, ":") + i = strings.Index(ver, "-") if i != -1 { - epoch, ver = ver[:i], ver[i+1:] + debian, ver = ver[i+1:], ver[:i] } upstream = ver @@ -50,7 +50,7 @@ func compareLexicographic(s1, s2 string) int { i := 0 l1, l2 := len(s1), len(s2) - for !(i == l1 && i == l2) { // break if s1 equal to s2 + for !(i == l1 && i == l2) { // break if s1 equal to s2 if i == l2 { // s1 is longer than s2 diff --git a/deb/version_test.go b/deb/version_test.go index 7dbcb145d1..5b06923138 100644 --- a/deb/version_test.go +++ b/deb/version_test.go @@ -20,10 +20,10 @@ func (s *VersionSuite) TestParseVersion(c *C) { c.Check([]string{e, u, d}, DeepEquals, []string{"", "1.3.4", "1"}) e, u, d = parseVersion("1.3-pre4-1") - c.Check([]string{e, u, d}, DeepEquals, []string{"", "1.3-pre4", "1"}) + c.Check([]string{e, u, d}, DeepEquals, []string{"", "1.3", "pre4-1"}) e, u, d = parseVersion("4:1.3-pre4-1") - c.Check([]string{e, u, d}, DeepEquals, []string{"4", "1.3-pre4", "1"}) + c.Check([]string{e, u, d}, DeepEquals, []string{"4", "1.3", "pre4-1"}) } func (s *VersionSuite) TestCompareLexicographic(c *C) { @@ -100,6 +100,7 @@ func (s *VersionSuite) TestCompareVersions(c *C) { c.Check(CompareVersions("1.0-133-avc", "1.0"), Equals, 1) c.Check(CompareVersions("5.2.0.3", "5.2.0.283"), Equals, -1) + c.Check(CompareVersions("4.3.5a", "4.3.5-rc3-1"), Equals, 1) } func (s *VersionSuite) TestParseDependency(c *C) { diff --git a/debian/aptly.conf b/debian/aptly.conf index 54771846cf..73d0ff3829 100644 --- a/debian/aptly.conf +++ b/debian/aptly.conf @@ -70,6 +70,9 @@ ppa_distributor_id: ubuntu # Codename for short PPA url expansion ppa_codename: "" +# PPA Base URL (default: launchpad) +# # ppa_baseurl: http://ppa.launchpad.net + # Aptly Server ############### diff --git a/debian/tests/system-test b/debian/tests/system-test index 8631199274..bb7c93cc23 100755 --- a/debian/tests/system-test +++ b/debian/tests/system-test @@ -23,7 +23,6 @@ export USER=root # for t07/RootDirInaccessible disable_test t01_version/version VersionTest "version" disable_test t02_config/config CreateConfigTest "different conf" -disable_test t04_mirror/create CreateMirror18Test "target repo down" disable_test t04_mirror/create CreateMirror31Test "public key not found" disable_test t04_mirror/create CreateMirror35Test "flaky on s390" disable_test t07_serve/serve Serve1Test "minor html diff" diff --git a/docs/Files.md b/docs/Files.md index 616a64ac87..4fcba73b68 100644 --- a/docs/Files.md +++ b/docs/Files.md @@ -2,7 +2,7 @@
In order to add debian package files to a local repository, files are first uploaded to a temporary directory. -Then the directory (or a specific file within) is added to a repository. After adding to a repositorty, the directory resp. files are removed bt default. +Then the directory (or a specific file within) is added to a repository. After adding to a repository, the directory resp. files are removed bt default. All uploaded files are stored under `/upload/` directory. diff --git a/docs/Packages.md b/docs/Packages.md index 18385c0e60..845f955a7b 100644 --- a/docs/Packages.md +++ b/docs/Packages.md @@ -1,5 +1,5 @@ # Search Package Collection
-Perform operations on the whole collection of packages in apty database. +Perform operations on the whole collection of packages in aptly database.
diff --git a/docs/Publish.md b/docs/Publish.md index 0077f4d9b6..d6a8fd754c 100644 --- a/docs/Publish.md +++ b/docs/Publish.md @@ -11,11 +11,30 @@ Repositories can be published to local directories, Amazon S3 buckets, Azure or GPG key is required to sign any published repository. The key pari should be generated before publishing. -Publiс part of the key should be exported from your keyring using `gpg --export --armor` and imported on the system which uses a published repository. +Public part of the key should be exported from your keyring using `gpg --export --armor` and imported on the system which uses a published repository. + +* Multiple signing keys can be defined in aptly.conf using the gpgKeys array: +``` +"gpgKeys": [ + "KEY_ID_x", + "KEY_ID_y" +] +``` + +* It is also possible to pass multiple keys via the CLI using the repeatable `--gpg-key` flag: +``` +aptly publish repo my-repo --gpg-key=KEY_ID_a --gpg-key=KEY_ID_b +``` +* When using the REST API, the `gpgKey` parameter supports a comma-separated list of key IDs: +``` +"gpgKey": "KEY_ID_a,KEY_ID_b" +``` +* If `--gpg-key` is specified on the command line, or `gpgKey` is provided via the REST API, it takes precedence over any gpgKeys configuration in aptly.conf. +* With multi-key support, aptly will sign all Release files (both clearsigned and detached signatures) with each provided key, ensuring a smooth key rotation process while maintaining compatibility for existing clients. #### Parameters Publish APIs use following convention to identify published repositories: `/api/publish/:prefix/:distribution`. `:distribution` is distribution name, while `:prefix` is `[:]` (storage is optional, it defaults to empty string), if publishing prefix contains slashes `/`, they should be replaced with underscores (`_`) and underscores -should be replaced with double underscore (`__`). To specify root `:prefix`, use `:.`, as `.` is ambigious in URLs. +should be replaced with double underscore (`__`). To specify root `:prefix`, use `:.`, as `.` is ambiguous in URLs.
diff --git a/docs/Repos.md b/docs/Repos.md index 05960c0339..8b1073a9f7 100644 --- a/docs/Repos.md +++ b/docs/Repos.md @@ -1,6 +1,6 @@ # Manage Local Repositories
-A local repository is a collection of versionned packages (usually custom packages created internally). +A local repository is a collection of versioned packages (usually custom packages created internally). Packages can be added, removed, moved or copied between repos. diff --git a/docs/index.go b/docs/index.go.disabled similarity index 100% rename from docs/index.go rename to docs/index.go.disabled diff --git a/files/package_pool.go b/files/package_pool.go index b50811c2cc..0aa27fdf8b 100644 --- a/files/package_pool.go +++ b/files/package_pool.go @@ -241,7 +241,7 @@ func (pool *PackagePool) Import(srcPath, basename string, checksums *utils.Check return "", err } defer func() { - _ = source.Close() + _ = source.Close() }() sourceInfo, err := source.Stat() diff --git a/files/public.go b/files/public.go index f3756aeb8c..dea35ea81b 100644 --- a/files/public.go +++ b/files/public.go @@ -15,6 +15,10 @@ import ( "github.com/saracen/walker" ) +// syncFile is a seam to allow tests to force fsync failures (e.g. ENOSPC). +// In production it calls (*os.File).Sync(). +var syncFile = func(f *os.File) error { return f.Sync() } + // PublishedStorage abstract file system with public dirs (published repos) type PublishedStorage struct { rootPath string @@ -99,7 +103,17 @@ func (storage *PublishedStorage) PutFile(path string, sourceFilename string) err }() _, err = io.Copy(f, source) - return err + if err != nil { + return err + } + + // Sync to ensure all data is written to disk and catch ENOSPC errors + err = syncFile(f) + if err != nil { + return fmt.Errorf("error syncing file %s: %s", path, err) + } + + return nil } // Remove removes single file under public path @@ -136,6 +150,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, baseName := filepath.Base(fileName) poolPath := filepath.Join(storage.rootPath, publishedPrefix, publishedRelPath, filepath.Dir(fileName)) + destinationPath := filepath.Join(poolPath, baseName) var localSourcePool aptly.LocalPackagePool if storage.linkMethod != LinkMethodCopy { @@ -154,7 +169,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, var dstStat os.FileInfo - dstStat, err = os.Stat(filepath.Join(poolPath, baseName)) + dstStat, err = os.Stat(destinationPath) if err == nil { // already exists, check source file @@ -173,7 +188,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, } else { // if source and destination have the same checksums, no need to copy var dstMD5 string - dstMD5, err = utils.MD5ChecksumForFile(filepath.Join(poolPath, baseName)) + dstMD5, err = utils.MD5ChecksumForFile(destinationPath) if err != nil { return err @@ -204,11 +219,11 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, // source and destination have different inodes, if !forced, this is fatal error if !force { - return fmt.Errorf("error linking file to %s: file already exists and is different", filepath.Join(poolPath, baseName)) + return fmt.Errorf("error linking file to %s: file already exists and is different", destinationPath) } // forced, so remove destination - err = os.Remove(filepath.Join(poolPath, baseName)) + err = os.Remove(destinationPath) if err != nil { return err } @@ -223,7 +238,7 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, } var dst *os.File - dst, err = os.Create(filepath.Join(poolPath, baseName)) + dst, err = os.Create(destinationPath) if err != nil { _ = r.Close() return err @@ -242,11 +257,18 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, return err } + // Sync to ensure all data is written to disk and catch ENOSPC errors + err = syncFile(dst) + if err != nil { + _ = dst.Close() + return fmt.Errorf("error syncing file %s: %s", destinationPath, err) + } + err = dst.Close() } else if storage.linkMethod == LinkMethodSymLink { - err = localSourcePool.Symlink(sourcePath, filepath.Join(poolPath, baseName)) + err = localSourcePool.Symlink(sourcePath, destinationPath) } else { - err = localSourcePool.Link(sourcePath, filepath.Join(poolPath, baseName)) + err = localSourcePool.Link(sourcePath, destinationPath) } return err diff --git a/files/public_test.go b/files/public_test.go index 14413167ee..9f9198f311 100644 --- a/files/public_test.go +++ b/files/public_test.go @@ -1,8 +1,14 @@ package files import ( + "bytes" + "errors" + "io" "os" + "os/exec" "path/filepath" + "runtime" + "strings" "syscall" "github.com/aptly-dev/aptly/aptly" @@ -11,6 +17,77 @@ import ( . "gopkg.in/check.v1" ) +type fakeProgress struct{ bytes.Buffer } + +func (p *fakeProgress) Start() {} +func (p *fakeProgress) Shutdown() {} +func (p *fakeProgress) Flush() {} +func (p *fakeProgress) InitBar(count int64, isBytes bool, barType aptly.BarType) { +} +func (p *fakeProgress) ShutdownBar() {} +func (p *fakeProgress) AddBar(count int) {} +func (p *fakeProgress) SetBar(count int) {} +func (p *fakeProgress) Printf(msg string, a ...interface{}) { +} +func (p *fakeProgress) ColoredPrintf(msg string, a ...interface{}) { +} +func (p *fakeProgress) PrintfStdErr(msg string, a ...interface{}) { +} + +type fakeRSC struct { + *bytes.Reader + closeErr error +} + +func (r *fakeRSC) Close() error { return r.closeErr } + +type fakePool struct { + sizeErr error + openFn func(string) (aptly.ReadSeekerCloser, error) +} + +type fakeLocalPool struct { + fakePool + statErr error +} + +func (p *fakeLocalPool) Stat(path string) (os.FileInfo, error) { return nil, p.statErr } +func (p *fakeLocalPool) GenerateTempPath(filename string) (string, error) { + return "", nil +} +func (p *fakeLocalPool) Link(path, dstPath string) error { return nil } +func (p *fakeLocalPool) Symlink(path, dstPath string) error { return nil } +func (p *fakeLocalPool) FullPath(path string) string { return path } + +func (p *fakePool) Verify(poolPath, basename string, checksums *utils.ChecksumInfo, checksumStorage aptly.ChecksumStorage) (string, bool, error) { + return "", false, nil +} + +func (p *fakePool) Import(srcPath, basename string, checksums *utils.ChecksumInfo, move bool, storage aptly.ChecksumStorage) (string, error) { + return "", nil +} + +func (p *fakePool) LegacyPath(filename string, checksums *utils.ChecksumInfo) (string, error) { + return "", nil +} + +func (p *fakePool) Size(path string) (int64, error) { + if p.sizeErr != nil { + return 0, p.sizeErr + } + return int64(len(path)), nil +} + +func (p *fakePool) Open(path string) (aptly.ReadSeekerCloser, error) { + if p.openFn != nil { + return p.openFn(path) + } + return nil, io.EOF +} + +func (p *fakePool) FilepathList(progress aptly.Progress) ([]string, error) { return nil, nil } +func (p *fakePool) Remove(path string) (int64, error) { return 0, nil } + type PublishedStorageSuite struct { root string storage *PublishedStorage @@ -69,6 +146,14 @@ func (s *PublishedStorageSuite) TestPutFile(c *C) { c.Assert(err, IsNil) } +func (s *PublishedStorageSuite) TestPutFileReturnsErrorIfSourceMissing(c *C) { + err := s.storage.MkDir("ppa/dists/squeeze/") + c.Assert(err, IsNil) + + err = s.storage.PutFile("ppa/dists/squeeze/Release", filepath.Join(s.root, "no-such-file")) + c.Assert(err, NotNil) +} + func (s *PublishedStorageSuite) TestFilelist(c *C) { err := s.storage.MkDir("ppa/pool/main/a/ab/") c.Assert(err, IsNil) @@ -134,6 +219,11 @@ func (s *PublishedStorageSuite) TestSymLink(c *C) { c.Assert(linkTarget, Equals, "ppa/dists/squeeze/Release") } +func (s *PublishedStorageSuite) TestReadLinkReturnsErrorOnMissingPath(c *C) { + _, err := s.storage.ReadLink("does/not/exist") + c.Assert(err, NotNil) +} + func (s *PublishedStorageSuite) TestHardLink(c *C) { err := s.storage.MkDir("ppa/dists/squeeze/") c.Assert(err, IsNil) @@ -163,6 +253,18 @@ func (s *PublishedStorageSuite) TestRemoveDirs(c *C) { c.Assert(os.IsNotExist(err), Equals, true) } +func (s *PublishedStorageSuite) TestRemoveDirsWithProgress(c *C) { + err := s.storage.MkDir("ppa/dists/squeeze/") + c.Assert(err, IsNil) + + err = s.storage.PutFile("ppa/dists/squeeze/Release", "/dev/null") + c.Assert(err, IsNil) + + p := &fakeProgress{} + err = s.storage.RemoveDirs("ppa/dists/", p) + c.Assert(err, IsNil) +} + func (s *PublishedStorageSuite) TestRemove(c *C) { err := s.storage.MkDir("ppa/dists/squeeze/") c.Assert(err, IsNil) @@ -337,3 +439,277 @@ func (s *PublishedStorageSuite) TestRootRemove(c *C) { dirStorage := NewPublishedStorage(pwd, "", "") c.Assert(func() { _ = dirStorage.RemoveDirs("", nil) }, PanicMatches, "trying to remove the root directory") } + +// DiskFullSuite uses a loopback mount; requires Linux + root. + +type DiskFullSuite struct { + root string +} + +var _ = Suite(&DiskFullSuite{}) + +func (s *DiskFullSuite) SetUpTest(c *C) { + if runtime.GOOS != "linux" { + c.Skip("disk full tests only run on Linux") + } + + s.root = c.MkDir() +} + +func (s *DiskFullSuite) TestPutFileOutOfSpace(c *C) { + mountPoint := "/smallfs" + if os.Geteuid() == 0 { + mountPoint = filepath.Join(s.root, "smallfs") + err := os.MkdirAll(mountPoint, 0777) + c.Assert(err, IsNil) + fsImage := filepath.Join(s.root, "small.img") + cmd := exec.Command("dd", "if=/dev/zero", "of="+fsImage, "bs=1M", "count=1") + err = cmd.Run() + c.Assert(err, IsNil) + cmd = exec.Command("mkfs.ext4", "-F", fsImage) + err = cmd.Run() + c.Assert(err, IsNil) + cmd = exec.Command("mount", "-o", "loop", fsImage, mountPoint) + err = cmd.Run() + c.Assert(err, IsNil) + defer func() { + _ = exec.Command("umount", mountPoint).Run() + }() + } + + storage := NewPublishedStorage(mountPoint, "", "") + largeFile := filepath.Join(s.root, "largefile") + cmd := exec.Command("dd", "if=/dev/zero", "of="+largeFile, "bs=1M", "count=2") + err := cmd.Run() + c.Assert(err, IsNil) + + err = storage.PutFile("testfile", largeFile) + c.Assert(err, NotNil) + c.Check(strings.Contains(err.Error(), "no space left on device") || + strings.Contains(err.Error(), "sync"), Equals, true, + Commentf("Expected disk full error, got: %v", err)) +} + +func (s *DiskFullSuite) TestLinkFromPoolCopyOutOfSpace(c *C) { + mountPoint := "/smallfs" + if os.Geteuid() == 0 { + mountPoint = filepath.Join(s.root, "smallfs") + err := os.MkdirAll(mountPoint, 0777) + c.Assert(err, IsNil) + fsImage := filepath.Join(s.root, "small.img") + + cmd := exec.Command("dd", "if=/dev/zero", "of="+fsImage, "bs=1M", "count=1") + err = cmd.Run() + c.Assert(err, IsNil) + + cmd = exec.Command("mkfs.ext4", "-F", fsImage) + err = cmd.Run() + c.Assert(err, IsNil) + + cmd = exec.Command("mount", "-o", "loop", fsImage, mountPoint) + err = cmd.Run() + c.Assert(err, IsNil) + defer func() { + _ = exec.Command("umount", mountPoint).Run() + }() + } + + storage := NewPublishedStorage(mountPoint, "copy", "") + + poolPath := filepath.Join(s.root, "pool") + pool := NewPackagePool(poolPath, false) + cs := NewMockChecksumStorage() + + largeFile := filepath.Join(s.root, "package.deb") + cmd := exec.Command("dd", "if=/dev/zero", "of="+largeFile, "bs=1M", "count=2") + err := cmd.Run() + c.Assert(err, IsNil) + + sourceChecksum, err := utils.ChecksumsForFile(largeFile) + c.Assert(err, IsNil) + + srcPoolPath, err := pool.Import(largeFile, "package.deb", + &utils.ChecksumInfo{MD5: "d41d8cd98f00b204e9800998ecf8427e"}, false, cs) + c.Assert(err, IsNil) + + err = storage.LinkFromPool("", "pool/main/p/package", "package.deb", + pool, srcPoolPath, sourceChecksum, false) + c.Assert(err, NotNil) + c.Check(strings.Contains(err.Error(), "no space left on device") || + strings.Contains(err.Error(), "sync"), Equals, true, + Commentf("Expected disk full error, got: %v", err)) +} + +type DiskFullNoRootSuite struct { + root string +} + +var _ = Suite(&DiskFullNoRootSuite{}) + +func (s *DiskFullNoRootSuite) SetUpTest(c *C) { + s.root = c.MkDir() +} + +func (s *DiskFullNoRootSuite) TestSyncIsCalled(c *C) { + storage := NewPublishedStorage(s.root, "", "") + sourceFile := filepath.Join(s.root, "source.txt") + err := os.WriteFile(sourceFile, []byte("test content"), 0644) + c.Assert(err, IsNil) + err = storage.PutFile("dest.txt", sourceFile) + c.Assert(err, IsNil) + content, err := os.ReadFile(filepath.Join(s.root, "dest.txt")) + c.Assert(err, IsNil) + c.Check(string(content), Equals, "test content") +} + +func (s *DiskFullNoRootSuite) TestLinkFromPoolCopySyncIsCalled(c *C) { + storage := NewPublishedStorage(s.root, "copy", "") + poolPath := filepath.Join(s.root, "pool") + pool := NewPackagePool(poolPath, false) + cs := NewMockChecksumStorage() + + pkgFile := filepath.Join(s.root, "package.deb") + err := os.WriteFile(pkgFile, []byte("package content"), 0644) + c.Assert(err, IsNil) + + sourceChecksum, err := utils.ChecksumsForFile(pkgFile) + c.Assert(err, IsNil) + + srcPoolPath, err := pool.Import(pkgFile, "package.deb", + &utils.ChecksumInfo{MD5: "d41d8cd98f00b204e9800998ecf8427e"}, false, cs) + c.Assert(err, IsNil) + + err = storage.LinkFromPool("", "pool/main/p/package", "package.deb", + pool, srcPoolPath, sourceChecksum, false) + c.Assert(err, IsNil) + + destPath := filepath.Join(s.root, "pool/main/p/package/package.deb") + content, err := os.ReadFile(destPath) + c.Assert(err, IsNil) + c.Check(string(content), Equals, "package content") +} + +func (s *DiskFullNoRootSuite) TestPutFileSyncErrorIsReturned(c *C) { + storage := NewPublishedStorage(s.root, "", "") + + sourceFile := filepath.Join(s.root, "source-syncfail.txt") + err := os.WriteFile(sourceFile, []byte("test content"), 0644) + c.Assert(err, IsNil) + + oldSyncFile := syncFile + syncFile = func(_ *os.File) error { return syscall.ENOSPC } + defer func() { syncFile = oldSyncFile }() + + err = storage.PutFile("dest-syncfail.txt", sourceFile) + c.Assert(err, NotNil) + c.Check(strings.Contains(err.Error(), "error syncing file"), Equals, true) +} + +func (s *DiskFullNoRootSuite) TestLinkFromPoolCopySyncErrorIsReturned(c *C) { + storage := NewPublishedStorage(s.root, "copy", "") + poolPath := filepath.Join(s.root, "pool") + pool := NewPackagePool(poolPath, false) + cs := NewMockChecksumStorage() + + pkgFile := filepath.Join(s.root, "package-syncfail.deb") + err := os.WriteFile(pkgFile, []byte("package content"), 0644) + c.Assert(err, IsNil) + + sourceChecksum, err := utils.ChecksumsForFile(pkgFile) + c.Assert(err, IsNil) + + srcPoolPath, err := pool.Import(pkgFile, "package-syncfail.deb", + &utils.ChecksumInfo{MD5: "d41d8cd98f00b204e9800998ecf8427e"}, false, cs) + c.Assert(err, IsNil) + + oldSyncFile := syncFile + syncFile = func(_ *os.File) error { return syscall.ENOSPC } + defer func() { syncFile = oldSyncFile }() + + err = storage.LinkFromPool("", "pool/main/p/package", "package-syncfail.deb", + pool, srcPoolPath, sourceChecksum, false) + c.Assert(err, NotNil) + c.Check(strings.Contains(err.Error(), "error syncing file"), Equals, true) +} + +func (s *DiskFullNoRootSuite) TestPutFileFailsIfDestinationDirMissing(c *C) { + storage := NewPublishedStorage(s.root, "", "") + + sourceFile := filepath.Join(s.root, "src.txt") + err := os.WriteFile(sourceFile, []byte("x"), 0644) + c.Assert(err, IsNil) + + err = storage.PutFile("missingdir/dest.txt", sourceFile) + c.Assert(err, NotNil) +} + +func (s *DiskFullNoRootSuite) TestLinkFromPoolRejectsNonLocalPoolForHardlink(c *C) { + storage := NewPublishedStorage(s.root, "", "") + pool := &fakePool{} + + err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false) + c.Assert(err, NotNil) + c.Check(strings.Contains(err.Error(), "cannot link"), Equals, true) +} + +func (s *DiskFullNoRootSuite) TestLinkFromPoolCopyReturnsErrorIfOpenFails(c *C) { + storage := NewPublishedStorage(s.root, "copy", "") + pool := &fakePool{openFn: func(string) (aptly.ReadSeekerCloser, error) { return nil, io.ErrUnexpectedEOF }} + + err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false) + c.Assert(err, NotNil) +} + +func (s *DiskFullNoRootSuite) TestLinkFromPoolCopyReturnsErrorIfReaderCloseFails(c *C) { + storage := NewPublishedStorage(s.root, "copy", "") + + pool := &fakePool{openFn: func(string) (aptly.ReadSeekerCloser, error) { + return &fakeRSC{Reader: bytes.NewReader([]byte("data")), closeErr: io.ErrClosedPipe}, nil + }} + + err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false) + c.Assert(err, NotNil) + c.Check(err, Equals, io.ErrClosedPipe) +} + +func (s *DiskFullNoRootSuite) TestLinkFromPoolCopyReturnsErrorIfSizeFailsWhenDestExists(c *C) { + storage := NewPublishedStorage(s.root, "copy", "size") + pool := &fakePool{sizeErr: io.ErrUnexpectedEOF, openFn: func(string) (aptly.ReadSeekerCloser, error) { + return &fakeRSC{Reader: bytes.NewReader([]byte("data")), closeErr: nil}, nil + }} + + destDir := filepath.Join(s.root, "pool/main/p/pkg") + c.Assert(os.MkdirAll(destDir, 0777), IsNil) + c.Assert(os.WriteFile(filepath.Join(destDir, "x.deb"), []byte("old"), 0644), IsNil) + + err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false) + c.Assert(err, NotNil) + c.Check(err, Equals, io.ErrUnexpectedEOF) +} + +func (s *DiskFullNoRootSuite) TestLinkFromPoolCopyChecksumReturnsErrorIfDstMD5Fails(c *C) { + storage := NewPublishedStorage(s.root, "copy", "") + pool := &fakePool{openFn: func(string) (aptly.ReadSeekerCloser, error) { + return &fakeRSC{Reader: bytes.NewReader([]byte("data")), closeErr: nil}, nil + }} + + // Make destinationPath a directory so MD5ChecksumForFile fails. + destDir := filepath.Join(s.root, "pool/main/p/pkg") + c.Assert(os.MkdirAll(destDir, 0777), IsNil) + c.Assert(os.MkdirAll(filepath.Join(destDir, "x.deb"), 0777), IsNil) + + err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false) + c.Assert(err, NotNil) +} + +func (s *DiskFullNoRootSuite) TestLinkFromPoolHardlinkReturnsErrorIfStatFailsWhenDestExists(c *C) { + storage := NewPublishedStorage(c.MkDir(), "hardlink", "") + pool := &fakeLocalPool{statErr: errors.New("stat failed")} + + destDir := filepath.Join(storage.rootPath, "pool", "main", "p", "pkg") + c.Assert(os.MkdirAll(destDir, 0777), IsNil) + c.Assert(os.WriteFile(filepath.Join(destDir, "x.deb"), []byte("x"), 0644), IsNil) + + err := storage.LinkFromPool("", "pool/main/p/pkg", "x.deb", pool, "x", utils.ChecksumInfo{MD5: "x"}, false) + c.Assert(err, NotNil) +} diff --git a/go.mod b/go.mod index 53c5e78cbf..46d07bb420 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect + github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect @@ -87,6 +87,7 @@ require ( github.com/kr/text v0.2.0 // indirect github.com/leodido/go-urn v1.2.4 // indirect github.com/mailru/easyjson v0.7.6 // indirect + github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect @@ -115,8 +116,7 @@ require ( ) require ( - github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 - github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 + github.com/Azure/azure-storage-blob-go v0.15.0 github.com/ProtonMail/go-crypto v1.0.0 github.com/aws/aws-sdk-go-v2 v1.32.5 github.com/aws/aws-sdk-go-v2/config v1.28.5 @@ -124,8 +124,6 @@ require ( github.com/aws/aws-sdk-go-v2/service/s3 v1.67.1 github.com/aws/smithy-go v1.22.1 github.com/google/uuid v1.6.0 - github.com/swaggo/files v1.0.1 - github.com/swaggo/gin-swagger v1.6.0 github.com/swaggo/swag v1.16.3 go.etcd.io/etcd/client/v3 v3.5.15 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 502f4b2168..349da969ed 100644 --- a/go.sum +++ b/go.sum @@ -1,17 +1,20 @@ github.com/AlekSi/pointer v1.1.0 h1:SSDMPcXD9jSl8FPy9cRzoRaMJtm9g9ggGTxecRUbQoI= github.com/AlekSi/pointer v1.1.0/go.mod h1:y7BvfRI3wXPWKXEBhU71nbnIEEZX0QTSB2Bj48UJIZE= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0 h1:nyQWyZvwGTvunIMxi1Y9uXkcyr+I7TeNrr/foo4Kpk8= -github.com/Azure/azure-sdk-for-go/sdk/azcore v1.14.0/go.mod h1:l38EPgmsp71HHLq9j7De57JcKOWPyhrsW1Awm1JS6K0= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0 h1:tfLQ34V6F7tVSwoTf/4lH5sE0o6eCJuNDTmH09nDpbc= -github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.7.0/go.mod h1:9kIvujWAA58nmPmWB1m23fyWic1kYZMxD9CxaWn4Qpg= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 h1:ywEEhmNahHBihViHepv3xPBn1663uRv2t2q/ESv9seY= -github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0/go.mod h1:iZDifYGJTIgIIkYRNWPENUnqx6bJ2xnSDFI2tjwZNuY= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0 h1:PiSrjRPpkQNjrM8H0WwKMnZUdu1RGMtd/LdGKUrOo+c= -github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.6.0/go.mod h1:oDrbWx4ewMylP7xHivfgixbfGBT6APAwsSoHRKotnIc= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1 h1:cf+OIKbkmMHBaC3u78AXomweqM0oxQSgBXRZf3WH4yM= -github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.4.1/go.mod h1:ap1dmS6vQKJxSMNiGJcq4QuUQkOynyD93gLw6MDF7ek= -github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mxXfQidrMEnLlPk9UMeRtyBTnEFtxkV0kU= -github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/Azure/azure-pipeline-go v0.2.3 h1:7U9HBg1JFK3jHl5qmo4CTZKFTVgMwdFHMVtCdfBE21U= +github.com/Azure/azure-pipeline-go v0.2.3/go.mod h1:x841ezTBIMG6O3lAcl8ATHnsOPVl2bqk7S3ta6S6u4k= +github.com/Azure/azure-storage-blob-go v0.15.0 h1:rXtgp8tN1p29GvpGgfJetavIG0V7OgcSXPpwp3tx6qk= +github.com/Azure/azure-storage-blob-go v0.15.0/go.mod h1:vbjsVbX0dlxnRc4FFMPsS9BsJWPcne7GB7onqlPvz58= +github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest/adal v0.9.13 h1:Mp5hbtOePIzM8pJVRa3YLrWWmZtoxRXqUEzCfJt3+/Q= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/date v0.3.0 h1:7gUk1U5M/CQbp9WoqinNzJar+8KY+LPI6wiWrP/myHw= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/logger v0.2.1 h1:IG7i4p/mDa2Ce4TRyAO8IHnVhAVF3RFU+ZtXWSmf4Tg= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUMfuitfgcfuo= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55 h1:jbGlDKdzAZ92NzK65hUP98ri0/r50vVVvmZsFP/nIqo= github.com/DisposaBoy/JsonConfigReader v0.0.0-20171218180944-5ea4d0ddac55/go.mod h1:GCzqZQHydohgVLSIqRKZeTt8IGb1Y4NaFfim3H40uUI= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= @@ -91,14 +94,14 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible h1:TcekIExNqud5crz4xD2pavyTgWiPvpYe4Xau31I0PRk= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= -github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.9.1 h1:4idEAncQnU5cB7BeOkPtxjfCSye0AAm1R0RVIqJ+Jmg= @@ -127,8 +130,6 @@ github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MG github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang-jwt/jwt/v5 v5.2.1 h1:OuVbFODueb089Lh128TAcimifWaLhJwVflnrgM17wHk= -github.com/golang-jwt/jwt/v5 v5.2.1/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= @@ -150,6 +151,7 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= @@ -197,6 +199,8 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= +github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= @@ -235,8 +239,6 @@ github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= -github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= -github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -284,10 +286,6 @@ github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= -github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= -github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= -github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo= github.com/swaggo/swag v1.16.3 h1:PnCYjPCah8FK4I26l2F/KQ4yz3sILcVUN3cTlBFA9Pg= github.com/swaggo/swag v1.16.3/go.mod h1:DImHIuOFXKpMFAQjcC7FG4m3Dg4+QuUgUzJmKjI/gRk= github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d h1:vfofYNRScrDdvS342BElfbETmL1Aiz3i2t0zfRj16Hs= @@ -319,6 +317,8 @@ golang.org/x/arch v0.3.0/go.mod h1:5om86z9Hs0C8fWVUuoMHwpExlXzs5Tkyp9hOrfG7pp8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= @@ -333,18 +333,19 @@ golang.org/x/mod v0.23.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191112182307-2180aed22343/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.38.0 h1:vRMAPTMaeGqVhG5QyLJHqNDwecKTomGeqbnfZyKlBI8= golang.org/x/net v0.38.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= @@ -362,6 +363,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191112214154-59a1497f0cea/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -393,6 +395,7 @@ golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/term v0.30.0 h1:PQ39fJZ+mfadBm0y5WlL4vlM7Sx1Hgf13sMIY2+QS9Y= golang.org/x/term v0.30.0/go.mod h1:NYYFdzHoI5wRh/h5tDMdMqCqPJZEuNqVR5xJLd/n67g= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= diff --git a/http/download.go b/http/download.go index 3dc9c3782b..887f9b3bf7 100644 --- a/http/download.go +++ b/http/download.go @@ -240,7 +240,7 @@ func (downloader *downloaderImpl) download(req *http.Request, url, destination s } if resp.Body != nil { defer func() { - _ = resp.Body.Close() + _ = resp.Body.Close() }() } diff --git a/http/download_go16.go b/http/download_go16.go index f4df51562f..bde6560c68 100644 --- a/http/download_go16.go +++ b/http/download_go16.go @@ -1,3 +1,4 @@ +//go:build !go1.7 // +build !go1.7 package http diff --git a/http/grab.go b/http/grab.go index fd6420a11b..8478afcec4 100644 --- a/http/grab.go +++ b/http/grab.go @@ -49,9 +49,9 @@ func (d *GrabDownloader) Download(ctx context.Context, url string, destination s func (d *GrabDownloader) DownloadWithChecksum(ctx context.Context, url string, destination string, expected *utils.ChecksumInfo, ignoreMismatch bool) error { maxTries := d.maxTries - // FIXME: const delayMax = time.Duration(5 * time.Minute) + // FIXME: const delayMax = time.Duration(5 * time.Minute) delay := time.Duration(1 * time.Second) - // FIXME: const delayMultiplier = 2 + // FIXME: const delayMultiplier = 2 err := fmt.Errorf("no tries available") for maxTries > 0 { err = d.download(ctx, url, destination, expected, ignoreMismatch) @@ -133,17 +133,17 @@ func (d *GrabDownloader) download(_ context.Context, url string, destination str resp := d.client.Do(req) - <-resp.Done + <-resp.Done // download is complete -// Loop: -// for { -// select { -// case <-resp.Done: -// // download is complete -// break Loop -// } -// } + // Loop: + // for { + // select { + // case <-resp.Done: + // // download is complete + // break Loop + // } + // } err = resp.Err() if err != nil && err == grab.ErrBadChecksum && ignoreMismatch { fmt.Printf("Ignoring checksum mismatch for %s\n", url) diff --git a/man/aptly.1 b/man/aptly.1 index bd6ad22317..1391903d8c 100644 --- a/man/aptly.1 +++ b/man/aptly.1 @@ -111,8 +111,9 @@ The legacy json configuration is still supported (and also supports comments): // Enable metrics for Prometheus client "enableMetricsEndpoint": false, + // Not implemented in this version\. // Enable API documentation on /docs - "enableSwaggerEndpoint": false, + //"enableSwaggerEndpoint": false, // OBSOLETE: use via url param ?_async=true "AsyncAPI": false, @@ -2452,6 +2453,9 @@ show yaml config .SH "ENVIRONMENT" If environment variable \fBHTTP_PROXY\fR is set \fBaptly\fR would use its value to proxy all HTTP requests\. . +.P +If environment variable \fBSOURCE_DATE_EPOCH\fR is set to a Unix timestamp, \fBaptly\fR would use that timestamp for the \fBDate\fR and \fBValid\-Until\fR fields in the \fBRelease\fR file when publishing\. This enables reproducible builds as specified by \fIhttps://reproducible\-builds\.org/specs/source\-date\-epoch/\fR\. +. .SH "RETURN VALUES" \fBaptly\fR exists with: . diff --git a/man/aptly.1.ronn.tmpl b/man/aptly.1.ronn.tmpl index 203cc7fe24..ed1b89f109 100644 --- a/man/aptly.1.ronn.tmpl +++ b/man/aptly.1.ronn.tmpl @@ -100,8 +100,9 @@ The legacy json configuration is still supported (and also supports comments): // Enable metrics for Prometheus client "enableMetricsEndpoint": false, + // Not implemented in this version. // Enable API documentation on /docs - "enableSwaggerEndpoint": false, + //"enableSwaggerEndpoint": false, // OBSOLETE: use via url param ?_async=true "AsyncAPI": false, @@ -533,6 +534,11 @@ For example, default aptly display format could be presented with the following If environment variable `HTTP_PROXY` is set `aptly` would use its value to proxy all HTTP requests. +If environment variable `SOURCE_DATE_EPOCH` is set to a Unix timestamp, +`aptly` would use that timestamp for the `Date` and `Valid-Until` fields +in the `Release` file when publishing. This enables reproducible builds +as specified by https://reproducible-builds.org/specs/source-date-epoch/. + ## RETURN VALUES `aptly` exists with: diff --git a/pgp/gnupg.go b/pgp/gnupg.go index 3edf121024..bab2db21b2 100644 --- a/pgp/gnupg.go +++ b/pgp/gnupg.go @@ -22,7 +22,7 @@ var ( type GpgSigner struct { gpg string version GPGVersion - keyRef string + keyRefs []string keyring, secretKeyring string passphrase, passphraseFile string batch bool @@ -35,7 +35,14 @@ func (g *GpgSigner) SetBatch(batch bool) { // SetKey sets key ID to use when signing files func (g *GpgSigner) SetKey(keyRef string) { - g.keyRef = keyRef + keyRef = strings.TrimSpace(keyRef) + if keyRef != "" { + if g.keyRefs == nil { + g.keyRefs = []string{keyRef} + } else { + g.keyRefs = append(g.keyRefs, keyRef) + } + } } // SetKeyRing allows to set custom keyring and secretkeyring @@ -57,8 +64,8 @@ func (g *GpgSigner) gpgArgs() []string { args = append(args, "--secret-keyring", g.secretKeyring) } - if g.keyRef != "" { - args = append(args, "-u", g.keyRef) + for _, k := range g.keyRefs { + args = append(args, "-u", k) } if g.passphrase != "" || g.passphraseFile != "" { diff --git a/s3/public.go b/s3/public.go index 09b1afdbf2..c35de42504 100644 --- a/s3/public.go +++ b/s3/public.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" @@ -51,6 +52,7 @@ type PublishedStorage struct { plusWorkaround bool disableMultiDel bool pathCache map[string]string + pathCacheMutex sync.RWMutex // True if the bucket encrypts objects by default. encryptByDefault bool @@ -251,7 +253,9 @@ func (storage *PublishedStorage) Remove(path string) error { _ = storage.Remove(strings.Replace(path, "+", " ", -1)) } + storage.pathCacheMutex.Lock() delete(storage.pathCache, path) + storage.pathCacheMutex.Unlock() return nil } @@ -280,7 +284,9 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error if err != nil { return fmt.Errorf("error deleting path %s from %s: %s", filelist[i], storage, err) } + storage.pathCacheMutex.Lock() delete(storage.pathCache, filepath.Join(path, filelist[i])) + storage.pathCacheMutex.Unlock() } } else { numParts := (len(filelist) + page - 1) / page @@ -313,9 +319,11 @@ func (storage *PublishedStorage) RemoveDirs(path string, _ aptly.Progress) error if err != nil { return fmt.Errorf("error deleting multiple paths from %s: %s", storage, err) } + storage.pathCacheMutex.Lock() for i := range part { delete(storage.pathCache, filepath.Join(path, part[i])) } + storage.pathCacheMutex.Unlock() } } @@ -337,20 +345,31 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, relPath := filepath.Join(publishedDirectory, fileName) poolPath := filepath.Join(storage.prefix, relPath) - if storage.pathCache == nil { - paths, md5s, err := storage.internalFilelist(filepath.Join(publishedPrefix, "pool"), true) - if err != nil { - return errors.Wrap(err, "error caching paths under prefix") - } + storage.pathCacheMutex.RLock() + cacheNil := storage.pathCache == nil + storage.pathCacheMutex.RUnlock() + + if cacheNil { + storage.pathCacheMutex.Lock() + if storage.pathCache == nil { + paths, md5s, err := storage.internalFilelist(filepath.Join(publishedPrefix, "pool"), true) + if err != nil { + storage.pathCacheMutex.Unlock() + return errors.Wrap(err, "error caching paths under prefix") + } - storage.pathCache = make(map[string]string, len(paths)) + storage.pathCache = make(map[string]string, len(paths)) - for i := range paths { - storage.pathCache[filepath.Join("pool", paths[i])] = md5s[i] + for i := range paths { + storage.pathCache[filepath.Join(publishedPrefix, "pool", paths[i])] = md5s[i] + } } + storage.pathCacheMutex.Unlock() } + storage.pathCacheMutex.RLock() destinationMD5, exists := storage.pathCache[relPath] + storage.pathCacheMutex.RUnlock() sourceMD5 := sourceChecksums.MD5 if exists { @@ -367,7 +386,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, err = errors.Wrap(err, fmt.Sprintf("error verifying MD5 for %s: %s", storage, poolPath)) return err } + storage.pathCacheMutex.Lock() storage.pathCache[relPath] = destinationMD5 + storage.pathCacheMutex.Unlock() } if destinationMD5 == sourceMD5 { @@ -388,7 +409,9 @@ func (storage *PublishedStorage) LinkFromPool(publishedPrefix, publishedRelPath, log.Debug().Msgf("S3: LinkFromPool '%s'", relPath) err = storage.putFile(relPath, source, sourceMD5) if err == nil { + storage.pathCacheMutex.Lock() storage.pathCache[relPath] = sourceMD5 + storage.pathCacheMutex.Unlock() } else { err = errors.Wrap(err, fmt.Sprintf("error uploading %s to %s: %s", sourcePath, storage, poolPath)) } diff --git a/s3/server_test.go b/s3/server_test.go index a8a304754c..4a2e7111b8 100644 --- a/s3/server_test.go +++ b/s3/server_test.go @@ -112,9 +112,11 @@ func NewServer(config *Config) (*Server, error) { buckets: make(map[string]*bucket), config: config, } - go func() { _ = http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - srv.serveHTTP(w, req) - })) }() + go func() { + _ = http.Serve(l, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + srv.serveHTTP(w, req) + })) + }() return srv, nil } @@ -527,14 +529,13 @@ func (bucketResource) post(a *action) interface{} { // and dashes (-). You can use uppercase letters for buckets only in the // US Standard region. // -// Must start with a number or letter +// # Must start with a number or letter // -// Must be between 3 and 255 characters long +// # Must be between 3 and 255 characters long // // There's one extra rule (Must not be formatted as an IP address (e.g., 192.168.5.4) // but the real S3 server does not seem to check that rule, so we will not // check it either. -// func validBucketName(name string) bool { if len(name) < 3 || len(name) > 255 { return false diff --git a/system/api_lib.py b/system/api_lib.py index 20036b861d..26245c81f0 100644 --- a/system/api_lib.py +++ b/system/api_lib.py @@ -34,6 +34,7 @@ class APITest(BaseTest): "linkMethod": "symlink" } }, + "enableMetricsEndpoint": True, "enableSwaggerEndpoint": True } diff --git a/system/docker-wrapper b/system/docker-wrapper index 1d92f392d9..a1396616ed 100755 --- a/system/docker-wrapper +++ b/system/docker-wrapper @@ -15,4 +15,4 @@ else fi cd /work/src -sudo -u aptly PATH=$PATH:/work/src/build GOPATH=/work/src/.go $cmd +sudo -u aptly PATH=$PATH:/work/src/build GOPATH=/work/src/.go GOCACHE=/work/src/.go/cache $cmd diff --git a/system/files/aptly-dual.pub b/system/files/aptly-dual.pub new file mode 100644 index 0000000000..7a7d1e92bb --- /dev/null +++ b/system/files/aptly-dual.pub @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- + +mQGiBFL7pY8RBAC5uHg/9AuGJ7EF7RYty89IDLeqvlPe710eDQpJ+itsOaA/5rr3 +IV1LMlqHpM2rkZkAPpARwjrga2ByJ1ww77Zq2uPqJIO2LZYWTLXic9Zity2OVu3Z +XwtdsqagIMfT5dAgNmhe5lL7qgGUwYcFFa52s7U4qO0z2FfwHW1IQrnMpwCg5RQh +Uqs5iUKdDtoeQjX5mWgQhjEEAI1zfXUvvcOrRsDlGNKYZigZiWC6J46jeR8Nnf9C +WwhXS2fzQaJyDq9DorkvPZgWUAaLLCdfGETqLzDKajynhS1+OnfFQNzvkvEPRBSb +C5k+GOF2E1E9rGXb31+1XZTcdIprp4/F3RNLLWNUwfgPLWJx9NzHTYqgBStecHkC +ySZRA/9PNFAbeJZ27HNuzoGnAa0piZDLeAAHsM1V6cosMh7U1IZqjZcrMC9YXNxH +2D90PvoBvpufCMRzL/fOVPT1JzQGYoKIX17Nmzvdq/a4YyLWRODjvWXd94bae2Xd +Vy03DYhfp8VOVJW6HuAX9JN6MKXSNxaibgOPjU822Hxd1iCIQ7QtQXB0bHkgVGVz +dGVyIChkb24ndCB1c2UgaXQpIDx0ZXN0QGFwdGx5LmluZm8+iGIEExECACIFAlL7 +pY8CGyMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJECHbuJwW2z5t2sQAoNn+ +0cADZa66HZNY2qJi44Oq4hjaAJsHzj9JKAHEpdix5N7b6QvaZQZYhrkBDQRS+6WP +EAQA9BX+kbIM6VJYoyY9vUHXfAF4E2y2M7vl9knZ+jMPfMbI7dE3gRJQb3mngST5 +7eZWawo1DNE6h3LbHsB4mpro9XLUXUMBgXRsOq4D5E0ygvDZ/tJhy0AwFiTOXKEs +/erzmbF7j/TWh4LVHXFI9DrnN0+EeF/mQC/wzX7WGCKe70cAAwUEAMr7959zUYNp +E3v4IquIJpD22bT/FiyQjFG8yGy36c+7mOP3VWi0lz5yFqqeR9NDFuLDSwOEi0nB +zXNmimLy+hIwMaHjbQLjLODmy/T9wKCgeAmK1ygT6YBGJJflThZ05M80T5hBtRA9 +z2eoTn0wbi6MLmD/rbEt+lUPfSA4V0t2iEkEGBECAAkFAlL7pY8CGwwACgkQIdu4 +nBbbPm05hgCgvYatZXRbEdZ91jJCQi1KI7lJ5Y8AnjvrHU0g84mE45QZFegZzzQo +9relmDMEZ3YCRhYJKwYBBAHaRw8BAQdAYDU0VSBcurX+uqAeR/w/XOLSZcghvOqz +Y8yWdcj3HUy0L0FwdGx5IFNlY29uZGFyeSBTaWduaW5nIEtleSA8YXB0bHlAZXhh +bXBsZS5jb20+iJYEExYKAD4WIQSu4W3wGDVPZ/5fXHK79OGUNOkeTgUCZ3YCRgIb +AwUJA8JnAAULCQgHAgYVCgkICwIEFgIDAQIeAQIXgAAKCRC79OGUNOkeTid/AP9A +kIMn2qI5TqZgzrnPt7SN16VvpMppPb2H0m0P6knQKQD8DHcLcrqAl2cjcEuntv75 +gOnEvmPDAO6S1rc8UgcWdQQ= +=XPoo +-----END PGP PUBLIC KEY BLOCK----- diff --git a/system/files/aptly3.sec b/system/files/aptly3.sec new file mode 100644 index 0000000000..cba4b2f1a2 --- /dev/null +++ b/system/files/aptly3.sec @@ -0,0 +1,11 @@ +-----BEGIN PGP PRIVATE KEY BLOCK----- + +lFgEZ3YCRhYJKwYBBAHaRw8BAQdAYDU0VSBcurX+uqAeR/w/XOLSZcghvOqzY8yW +dcj3HUwAAP9lsZgE1YQfaS9xfVOSi3f91lbq13+U9FPdwxfiET0+bBFrtC9BcHRs +eSBTZWNvbmRhcnkgU2lnbmluZyBLZXkgPGFwdGx5QGV4YW1wbGUuY29tPoiWBBMW +CgA+FiEEruFt8Bg1T2f+X1xyu/ThlDTpHk4FAmd2AkYCGwMFCQPCZwAFCwkIBwIG +FQoJCAsCBBYCAwECHgECF4AACgkQu/ThlDTpHk4nfwD/QJCDJ9qiOU6mYM65z7e0 +jdelb6TKaT29h9JtD+pJ0CkA/Ax3C3K6gJdnI3BLp7b++YDpxL5jwwDukta3PFIH +FnUE +=IXTY +-----END PGP PRIVATE KEY BLOCK----- diff --git a/system/files/corruptdb.go b/system/files/corruptdb.go index 4febcbc9bc..15b4e094a6 100644 --- a/system/files/corruptdb.go +++ b/system/files/corruptdb.go @@ -20,7 +20,7 @@ func main() { if err != nil { log.Fatalf("Error opening DB %q: %s", dbPath, err) } - defer db.Close() + defer func() { _ = db.Close() }() keys := db.KeysByPrefix([]byte(prefix)) if len(keys) == 0 { diff --git a/system/lib.py b/system/lib.py index f776199778..6b79d3de0c 100644 --- a/system/lib.py +++ b/system/lib.py @@ -272,6 +272,9 @@ def prepare_fixture(self): self.run_cmd([ self.gpgFinder.gpg2, "--import", os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files") + "/aptly.sec"], expected_code=None) + self.run_cmd([ + self.gpgFinder.gpg2, "--import", + os.path.join(os.path.dirname(inspect.getsourcefile(BaseTest)), "files") + "/aptly3.sec"], expected_code=None) if self.fixtureGpg: self.run_cmd([self.gpgFinder.gpg, "--no-default-keyring", "--trust-model", "always", "--batch", "--keyring", "aptlytest.gpg", "--import"] + @@ -310,7 +313,9 @@ def _start_process(self, command, stderr=subprocess.STDOUT, stdout=None): if command[0] == "aptly": aptly_testing_bin = Path(__file__).parent / ".." / "aptly.test" - command = [str(aptly_testing_bin), f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out", *command[1:]] + command = [str(aptly_testing_bin), *command[1:]] + if self.coverage_dir is not None: + command.insert(1, f"-test.coverprofile={Path(self.coverage_dir) / self.__class__.__name__}-{uuid4()}.out") if self.faketime: command = ["faketime", os.environ.get("TEST_FAKETIME", "2025-01-02 03:04:05")] + command @@ -337,7 +342,7 @@ def run_cmd(self, command, expected_code=0): if is_aptly_command: # remove the last two rows as go tests always print PASS/FAIL and coverage in those # two lines. This would otherwise fail the tests as they would not match gold - matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*\ncoverage: .*", raw_output) + matches = re.findall(r"((.|\n)*)EXIT: (\d)\n.*(?:\ncoverage: .*|$)", raw_output) if not matches: raise Exception("no matches found in command output '%s'" % raw_output) @@ -517,7 +522,7 @@ def verify_match(self, gold, orig, match_prepare=None, ensure_utf8=True): if gold != output: diff = "".join(difflib.unified_diff( [l + "\n" for l in gold.split("\n")], [l + "\n" for l in output.split("\n")])) - raise Exception("content doesn't match:\n" + diff + "\n\nOutput:\n" + orig + "\n") + raise Exception(f"content doesn't match:\n{diff}\n\nOutput:\n{orig}\n") check = check_output diff --git a/system/run.py b/system/run.py index 4e73fb2d8a..2b0de524d9 100755 --- a/system/run.py +++ b/system/run.py @@ -36,7 +36,7 @@ def natural_key(string_): return [int(s) if s.isdigit() else s for s in re.split(r'(\d+)', string_)] -def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None): +def run(include_long_tests=False, capture_results=False, tests=None, filters=None, coverage_dir=None, coverage_skip=False): """ Run system test. """ @@ -47,7 +47,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non fails = [] numTests = numFailed = numSkipped = 0 lastBase = None - if not coverage_dir: + if not coverage_dir and not coverage_skip: coverage_dir = mkdtemp(suffix="aptly-coverage") failed = False @@ -213,6 +213,7 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non include_long_tests = False capture_results = False coverage_dir = None + coverage_skip = False tests = None args = sys.argv[1:] @@ -224,6 +225,8 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non elif args[0] == "--coverage-dir": coverage_dir = args[1] args = args[1:] + elif args[0] == "--coverage-skip": + coverage_skip = True args = args[1:] @@ -236,4 +239,4 @@ def run(include_long_tests=False, capture_results=False, tests=None, filters=Non else: filters.append(arg) - run(include_long_tests, capture_results, tests, filters, coverage_dir) + run(include_long_tests, capture_results, tests, filters, coverage_dir, coverage_skip) diff --git a/system/t02_config/ConfigShowTest_gold b/system/t02_config/ConfigShowTest_gold index 5a4a2273ae..ccb8ed32f3 100644 --- a/system/t02_config/ConfigShowTest_gold +++ b/system/t02_config/ConfigShowTest_gold @@ -12,6 +12,7 @@ "dependencyVerboseResolve": false, "ppaDistributorID": "ubuntu", "ppaCodename": "", + "ppaBaseURL": "http://ppa.launchpad.net", "serveInAPIMode": true, "enableMetricsEndpoint": true, "enableSwaggerEndpoint": false, @@ -29,6 +30,7 @@ "gpgProvider": "gpg", "gpgDisableSign": false, "gpgDisableVerify": false, + "gpgKeys": [], "skipContentsPublishing": false, "skipBz2Publishing": false, "FileSystemPublishEndpoints": {}, diff --git a/system/t02_config/ConfigShowYAMLTest_gold b/system/t02_config/ConfigShowYAMLTest_gold index 615982b0db..c5a03a9c36 100644 --- a/system/t02_config/ConfigShowYAMLTest_gold +++ b/system/t02_config/ConfigShowYAMLTest_gold @@ -11,6 +11,7 @@ dep_follow_source: false dep_verboseresolve: false ppa_distributor_id: ubuntu ppa_codename: "" +ppa_baseurl: http://ppa.launchpad.net serve_in_api_mode: true enable_metrics_endpoint: true enable_swagger_endpoint: false @@ -27,6 +28,7 @@ download_sourcepackages: false gpg_provider: gpg gpg_disable_sign: false gpg_disable_verify: false +gpg_keys: [] skip_contents_publishing: false skip_bz2_publishing: false filesystem_publish_endpoints: {} diff --git a/system/t02_config/CreateConfigTest_gold b/system/t02_config/CreateConfigTest_gold index c233118fbc..73d0ff3829 100644 --- a/system/t02_config/CreateConfigTest_gold +++ b/system/t02_config/CreateConfigTest_gold @@ -70,6 +70,9 @@ ppa_distributor_id: ubuntu # Codename for short PPA url expansion ppa_codename: "" +# PPA Base URL (default: launchpad) +# # ppa_baseurl: http://ppa.launchpad.net + # Aptly Server ############### @@ -80,8 +83,9 @@ serve_in_api_mode: false # Enable metrics for Prometheus client enable_metrics_endpoint: false +# Not implemented in this version. # Enable API documentation on /docs -enable_swagger_endpoint: false +#enable_swagger_endpoint: false # OBSOLETE: use via url param ?_async=true async_api: false diff --git a/system/t04_mirror/CreateMirror18Test_gold b/system/t04_mirror/CreateMirror18Test_gold index d63b87452b..aed26b9462 100644 --- a/system/t04_mirror/CreateMirror18Test_gold +++ b/system/t04_mirror/CreateMirror18Test_gold @@ -1,4 +1,4 @@ -Downloading: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease +Downloading: http://repo.aptly.info/system-tests/ppa/gladky-anton/gnuplot/ubuntu/dists/maverick/InRelease gpgv: Signature made Sun Jul 28 07:57:01 2024 UTC gpgv: using RSA key 5BFCD481D86D5824470E469F9000B1C3A01F726C gpgv: Good signature from "Launchpad PPA for Anton Gladky" @@ -6,5 +6,5 @@ gpgv: Signature made Sun Jul 28 07:57:01 2024 UTC gpgv: using RSA key 02219381E9161C78A46CB2BFA5279A973B1F56C0 gpgv: Good signature from "Launchpad sim" -Mirror [mirror18]: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ maverick successfully added. +Mirror [mirror18]: http://repo.aptly.info/system-tests/ppa/gladky-anton/gnuplot/ubuntu/ maverick successfully added. You can run 'aptly mirror update mirror18' to download repository contents. diff --git a/system/t04_mirror/CreateMirror18Test_mirror_show b/system/t04_mirror/CreateMirror18Test_mirror_show index 750161b0d3..d0bd269da4 100644 --- a/system/t04_mirror/CreateMirror18Test_mirror_show +++ b/system/t04_mirror/CreateMirror18Test_mirror_show @@ -1,5 +1,5 @@ Name: mirror18 -Archive Root URL: http://ppa.launchpad.net/gladky-anton/gnuplot/ubuntu/ +Archive Root URL: http://repo.aptly.info/system-tests/ppa/gladky-anton/gnuplot/ubuntu/ Distribution: maverick Components: main Architectures: amd64, armel, i386, powerpc diff --git a/system/t04_mirror/create.py b/system/t04_mirror/create.py index 351bb20e84..9cf00a5fb4 100644 --- a/system/t04_mirror/create.py +++ b/system/t04_mirror/create.py @@ -221,6 +221,7 @@ class CreateMirror18Test(BaseTest): "max-tries": 1, "ppaDistributorID": "ubuntu", "ppaCodename": "maverick", + "ppaBaseURL": "http://repo.aptly.info/system-tests/ppa", } fixtureCmds = [ diff --git a/system/t06_publish/PublishRepo36Test_gold b/system/t06_publish/PublishRepo36Test_gold new file mode 100644 index 0000000000..ee9f9d5e0a --- /dev/null +++ b/system/t06_publish/PublishRepo36Test_gold @@ -0,0 +1,12 @@ +Loading packages... +Generating metadata files and linking package files... +Finalizing metadata files... + +Local repo local-repo has been successfully published. +Please setup your webserver to serve directory '${HOME}/.aptly/public' with autoindexing. +Now you can add following line to apt sources: + deb http://your-server/ maverick main + deb-src http://your-server/ maverick main +Don't forget to add your GPG key to apt with apt-key. + +You can also use `aptly serve` to publish your repositories over HTTP quickly. diff --git a/system/t06_publish/PublishRepo36Test_release b/system/t06_publish/PublishRepo36Test_release new file mode 100644 index 0000000000..4fe4832369 --- /dev/null +++ b/system/t06_publish/PublishRepo36Test_release @@ -0,0 +1,12 @@ +Origin: . maverick +Label: . maverick +Suite: maverick +Codename: maverick +Date: Fri, 13 Feb 2009 23:31:30 UTC +Architectures: i386 +Components: main +Description: Generated by aptly +MD5Sum: +SHA1: +SHA256: +SHA512: diff --git a/system/t06_publish/repo.py b/system/t06_publish/repo.py index f882a42862..c5e4015d11 100644 --- a/system/t06_publish/repo.py +++ b/system/t06_publish/repo.py @@ -9,6 +9,10 @@ def strip_processor(output): return "\n".join([l for l in output.split("\n") if not l.startswith(' ') and not l.startswith('Date:')]) +def strip_processor_keep_date(output): + return "\n".join([l for l in output.split("\n") if not l.startswith(' ')]) + + class PublishRepo1Test(BaseTest): """ publish repo: default @@ -951,3 +955,34 @@ def check(self): if 'main/dep11/README' not in pathsSeen: raise Exception("README file not included in release file") + + +class PublishRepo36Test(BaseTest): + """ + publish repo: SOURCE_DATE_EPOCH produces byte-identical output + """ + fixtureCmds = [ + "aptly repo create local-repo", + "aptly repo add local-repo ${files}", + ] + runCmd = "aptly publish repo -skip-signing -distribution=maverick local-repo" + gold_processor = BaseTest.expand_environ + environmentOverride = {"SOURCE_DATE_EPOCH": "1234567890"} + + def check(self): + super(PublishRepo36Test, self).check() + + # verify Release file includes the expected date from SOURCE_DATE_EPOCH + self.check_file_contents( + 'public/dists/maverick/Release', 'release', match_prepare=strip_processor_keep_date) + + # save Release file from first publish + first_release = self.read_file('public/dists/maverick/Release') + + # drop and republish with same SOURCE_DATE_EPOCH + self.run_cmd("aptly publish drop maverick") + self.run_cmd("aptly publish repo -skip-signing -distribution=maverick local-repo") + + # verify byte-identical output + second_release = self.read_file('public/dists/maverick/Release') + self.check_equal(first_release, second_release) diff --git a/system/t12_api/PublishAPITestDualSignature_Release.gpg b/system/t12_api/PublishAPITestDualSignature_Release.gpg new file mode 100644 index 0000000000..24381472d8 --- /dev/null +++ b/system/t12_api/PublishAPITestDualSignature_Release.gpg @@ -0,0 +1,14 @@ +gpg: Signature made Mon Jan 26 10:18:32 2026 UTC +gpg: using DSA key C5ACD2179B5231DFE842EE6121DBB89C16DB3E6D +gpg: checking the trustdb +gpg: no ultimately trusted keys found +gpg: Good signature from "Aptly Tester (don't use it) " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: C5AC D217 9B52 31DF E842 EE61 21DB B89C 16DB 3E6D +gpg: Signature made Mon Jan 26 10:18:32 2026 UTC +gpg: using EDDSA key AEE16DF018354F67FE5F5C72BBF4E19434E91E4E +gpg: Good signature from "Aptly Secondary Signing Key " [unknown] +gpg: WARNING: This key is not certified with a trusted signature! +gpg: There is no indication that the signature belongs to the owner. +Primary key fingerprint: AEE1 6DF0 1835 4F67 FE5F 5C72 BBF4 E194 34E9 1E4E diff --git a/system/t12_api/docs.py b/system/t12_api/docs.py deleted file mode 100644 index 50a7b1cb6e..0000000000 --- a/system/t12_api/docs.py +++ /dev/null @@ -1,17 +0,0 @@ -from api_lib import APITest - - -class TaskAPITestSwaggerDocs(APITest): - """ - GET /docs - """ - - def check(self): - resp = self.get("/docs/doc.json") - self.check_equal(resp.status_code, 200) - - resp = self.get("/docs/", allow_redirects=False) - self.check_equal(resp.status_code, 301) - - resp = self.get("/docs/index.html") - self.check_equal(resp.status_code, 200) diff --git a/system/t12_api/mirrors.py b/system/t12_api/mirrors.py index 6352b2caa4..76104f3986 100644 --- a/system/t12_api/mirrors.py +++ b/system/t12_api/mirrors.py @@ -56,7 +56,6 @@ def check(self): resp = self.get("/api/mirrors/" + mirror_name + "/packages") self.check_equal(resp.status_code, 404) - mirror_desc["Name"] = self.random_name() resp = self.put_task("/api/mirrors/" + mirror_name, json=mirror_desc) self.check_task(resp) _id = resp.json()['ID'] diff --git a/system/t12_api/publish.py b/system/t12_api/publish.py index 827515796f..25ba498d84 100644 --- a/system/t12_api/publish.py +++ b/system/t12_api/publish.py @@ -1,6 +1,7 @@ import inspect import os import threading +import re from api_lib import TASK_SUCCEEDED, APITest @@ -221,8 +222,6 @@ def check(self): "Distribution": "squeeze", "NotAutomatic": "yes", "ButAutomaticUpgrades": "yes", - "Origin": "earth", - "Label": "fun", } ) self.check_task(task) @@ -237,8 +236,8 @@ def check(self): 'Architectures': ['i386'], 'Codename': '', 'Distribution': 'squeeze', - 'Label': 'fun', - 'Origin': 'earth', + 'Label': '', + 'Origin': '', 'MultiDist': False, 'NotAutomatic': 'yes', 'ButAutomaticUpgrades': 'yes', @@ -444,6 +443,156 @@ def check(self): self.check_not_exists("public/" + prefix + "dists/") +class PublishUpdateAPIMultiDistToggle(APITest): + """ + POST /publish/:prefix with MultiDist=false, then PUT to enable MultiDist=true + """ + fixtureGpg = True + + def check(self): + repo_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo_name, "DefaultDistribution": "bookworm"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + task = self.post_task("/api/repos/" + repo_name + "/file/" + d) + self.check_task(task) + + # Publish with MultiDist=false (default) + prefix = self.random_name() + task = self.post_task( + "/api/publish/" + prefix, + json={ + "Architectures": ["i386", "source"], + "SourceKind": "local", + "Sources": [{"Name": repo_name}], + "Signing": DefaultSigningOptions, + "MultiDist": False, + } + ) + self.check_task(task) + + repo_expected = { + 'AcquireByHash': False, + 'Architectures': ['i386', 'source'], + 'Codename': '', + 'Distribution': 'bookworm', + 'Label': '', + 'Origin': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Path': prefix + '/' + 'bookworm', + 'Prefix': prefix, + 'SkipContents': False, + 'MultiDist': False, + 'SourceKind': 'local', + 'Sources': [{'Component': 'main', 'Name': repo_name}], + 'Storage': '', + 'Suite': ''} + + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) + + # With MultiDist=false packages are stored under pool/main/... + self.check_exists("public/" + prefix + "/dists/bookworm/Release") + self.check_exists("public/" + prefix + + "/dists/bookworm/main/binary-i386/Packages") + self.check_exists("public/" + prefix + + "/dists/bookworm/main/source/Sources") + self.check_exists( + "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists( + "public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + # MultiDist-style per-distribution pool must not exist yet + self.check_not_exists( + "public/" + prefix + "/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + + # Now update the published repo enabling MultiDist=true + task = self.put_task( + "/api/publish/" + prefix + "/bookworm", + json={ + "MultiDist": True, + "Signing": DefaultSigningOptions, + } + ) + self.check_task(task) + + repo_expected_multidist = { + 'AcquireByHash': False, + 'Architectures': ['i386', 'source'], + 'Codename': '', + 'Distribution': 'bookworm', + 'Label': '', + 'Origin': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Path': prefix + '/' + 'bookworm', + 'Prefix': prefix, + 'SkipContents': False, + 'MultiDist': True, + 'SourceKind': 'local', + 'Sources': [{'Component': 'main', 'Name': repo_name}], + 'Storage': '', + 'Suite': ''} + + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected_multidist, all_repos.json()) + + # After enabling MultiDist, packages are stored under pool//main/... + self.check_exists("public/" + prefix + "/dists/bookworm/Release") + self.check_exists("public/" + prefix + + "/dists/bookworm/main/binary-i386/Packages") + self.check_exists("public/" + prefix + + "/dists/bookworm/main/source/Sources") + self.check_exists( + "public/" + prefix + "/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists( + "public/" + prefix + "/pool/bookworm/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + # Flat pool must not exist while MultiDist is on + self.check_not_exists( + "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + + # Switch MultiDist back to false + task = self.put_task( + "/api/publish/" + prefix + "/bookworm", + json={ + "MultiDist": False, + "Signing": DefaultSigningOptions, + } + ) + self.check_task(task) + + repo_expected["MultiDist"] = False + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) + + # Packages are back under the flat pool/main/... + self.check_exists("public/" + prefix + "/dists/bookworm/Release") + self.check_exists("public/" + prefix + + "/dists/bookworm/main/binary-i386/Packages") + self.check_exists("public/" + prefix + + "/dists/bookworm/main/source/Sources") + self.check_exists( + "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists( + "public/" + prefix + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + # Per-distribution pool must be gone + self.check_not_exists( + "public/" + prefix + "/pool/bookworm/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + + task = self.delete_task("/api/publish/" + prefix + "/bookworm") + self.check_task(task) + self.check_not_exists("public/" + prefix + "dists/") + + class PublishConcurrentUpdateAPITestRepo(APITest): """ PUT /publish/:prefix/:distribution (local repos), DELETE /publish/:prefix/:distribution @@ -759,6 +908,219 @@ def check(self): self.check_not_exists("public/" + prefix + "dists/") +class PublishSwitchAPITestMirror(APITest): + """ + PUT /publish/:prefix/:distribution (snapshots), DELETE /publish/:prefix/:distribution + """ + fixtureGpg = True + + def check(self): + mirror_name = self.random_name() + mirror_desc = {'Name': mirror_name, + 'ArchiveURL': 'http://repo.aptly.info/system-tests/packagecloud.io/varnishcache/varnish30/debian/', + 'Distribution': 'wheezy', + 'Keyrings': ["aptlytest.gpg"], + 'Architectures': ["amd64"], + 'Components': ['main']} + mirror_desc['IgnoreSignatures'] = True + + # Create Mirror + resp = self.post("/api/mirrors", json=mirror_desc) + self.check_equal(resp.status_code, 201) + + # Get Mirror + resp = self.get("/api/mirrors/" + mirror_name + "/packages") + self.check_equal(resp.status_code, 404) + + # Update Mirror + resp = self.put_task("/api/mirrors/" + mirror_name, json=mirror_desc) + self.check_task(resp) + + # Snapshot Mirror + snapshot1_name = self.random_name() + task = self.post_task("/api/mirrors/" + mirror_name + '/snapshots', json={'Name': snapshot1_name}) + self.check_task(task) + + # Publish Snapshot + prefix = self.random_name() + task = self.post_task( + "/api/publish/" + prefix, + json={ + "Architectures": ["i386", "source"], + "SourceKind": "snapshot", + "Sources": [{"Name": snapshot1_name}], + "Signing": DefaultSigningOptions, + }) + self.check_task(task) + + repo_expected = { + 'AcquireByHash': False, + 'Architectures': ['i386', 'source'], + 'Codename': '', + 'Distribution': 'wheezy', + 'Label': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Origin': 'packagecloud.io/varnishcache/varnish30', + 'Path': prefix + '/' + 'wheezy', + 'Prefix': prefix, + 'SkipContents': False, + 'MultiDist': False, + 'SourceKind': 'snapshot', + 'Sources': [{'Component': 'main', 'Name': snapshot1_name}], + 'Storage': '', + 'Suite': ''} + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) + + # Snapshot Mirror 2 + snapshot2_name = self.random_name() + task = self.post_task("/api/mirrors/" + mirror_name + '/snapshots', json={'Name': snapshot2_name}) + self.check_task(task) + + task = self.put_task( + "/api/publish/" + prefix + "/wheezy", + json={ + "Snapshots": [{"Component": "main", "Name": snapshot2_name}], + "Signing": DefaultSigningOptions, + "SkipContents": True, + "Version": "13.3", + }) + self.check_task(task) + repo_expected = { + 'AcquireByHash': False, + 'Architectures': ['i386', 'source'], + 'Codename': '', + 'Distribution': 'wheezy', + 'Label': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Origin': 'packagecloud.io/varnishcache/varnish30', + 'Path': prefix + '/' + 'wheezy', + 'Prefix': prefix, + 'SkipContents': True, + 'MultiDist': False, + 'SourceKind': 'snapshot', + 'Sources': [{'Component': 'main', 'Name': snapshot2_name}], + 'Storage': '', + 'Suite': ''} + + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) + + task = self.delete_task("/api/publish/" + prefix + "/wheezy") + self.check_task(task) + self.check_not_exists("public/" + prefix + "dists/") + + +class PublishSwitchAPITestSnapshot(APITest): + """ + publish snapshot of snapshot + """ + fixtureGpg = True + + def check(self): + repo_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal( + self.upload("/api/files/" + d, + "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + task = self.post_task("/api/repos/" + repo_name + "/file/" + d) + self.check_task(task) + + snapshot1_name = self.random_name() + task = self.post_task("/api/repos/" + repo_name + '/snapshots', json={'Name': snapshot1_name}) + self.check_task(task) + + prefix = self.random_name() + task = self.post_task( + "/api/publish/" + prefix, + json={ + "Architectures": ["i386", "source"], + "SourceKind": "snapshot", + "Sources": [{"Name": snapshot1_name}], + "Signing": DefaultSigningOptions, + }) + self.check_task(task) + + repo_expected = { + 'AcquireByHash': False, + 'Architectures': ['i386', 'source'], + 'Codename': '', + 'Distribution': 'wheezy', + 'Label': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Origin': '', + 'Path': prefix + '/' + 'wheezy', + 'Prefix': prefix, + 'SkipContents': False, + 'MultiDist': False, + 'SourceKind': 'snapshot', + 'Sources': [{'Component': 'main', 'Name': snapshot1_name}], + 'Storage': '', + 'Suite': ''} + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) + + self.check_not_exists( + "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists("public/" + prefix + + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + + snapshot2_name = self.random_name() + task = self.post_task("/api/snapshots", json={"Name": snapshot2_name, 'SourceSnapshots': [snapshot1_name]}) + self.check_task(task) + + task = self.put_task( + "/api/publish/" + prefix + "/wheezy", + json={ + "Snapshots": [{"Component": "main", "Name": snapshot2_name}], + "Signing": DefaultSigningOptions, + "SkipContents": True, + "Version": "13.3", + }) + self.check_task(task) + repo_expected = { + 'AcquireByHash': False, + 'Architectures': ['i386', 'source'], + 'Codename': '', + 'Distribution': 'wheezy', + 'Label': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Origin': '', + 'Path': prefix + '/' + 'wheezy', + 'Prefix': prefix, + 'SkipContents': True, + 'MultiDist': False, + 'SourceKind': 'snapshot', + 'Sources': [{'Component': 'main', 'Name': snapshot2_name}], + 'Storage': '', + 'Suite': ''} + + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) + + self.check_not_exists( + "public/" + prefix + "/pool/main/b/boost-defaults/libboost-program-options-dev_1.49.0.1_i386.deb") + self.check_exists("public/" + prefix + + "/pool/main/p/pyspi/pyspi-0.6.1-1.3.stripped.dsc") + + task = self.delete_task("/api/publish/" + prefix + "/wheezy") + self.check_task(task) + self.check_not_exists("public/" + prefix + "dists/") + + class PublishSwitchAPISkipCleanupTestRepo(APITest): """ PUT /publish/:prefix/:distribution (snapshots), DELETE /publish/:prefix/:distribution @@ -1557,3 +1919,62 @@ def check(self): all_repos = self.get("/api/publish") self.check_equal(all_repos.status_code, 200) self.check_in(repo_expected, all_repos.json()) + + +class PublishAPITestDualSignature(APITest): + """ + POST /publish/:prefix (local repos), GET /publish + """ + fixtureGpg = True + + def check(self): + repo_name = self.random_name() + self.check_equal(self.post( + "/api/repos", json={"Name": repo_name, "DefaultDistribution": "wheezy"}).status_code, 201) + + d = self.random_name() + self.check_equal(self.upload("/api/files/" + d, + "libboost-program-options-dev_1.49.0.1_i386.deb", "pyspi_0.6.1-1.3.dsc", + "pyspi_0.6.1-1.3.diff.gz", "pyspi_0.6.1.orig.tar.gz", + "pyspi-0.6.1-1.3.stripped.dsc").status_code, 200) + + task = self.post_task("/api/repos/" + repo_name + "/file/" + d) + self.check_task(task) + + # publishing under prefix, default distribution + prefix = self.random_name() + task = self.post_task( + "/api/publish/" + prefix, + json={ + "SourceKind": "local", + "Sources": [{"Name": repo_name}], + "Signing": {"GPGKey": "C5ACD2179B5231DFE842EE6121DBB89C16DB3E6D,AEE16DF018354F67FE5F5C72BBF4E19434E91E4E"}, + } + ) + self.check_task(task) + repo_expected = { + 'AcquireByHash': False, + 'Architectures': ['i386', 'source'], + 'Codename': '', + 'Distribution': 'wheezy', + 'Label': '', + 'Origin': '', + 'NotAutomatic': '', + 'ButAutomaticUpgrades': '', + 'Path': prefix + '/' + 'wheezy', + 'Prefix': prefix, + 'SkipContents': False, + 'MultiDist': False, + 'SourceKind': 'local', + 'Sources': [{'Component': 'main', 'Name': repo_name}], + 'Storage': '', + 'Suite': ''} + + all_repos = self.get("/api/publish") + self.check_equal(all_repos.status_code, 200) + self.check_in(repo_expected, all_repos.json()) + + self.check_exists("public/" + prefix + "/dists/wheezy/Release") + path = os.path.join(os.environ["HOME"], self.aptlyDir, "public", prefix, "dists/wheezy") + self.check_cmd_output(f"gpg --verify {path}/Release.gpg {path}/Release", "Release.gpg", + match_prepare=lambda s: re.sub(r'Signature made .*', '', s)) diff --git a/system/t12_api/repos.py b/system/t12_api/repos.py index 424f9f49f3..6448a5576d 100644 --- a/system/t12_api/repos.py +++ b/system/t12_api/repos.py @@ -461,3 +461,34 @@ def check(self): self.check_equal(self.get(f"/api/repos/{repo2_name}/packages").json(), ['Pi386 libboost-program-options-dev 1.49.0.1 918d2f433384e378']) + + +class ReposAPITestCreateEdit(APITest): + """ + POST /api/repos, + """ + def check(self): + repo_name = self.random_name() + ' with space' + repo_desc = {'Comment': 'fun repo', + 'DefaultComponent': 'contrib', + 'DefaultDistribution': 'bookworm', + 'Name': repo_name} + + resp = self.post("/api/repos", json=repo_desc) + self.check_equal(resp.json(), repo_desc) + self.check_equal(resp.status_code, 201) + + repo_desc = {'Comment': 'modified repo', + 'DefaultComponent': 'main', + 'DefaultDistribution': 'trixie', + 'Name': repo_name + '@renamed'} + resp = self.put(f"/api/repos/{repo_name}", json=repo_desc) + self.check_equal(resp.json(), repo_desc) + self.check_equal(resp.status_code, 200) + + resp = self.get("/api/repos/" + repo_name + '@renamed') + self.check_equal(resp.json(), repo_desc) + self.check_equal(resp.status_code, 200) + + resp = self.delete("/api/repos/" + repo_name + '@renamed') + self.check_equal(resp.status_code, 200) diff --git a/system/testout.py b/system/testout.py index f7dc134d82..d63d6316a6 100644 --- a/system/testout.py +++ b/system/testout.py @@ -3,7 +3,7 @@ class TestOut: def __init__(self): - self.tmp_file = tempfile.NamedTemporaryFile(delete=False) + self.tmp_file = tempfile.NamedTemporaryFile(delete=True) self.read_pos = 0 def fileno(self): diff --git a/task/list.go b/task/list.go index 5b09d34c63..6a1a720d7d 100644 --- a/task/list.go +++ b/task/list.go @@ -44,45 +44,53 @@ func (list *List) consumer() { for { select { case task := <-list.queue: + // Set task state to RUNNING before processing list.Lock() - { - task.State = RUNNING - } + task.State = RUNNING list.Unlock() go func() { retValue, err := task.process(aptly.Progress(task.output), task.detail) + // Update task completion state and cleanup with list lock held list.Lock() { - task.processReturnValue = retValue - task.err = err if err != nil { task.output.Printf("Task failed with error: %v", err) task.State = FAILED + task.err = err + task.processReturnValue = retValue } else { task.output.Print("Task succeeded") task.State = SUCCEEDED + task.err = nil + task.processReturnValue = retValue } - list.usedResources.Free(task.resources) + list.usedResources.Free(task.Resources) task.wgTask.Done() list.wg.Done() + unlocked := false for _, t := range list.tasks { if t.State == IDLE { // check resources - blockingTasks := list.usedResources.UsedBy(t.resources) + blockingTasks := list.usedResources.UsedBy(t.Resources) if len(blockingTasks) == 0 { - list.usedResources.MarkInUse(task.resources, task) + list.usedResources.MarkInUse(t.Resources, t) + // unlock list since queueing may block + list.Unlock() + unlocked = true list.queue <- t break } } } + if !unlocked { + list.Unlock() + } } - list.Unlock() }() case <-list.queueDone: @@ -99,13 +107,15 @@ func (list *List) Stop() { // GetTasks gets complete list of tasks func (list *List) GetTasks() []Task { - tasks := []Task{} list.Lock() + defer list.Unlock() + + tasks := []Task{} for _, task := range list.tasks { + // Copy task while holding list lock tasks = append(tasks, *task) } - list.Unlock() return tasks } @@ -133,11 +143,11 @@ func (list *List) DeleteTaskByID(ID int) (Task, error) { // GetTaskByID returns task with given id func (list *List) GetTaskByID(ID int) (Task, error) { list.Lock() - tasks := list.tasks - list.Unlock() + defer list.Unlock() - for _, task := range tasks { + for _, task := range list.tasks { if task.ID == ID { + // Copy task while holding list lock return *task, nil } } @@ -174,20 +184,22 @@ func (list *List) GetTaskDetailByID(ID int) (interface{}, error) { // GetTaskReturnValueByID returns process return value of task with given id func (list *List) GetTaskReturnValueByID(ID int) (*ProcessReturnValue, error) { - task, err := list.GetTaskByID(ID) + list.Lock() + defer list.Unlock() - if err != nil { - return nil, err + for _, task := range list.tasks { + if task.ID == ID { + return task.processReturnValue, nil + } } - return task.processReturnValue, nil + return nil, fmt.Errorf("could not find task with id %v", ID) } // RunTaskInBackground creates task and runs it in background. This will block until the necessary resources // become available. func (list *List) RunTaskInBackground(name string, resources []string, process Process) (Task, *ResourceConflictError) { list.Lock() - defer list.Unlock() list.idCounter++ wgTask := &sync.WaitGroup{} @@ -199,20 +211,29 @@ func (list *List) RunTaskInBackground(name string, resources []string, process P list.wg.Add(1) task.wgTask.Add(1) + // Copy task while still holding the lock to avoid racing with consumer + // setting State=RUNNING after receiving from queue + taskCopy := *task + // add task to queue for processing if resources are available // if not, task will be queued by the consumer once resources are available tasks := list.usedResources.UsedBy(resources) if len(tasks) == 0 { - list.usedResources.MarkInUse(task.resources, task) + list.usedResources.MarkInUse(task.Resources, task) + // queueing task might block if channel not ready, unlock list before queueing + list.Unlock() list.queue <- task + } else { + list.Unlock() } - return *task, nil + return taskCopy, nil } // Clear removes finished tasks from list func (list *List) Clear() { list.Lock() + defer list.Unlock() var tasks []*Task for _, task := range list.tasks { @@ -221,8 +242,6 @@ func (list *List) Clear() { } } list.tasks = tasks - - list.Unlock() } // Wait waits till all tasks are processed @@ -245,11 +264,14 @@ func (list *List) WaitForTaskByID(ID int) (Task, error) { // GetTaskErrorByID returns the Task error for a given id func (list *List) GetTaskErrorByID(ID int) (error, error) { - task, err := list.GetTaskByID(ID) + list.Lock() + defer list.Unlock() - if err != nil { - return nil, err + for _, task := range list.tasks { + if task.ID == ID { + return task.err, nil + } } - return task.err, nil + return nil, fmt.Errorf("could not find task with id %v", ID) } diff --git a/task/list_test.go b/task/list_test.go index 8bce5377c0..eb6ebddd1f 100644 --- a/task/list_test.go +++ b/task/list_test.go @@ -50,5 +50,5 @@ func (s *ListSuite) TestList(c *check.C) { c.Check(detail, check.Equals, "Details") _, deleteErr := list.DeleteTaskByID(task.ID) c.Check(deleteErr, check.IsNil) - list.Stop() + list.Stop() } diff --git a/task/task.go b/task/task.go index 02aa7037bb..72f606997d 100644 --- a/task/task.go +++ b/task/task.go @@ -42,6 +42,7 @@ const ( ) // Task represents as task in a queue encapsulates process code +// All fields are protected by List.Mutex - access task fields only while holding list.Lock() type Task struct { output *Output detail *Detail @@ -51,7 +52,7 @@ type Task struct { Name string ID int State State - resources []string + Resources []string wgTask *sync.WaitGroup } @@ -64,7 +65,7 @@ func NewTask(process Process, name string, ID int, resources []string, wgTask *s Name: name, ID: ID, State: IDLE, - resources: resources, + Resources: resources, wgTask: wgTask, } return task diff --git a/utils/config.go b/utils/config.go index 4cfac039bd..08ca70ca79 100644 --- a/utils/config.go +++ b/utils/config.go @@ -31,6 +31,7 @@ type ConfigStructure struct { // nolint: maligned // PPA PpaDistributorID string `json:"ppaDistributorID" yaml:"ppa_distributor_id"` PpaCodename string `json:"ppaCodename" yaml:"ppa_codename"` + PpaBaseURL string `json:"ppaBaseURL" yaml:"ppa_baseurl"` // Server ServeInAPIMode bool `json:"serveInAPIMode" yaml:"serve_in_api_mode"` @@ -49,9 +50,10 @@ type ConfigStructure struct { // nolint: maligned DownloadSourcePackages bool `json:"downloadSourcePackages" yaml:"download_sourcepackages"` // Signing - GpgProvider string `json:"gpgProvider" yaml:"gpg_provider"` - GpgDisableSign bool `json:"gpgDisableSign" yaml:"gpg_disable_sign"` - GpgDisableVerify bool `json:"gpgDisableVerify" yaml:"gpg_disable_verify"` + GpgProvider string `json:"gpgProvider" yaml:"gpg_provider"` + GpgDisableSign bool `json:"gpgDisableSign" yaml:"gpg_disable_sign"` + GpgDisableVerify bool `json:"gpgDisableVerify" yaml:"gpg_disable_verify"` + GpgKeys []string `json:"gpgKeys" yaml:"gpg_keys"` // Publishing SkipContentsPublishing bool `json:"skipContentsPublishing" yaml:"skip_contents_publishing"` @@ -226,6 +228,7 @@ var Config = ConfigStructure{ GpgProvider: "gpg", GpgDisableSign: false, GpgDisableVerify: false, + GpgKeys: []string{}, DownloadSourcePackages: false, PackagePoolStorage: PackagePoolStorage{ Local: &LocalPoolStorage{Path: ""}, @@ -233,6 +236,7 @@ var Config = ConfigStructure{ SkipLegacyPool: false, PpaDistributorID: "ubuntu", PpaCodename: "", + PpaBaseURL: "http://ppa.launchpad.net", FileSystemPublishRoots: map[string]FileSystemPublishRoot{}, S3PublishRoots: map[string]S3PublishRoot{}, SwiftPublishRoots: map[string]SwiftPublishRoot{}, diff --git a/utils/config_test.go b/utils/config_test.go index da6f927e8b..c354596407 100644 --- a/utils/config_test.go +++ b/utils/config_test.go @@ -85,6 +85,7 @@ func (s *ConfigSuite) TestSaveConfig(c *C) { " \"dependencyVerboseResolve\": false,\n" + " \"ppaDistributorID\": \"\",\n" + " \"ppaCodename\": \"\",\n" + + " \"ppaBaseURL\": \"\",\n" + " \"serveInAPIMode\": false,\n" + " \"enableMetricsEndpoint\": false,\n" + " \"enableSwaggerEndpoint\": false,\n" + @@ -102,6 +103,7 @@ func (s *ConfigSuite) TestSaveConfig(c *C) { " \"gpgProvider\": \"gpg\",\n" + " \"gpgDisableSign\": false,\n" + " \"gpgDisableVerify\": false,\n" + + " \"gpgKeys\": null,\n" + " \"skipContentsPublishing\": false,\n" + " \"skipBz2Publishing\": false,\n" + " \"FileSystemPublishEndpoints\": {\n" + @@ -251,6 +253,7 @@ func (s *ConfigSuite) TestSaveYAML2Config(c *C) { "dep_verboseresolve: false\n" + "ppa_distributor_id: \"\"\n" + "ppa_codename: \"\"\n" + + "ppa_baseurl: \"\"\n" + "serve_in_api_mode: false\n" + "enable_metrics_endpoint: false\n" + "enable_swagger_endpoint: false\n" + @@ -267,6 +270,7 @@ func (s *ConfigSuite) TestSaveYAML2Config(c *C) { "gpg_provider: \"\"\n" + "gpg_disable_sign: false\n" + "gpg_disable_verify: false\n" + + "gpg_keys: []\n" + "skip_contents_publishing: false\n" + "skip_bz2_publishing: false\n" + "filesystem_publish_endpoints: {}\n" + @@ -306,6 +310,7 @@ dep_follow_source: true dep_verboseresolve: true ppa_distributor_id: Ubuntu ppa_codename: code +ppa_baseurl: http://ppa.launchpad.net serve_in_api_mode: true enable_metrics_endpoint: true enable_swagger_endpoint: true @@ -322,6 +327,7 @@ download_sourcepackages: true gpg_provider: gpg gpg_disable_sign: true gpg_disable_verify: true +gpg_keys: [] skip_contents_publishing: true skip_bz2_publishing: true filesystem_publish_endpoints: diff --git a/utils/utils_test.go b/utils/utils_test.go index 17c46f1325..4c304e53bf 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -34,7 +34,7 @@ func (s *UtilsSuite) TestDirIsAccessibleNotExist(c *C) { func (s *UtilsSuite) TestDirIsAccessibleNotAccessible(c *C) { accessible := DirIsAccessible(s.tempfile.Name()) if accessible == nil { - c.Fatalf("Test dir should not be accessible: %s", s.tempfile.Name()) - } + c.Fatalf("Test dir should not be accessible: %s", s.tempfile.Name()) + } c.Check(accessible.Error(), Equals, fmt.Errorf("'%s' is inaccessible, check access rights", s.tempfile.Name()).Error()) }