Skip to content

Commit 7975d7a

Browse files
leaftyleafty
authored
feat: add new backend to support DOIs - rclone 1.69.1 (#7)
Add support for DOIs on top of rclone [v1.69.1](https://github.com/rclone/rclone/releases/tag/v1.69.1).
1 parent 4e77a4f commit 7975d7a

20 files changed

Lines changed: 1721 additions & 50 deletions

‎.github/workflows/build-renku.yml‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
name: Build rclone for Renku
2+
3+
# Trigger the workflow on push or pull request
4+
on:
5+
push:
6+
branches:
7+
- "**"
8+
tags:
9+
- "**"
10+
pull_request:
11+
workflow_dispatch:
12+
inputs:
13+
manual:
14+
description: Manual run (bypass default conditions)
15+
type: boolean
16+
default: true
17+
18+
env:
19+
REGISTRY: ghcr.io
20+
IMAGE_NAME: ${{ github.repository }}
21+
22+
jobs:
23+
build:
24+
if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)
25+
timeout-minutes: 60
26+
strategy:
27+
fail-fast: false
28+
matrix:
29+
job_name: ["linux"]
30+
31+
include:
32+
- job_name: linux
33+
os: ubuntu-latest
34+
go: ">=1.23.0-rc.1"
35+
gotags: cmount
36+
build_flags: '-include "linux/amd64|linux/arm64"'
37+
deploy: true
38+
39+
name: ${{ matrix.job_name }}
40+
41+
runs-on: ${{ matrix.os }}
42+
43+
steps:
44+
- name: Checkout
45+
uses: actions/checkout@v4
46+
with:
47+
fetch-depth: 0
48+
49+
- name: Install Go
50+
uses: actions/setup-go@v5
51+
with:
52+
go-version: ${{ matrix.go }}
53+
check-latest: true
54+
55+
- name: Set environment variables
56+
shell: bash
57+
run: |
58+
echo 'GOTAGS=${{ matrix.gotags }}' >> $GITHUB_ENV
59+
echo 'BUILD_FLAGS=${{ matrix.build_flags }}' >> $GITHUB_ENV
60+
echo 'BUILD_ARGS=${{ matrix.build_args }}' >> $GITHUB_ENV
61+
if [[ "${{ matrix.goarch }}" != "" ]]; then echo 'GOARCH=${{ matrix.goarch }}' >> $GITHUB_ENV ; fi
62+
if [[ "${{ matrix.cgo }}" != "" ]]; then echo 'CGO_ENABLED=${{ matrix.cgo }}' >> $GITHUB_ENV ; fi
63+
64+
- name: Install Libraries on Linux
65+
shell: bash
66+
run: |
67+
sudo modprobe fuse
68+
sudo chmod 666 /dev/fuse
69+
sudo chown root:$USER /etc/fuse.conf
70+
sudo apt-get update
71+
sudo apt-get install -y fuse3 libfuse-dev rpm pkg-config git-annex git-annex-remote-rclone nfs-common
72+
if: matrix.os == 'ubuntu-latest'
73+
74+
- name: Install Libraries on macOS
75+
shell: bash
76+
run: |
77+
# https://github.com/Homebrew/brew/issues/15621#issuecomment-1619266788
78+
# https://github.com/orgs/Homebrew/discussions/4612#discussioncomment-6319008
79+
unset HOMEBREW_NO_INSTALL_FROM_API
80+
brew untap --force homebrew/core
81+
brew untap --force homebrew/cask
82+
brew update
83+
brew install --cask macfuse
84+
brew install git-annex git-annex-remote-rclone
85+
if: matrix.os == 'macos-latest'
86+
87+
- name: Install Libraries on Windows
88+
shell: powershell
89+
run: |
90+
$ProgressPreference = 'SilentlyContinue'
91+
choco install -y winfsp zip
92+
echo "CPATH=C:\Program Files\WinFsp\inc\fuse;C:\Program Files (x86)\WinFsp\inc\fuse" | Out-File -FilePath $env:GITHUB_ENV -Encoding utf8 -Append
93+
if ($env:GOARCH -eq "386") {
94+
choco install -y mingw --forcex86 --force
95+
echo "C:\\ProgramData\\chocolatey\\lib\\mingw\\tools\\install\\mingw32\\bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append
96+
}
97+
# Copy mingw32-make.exe to make.exe so the same command line
98+
# can be used on Windows as on macOS and Linux
99+
$path = (get-command mingw32-make.exe).Path
100+
Copy-Item -Path $path -Destination (Join-Path (Split-Path -Path $path) 'make.exe')
101+
if: matrix.os == 'windows-latest'
102+
103+
- name: Print Go version and environment
104+
shell: bash
105+
run: |
106+
printf "Using go at: $(which go)\n"
107+
printf "Go version: $(go version)\n"
108+
printf "\n\nGo environment:\n\n"
109+
go env
110+
printf "\n\nRclone environment:\n\n"
111+
make vars
112+
printf "\n\nSystem environment:\n\n"
113+
env
114+
115+
- name: Build rclone
116+
shell: bash
117+
run: |
118+
make
119+
120+
- name: Rclone version
121+
shell: bash
122+
run: |
123+
rclone version
124+
125+
- name: Run tests
126+
shell: bash
127+
run: |
128+
make quicktest
129+
if: matrix.quicktest
130+
131+
- name: Race test
132+
shell: bash
133+
run: |
134+
make racequicktest
135+
if: matrix.racequicktest
136+
137+
- name: Run librclone tests
138+
shell: bash
139+
run: |
140+
make -C librclone/ctest test
141+
make -C librclone/ctest clean
142+
librclone/python/test_rclone.py
143+
if: matrix.librclonetest
144+
145+
- name: Compile all architectures test
146+
shell: bash
147+
run: |
148+
make
149+
make compile_all
150+
if: matrix.compile_all
151+
152+
- name: Build binaries for release
153+
shell: bash
154+
run: |
155+
if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
156+
make ci_gha
157+
ls -al build/
158+
if: matrix.deploy
159+
160+
- name: Upload artifacts to GitHub
161+
uses: actions/upload-artifact@v4
162+
with:
163+
name: build-${{ matrix.job_name }}
164+
path: build
165+
retention-days: 1
166+
if: matrix.deploy
167+
168+
image:
169+
if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)
170+
timeout-minutes: 60
171+
runs-on: ubuntu-latest
172+
needs: [build]
173+
174+
steps:
175+
- name: Checkout
176+
uses: actions/checkout@v4
177+
with:
178+
fetch-depth: 0
179+
180+
- name: Download artifacts
181+
uses: actions/download-artifact@v4
182+
with:
183+
name: build-linux
184+
path: rclone
185+
186+
- name: List downloaded files
187+
run: ls -R rclone
188+
189+
- name: Extract rclone binary - amd64
190+
run: |
191+
unzip "rclone/*-amd64.zip" -d rclone-unzip
192+
ls -R rclone-unzip
193+
mkdir -p linux/amd64
194+
mv rclone-unzip/*/rclone linux/amd64/rclone
195+
- name: Extract rclone binary - arm64
196+
run: |
197+
unzip "rclone/*-arm64.zip" -d rclone-unzip
198+
ls -R rclone-unzip
199+
mkdir -p linux/arm64
200+
mv rclone-unzip/*/rclone linux/arm64/rclone
201+
- name: Docker image metadata
202+
id: meta
203+
uses: docker/metadata-action@v5
204+
with:
205+
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
206+
tags: type=sha
207+
208+
- name: Extract Docker image name
209+
id: docker_image
210+
env:
211+
IMAGE_TAGS: ${{ steps.meta.outputs.tags }}
212+
run: |
213+
IMAGE=$(echo "$IMAGE_TAGS" | cut -d" " -f1)
214+
IMAGE_REPOSITORY=$(echo "$IMAGE" | cut -d":" -f1)
215+
IMAGE_TAG=$(echo "$IMAGE" | cut -d":" -f2)
216+
echo "image=$IMAGE" >> "$GITHUB_OUTPUT"
217+
echo "image_repository=$IMAGE_REPOSITORY" >> "$GITHUB_OUTPUT"
218+
echo "image_tag=$IMAGE_TAG" >> "$GITHUB_OUTPUT"
219+
- name: Set up Docker buildx
220+
uses: docker/setup-buildx-action@v3
221+
222+
- name: Set up Docker
223+
uses: docker/login-action@v3
224+
with:
225+
registry: ${{ env.REGISTRY }}
226+
username: ${{ github.actor }}
227+
password: ${{ secrets.GITHUB_TOKEN }}
228+
229+
- name: Build and push Docker image
230+
uses: docker/build-push-action@v6
231+
with:
232+
context: .
233+
file: .renku/Dockerfile
234+
platforms: linux/amd64,linux/arm64
235+
push: true
236+
tags: ${{ steps.meta.outputs.tags }}
237+
labels: ${{ steps.meta.outputs.labels }}

‎.github/workflows/build.yml‎

Lines changed: 45 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -21,12 +21,12 @@ on:
2121

2222
jobs:
2323
build:
24-
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
24+
if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)
2525
timeout-minutes: 60
2626
strategy:
2727
fail-fast: false
2828
matrix:
29-
job_name: ['linux', 'linux_386', 'mac_amd64', 'mac_arm64', 'windows', 'other_os', 'go1.21', 'go1.22']
29+
job_name: ['linux', 'linux_386', 'go1.21', 'go1.22']
3030

3131
include:
3232
- job_name: linux
@@ -47,38 +47,38 @@ jobs:
4747
gotags: cmount
4848
quicktest: true
4949

50-
- job_name: mac_amd64
51-
os: macos-latest
52-
go: '>=1.23.0-rc.1'
53-
gotags: 'cmount'
54-
build_flags: '-include "^darwin/amd64" -cgo'
55-
quicktest: true
56-
racequicktest: true
57-
deploy: true
58-
59-
- job_name: mac_arm64
60-
os: macos-latest
61-
go: '>=1.23.0-rc.1'
62-
gotags: 'cmount'
63-
build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
64-
deploy: true
65-
66-
- job_name: windows
67-
os: windows-latest
68-
go: '>=1.23.0-rc.1'
69-
gotags: cmount
70-
cgo: '0'
71-
build_flags: '-include "^windows/"'
72-
build_args: '-buildmode exe'
73-
quicktest: true
74-
deploy: true
75-
76-
- job_name: other_os
77-
os: ubuntu-latest
78-
go: '>=1.23.0-rc.1'
79-
build_flags: '-exclude "^(windows/|darwin/|linux/)"'
80-
compile_all: true
81-
deploy: true
50+
# - job_name: mac_amd64
51+
# os: macos-latest
52+
# go: '>=1.23.0-rc.1'
53+
# gotags: 'cmount'
54+
# build_flags: '-include "^darwin/amd64" -cgo'
55+
# quicktest: true
56+
# racequicktest: true
57+
# deploy: true
58+
59+
# - job_name: mac_arm64
60+
# os: macos-latest
61+
# go: '>=1.23.0-rc.1'
62+
# gotags: 'cmount'
63+
# build_flags: '-include "^darwin/arm64" -cgo -macos-arch arm64 -cgo-cflags=-I/usr/local/include -cgo-ldflags=-L/usr/local/lib'
64+
# deploy: true
65+
66+
# - job_name: windows
67+
# os: windows-latest
68+
# go: '>=1.23.0-rc.1'
69+
# gotags: cmount
70+
# cgo: '0'
71+
# build_flags: '-include "^windows/"'
72+
# build_args: '-buildmode exe'
73+
# quicktest: true
74+
# deploy: true
75+
76+
# - job_name: other_os
77+
# os: ubuntu-latest
78+
# go: '>=1.23.0-rc.1'
79+
# build_flags: '-exclude "^(windows/|darwin/|linux/)"'
80+
# compile_all: true
81+
# deploy: true
8282

8383
- job_name: go1.21
8484
os: ubuntu-latest
@@ -205,19 +205,19 @@ jobs:
205205
make compile_all
206206
if: matrix.compile_all
207207

208-
- name: Deploy built binaries
209-
shell: bash
210-
run: |
211-
if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
212-
make ci_beta
213-
env:
214-
RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
215-
# working-directory: '$(modulePath)'
216-
# Deploy binaries if enabled in config && not a PR && not a fork
217-
if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
208+
# - name: Deploy built binaries
209+
# shell: bash
210+
# run: |
211+
# if [[ "${{ matrix.os }}" == "ubuntu-latest" ]]; then make release_dep_linux ; fi
212+
# make ci_beta
213+
# env:
214+
# RCLONE_CONFIG_PASS: ${{ secrets.RCLONE_CONFIG_PASS }}
215+
# # working-directory: '$(modulePath)'
216+
# # Deploy binaries if enabled in config && not a PR && not a fork
217+
# if: env.RCLONE_CONFIG_PASS != '' && matrix.deploy && github.head_ref == '' && github.repository == 'rclone/rclone'
218218

219219
lint:
220-
if: inputs.manual || (github.repository == 'rclone/rclone' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name))
220+
if: inputs.manual || (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name != github.event.pull_request.base.repo.full_name)
221221
timeout-minutes: 30
222222
name: "lint"
223223
runs-on: ubuntu-latest

‎.renku/Dockerfile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
FROM scratch
2+
ARG TARGETPLATFORM
3+
COPY ${TARGETPLATFORM}/rclone rclone

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -238,6 +238,9 @@ ifeq ($(or $(BRANCH_PATH),$(RELEASE_TAG)),)
238238
endif
239239
@echo Beta release ready at $(BETA_URL)
240240

241+
ci_gha:
242+
go run bin/cross-compile.go $(BUILD_FLAGS) $(BUILDTAGS) $(BUILD_ARGS) $(TAG)
243+
241244
# Fetch the binary builds from GitHub actions
242245
fetch_binaries:
243246
rclone -P sync --exclude "/testbuilds/**" --delete-excluded $(BETA_UPLOAD) build/

‎backend/all/all.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@ import (
1414
_ "github.com/rclone/rclone/backend/combine"
1515
_ "github.com/rclone/rclone/backend/compress"
1616
_ "github.com/rclone/rclone/backend/crypt"
17+
_ "github.com/rclone/rclone/backend/doi"
1718
_ "github.com/rclone/rclone/backend/drive"
1819
_ "github.com/rclone/rclone/backend/dropbox"
1920
_ "github.com/rclone/rclone/backend/fichier"

0 commit comments

Comments
 (0)