Skip to content

Commit 34fccc7

Browse files
leaftyleafty
authored
build: bump rclone to version 1.69.1 (#805)
Providers diff: ``` - amazon cloud drive + azurefiles + cloudinary + filescom + gofile + iclouddrive + imagekit + linkbox + pixeldrain + quatrix + ulozto ```
1 parent 7e1be25 commit 34fccc7

15 files changed

Lines changed: 14257 additions & 8022 deletions

File tree

‎.devcontainer/devcontainer.json‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,14 @@
2929
"ghcr.io/EliiseS/devcontainer-features/bash-profile:1": {
3030
"command": "alias k=kubectl"
3131
},
32-
"ghcr.io/devcontainers-extra/features/rclone:1": {},
3332
"./k3d": {},
3433
"ghcr.io/devcontainers/features/java:1": {
3534
"version": "21",
3635
"jdkDistro": "open"
3736
},
38-
"./solr": {}
37+
"./solr": {},
38+
"ghcr.io/devcontainers/features/go:1": {},
39+
"./rclone": {}
3940
},
4041
"overrideFeatureInstallOrder": [
4142
"ghcr.io/devcontainers-extra/features/poetry",
@@ -45,12 +46,12 @@
4546
"customizations": {
4647
"vscode": {
4748
"extensions": [
48-
"ms-python.python",
49-
"ms-python.vscode-pylance",
50-
"ms-vscode.makefile-tools",
49+
"charliermarsh.ruff",
5150
"matangover.mypy",
5251
"ms-python.mypy-type-checker",
53-
"charliermarsh.ruff"
52+
"ms-python.python",
53+
"ms-python.vscode-pylance",
54+
"ms-vscode.makefile-tools"
5455
]
5556
}
5657
}

‎.devcontainer/rclone/devcontainer-feature.json‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
{
2+
"id": "rclone",
3+
"version": "1.0.0",
4+
"name": "A feature adding a custom version of rclone",
5+
"postCreateCommand": "rclone --version",
6+
"installsAfter": [
7+
"ghcr.io/devcontainers-contrib/features/bash-command",
8+
"ghcr.io/devcontainers/features/go"
9+
],
10+
"options": {
11+
"rclone_repository": {
12+
"type": "string",
13+
"description": "rclone repository",
14+
"proposals": [
15+
"https://github.com/SwissDataScienceCenter/rclone.git",
16+
"https://github.com/rclone/rclone.git"
17+
],
18+
"default": "https://github.com/rclone/rclone.git"
19+
},
20+
"rclone_ref": {
21+
"type": "string",
22+
"description": "git reference",
23+
"proposals": [
24+
"master"
25+
],
26+
"default": "v1.69.1"
27+
}
28+
}
29+
}

‎.devcontainer/rclone/install.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/bin/bash
2+
3+
set -ex
4+
5+
echo "Downloading rclone sources from ${RCLONE_REPOSITORY}:${RCLONE_REF}"
6+
mkdir -p /tmp/rclone
7+
cd /tmp/rclone
8+
git clone "${RCLONE_REPOSITORY}"
9+
cd rclone
10+
git checkout "${RCLONE_REF}"
11+
12+
echo "Building rclone"
13+
make rclone
14+
cd $HOME
15+
rm -rf /tmp/rclone

‎components/renku_data_services/data_connectors/api.spec.yaml‎

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -683,6 +683,9 @@ components:
683683
provider:
684684
type: string
685685
description: The provider this value is applicable for. Empty if valid for all providers.
686+
required:
687+
- value
688+
- help
686689
required:
687690
type: boolean
688691
description: whether the option is required or not
@@ -701,7 +704,18 @@ components:
701704
type:
702705
type: string
703706
description: data type of option value. RClone has more options but they map to the ones listed here.
704-
enum: ["int", "bool", "string", "Time", "Duration", "MultiEncoder", "SizeSuffix", "SpaceSepList", "CommaSepList", "Tristate"]
707+
enum: ["int", "bool", "string", "Time", "Duration", "MultiEncoder", "SizeSuffix", "SpaceSepList", "CommaSepList", "Tristate", "Encoding", "Bits"]
708+
required:
709+
- name
710+
- help
711+
- default
712+
- default_str
713+
- required
714+
- ispassword
715+
- sensitive
716+
- advanced
717+
- exclusive
718+
- type
705719
Ulid:
706720
description: ULID identifier
707721
type: string

‎components/renku_data_services/data_connectors/apispec.py‎

Lines changed: 23 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# generated by datamodel-codegen:
22
# filename: api.spec.yaml
3-
# timestamp: 2025-04-04T08:05:02+00:00
3+
# timestamp: 2025-04-17T09:44:12+00:00
44

55
from __future__ import annotations
66

@@ -13,10 +13,8 @@
1313

1414

1515
class Example(BaseAPISpec):
16-
value: Optional[str] = Field(
17-
None, description="a potential value for the option (think enum)"
18-
)
19-
help: Optional[str] = Field(None, description="help text for the value")
16+
value: str = Field(..., description="a potential value for the option (think enum)")
17+
help: str = Field(..., description="help text for the value")
2018
provider: Optional[str] = Field(
2119
None,
2220
description="The provider this value is applicable for. Empty if valid for all providers.",
@@ -34,45 +32,45 @@ class Type(Enum):
3432
SpaceSepList = "SpaceSepList"
3533
CommaSepList = "CommaSepList"
3634
Tristate = "Tristate"
35+
Encoding = "Encoding"
36+
Bits = "Bits"
3737

3838

3939
class RCloneOption(BaseAPISpec):
40-
name: Optional[str] = Field(None, description="name of the option")
41-
help: Optional[str] = Field(None, description="help text for the option")
40+
name: str = Field(..., description="name of the option")
41+
help: str = Field(..., description="help text for the option")
4242
provider: Optional[str] = Field(
4343
None,
4444
description="The cloud provider the option is for (See 'provider' RCloneOption in the schema for potential values)",
4545
examples=["AWS"],
4646
)
47-
default: Optional[Union[float, str, bool, Dict[str, Any], List]] = Field(
48-
None, description="default value for the option"
47+
default: Union[float, str, bool, Dict[str, Any], List] = Field(
48+
..., description="default value for the option"
4949
)
50-
default_str: Optional[str] = Field(
51-
None, description="string representation of the default value"
50+
default_str: str = Field(
51+
..., description="string representation of the default value"
5252
)
5353
examples: Optional[List[Example]] = Field(
5454
None,
5555
description="These list potential values for this option, like an enum. With `exclusive: true`, only a value from the list is allowed.",
5656
)
57-
required: Optional[bool] = Field(
58-
None, description="whether the option is required or not"
59-
)
60-
ispassword: Optional[bool] = Field(
61-
None, description="whether the field is a password (use **** for display)"
57+
required: bool = Field(..., description="whether the option is required or not")
58+
ispassword: bool = Field(
59+
..., description="whether the field is a password (use **** for display)"
6260
)
63-
sensitive: Optional[bool] = Field(
64-
None,
61+
sensitive: bool = Field(
62+
...,
6563
description="whether the value is sensitive (not stored in the service). Do not send this in requests to the service.",
6664
)
67-
advanced: Optional[bool] = Field(
68-
None,
65+
advanced: bool = Field(
66+
...,
6967
description="whether this is an advanced config option (probably don't show these to users)",
7068
)
71-
exclusive: Optional[bool] = Field(
72-
None, description="if true, only values from 'examples' can be used"
69+
exclusive: bool = Field(
70+
..., description="if true, only values from 'examples' can be used"
7371
)
74-
type: Optional[Type] = Field(
75-
None,
72+
type: Type = Field(
73+
...,
7674
description="data type of option value. RClone has more options but they map to the ones listed here.",
7775
)
7876

@@ -207,7 +205,7 @@ class CloudStorageCorePatch(BaseAPISpec):
207205
examples=["my/project/folder"],
208206
)
209207
readonly: Optional[bool] = Field(
210-
None, description="Whether this storage should be mounted readonly or not"
208+
True, description="Whether this storage should be mounted readonly or not"
211209
)
212210

213211

‎components/renku_data_services/data_connectors/core.py‎

Lines changed: 17 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,8 @@
33
from dataclasses import asdict
44
from typing import Any
55

6+
from pydantic import ValidationError as PydanticValidationError
7+
68
from renku_data_services import base_models, errors
79
from renku_data_services.authz.models import Visibility
810
from renku_data_services.base_models.core import (
@@ -18,9 +20,21 @@ def dump_storage_with_sensitive_fields(
1820
storage: models.CloudStorageCore, validator: RCloneValidator
1921
) -> models.CloudStorageCoreWithSensitiveFields:
2022
"""Add sensitive fields to a storage configuration."""
21-
return models.CloudStorageCoreWithSensitiveFields(
22-
sensitive_fields=list(validator.get_private_fields(storage.configuration)), **asdict(storage)
23-
)
23+
try:
24+
body = models.CloudStorageCoreWithSensitiveFields(
25+
sensitive_fields=[
26+
apispec.RCloneOption.model_validate(option.model_dump(exclude_none=True, by_alias=True))
27+
for option in validator.get_private_fields(storage.configuration)
28+
],
29+
**asdict(storage),
30+
)
31+
except PydanticValidationError as err:
32+
parts = [".".join(str(i) for i in field["loc"]) + ": " + field["msg"] for field in err.errors()]
33+
message = (
34+
f"The server could not construct a valid response. Errors found in the following fields: {', '.join(parts)}"
35+
)
36+
raise errors.ProgrammingError(message=message) from err
37+
return body
2438

2539

2640
def validate_unsaved_storage(

‎components/renku_data_services/data_connectors/models.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
from renku_data_services.utils.etag import compute_etag_from_fields
1919

2020
if TYPE_CHECKING:
21-
from renku_data_services.storage.rclone import RCloneOption
21+
from renku_data_services.data_connectors.apispec import RCloneOption
2222

2323

2424
@dataclass(frozen=True, eq=True, kw_only=True)
@@ -113,6 +113,7 @@ class DataConnectorPatch:
113113
class CloudStorageCoreWithSensitiveFields(CloudStorageCore):
114114
"""Remote storage configuration model with sensitive fields."""
115115

116+
# sensitive_fields: list["RCloneOption"]
116117
sensitive_fields: list["RCloneOption"]
117118

118119

‎components/renku_data_services/storage/api.spec.yaml‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,6 +365,11 @@ components:
365365
type: array
366366
items:
367367
$ref: "#/components/schemas/RCloneOption"
368+
required:
369+
- name
370+
- description
371+
- prefix
372+
- options
368373
RCloneOption:
369374
type: object
370375
description: Single field on an RClone storage, like "remote" or "access_key_id"
@@ -405,6 +410,9 @@ components:
405410
provider:
406411
type: string
407412
description: The provider this value is applicable for. Empty if valid for all providers.
413+
required:
414+
- value
415+
- help
408416
required:
409417
type: boolean
410418
description: whether the option is required or not
@@ -423,7 +431,18 @@ components:
423431
type:
424432
type: string
425433
description: data type of option value. RClone has more options but they map to the ones listed here.
426-
enum: ["int", "bool", "string", "Time", "Duration", "MultiEncoder", "SizeSuffix", "SpaceSepList", "CommaSepList", "Tristate"]
434+
enum: ["int", "bool", "string", "Time", "Duration", "MultiEncoder", "SizeSuffix", "SpaceSepList", "CommaSepList", "Tristate", "Encoding", "Bits"]
435+
required:
436+
- name
437+
- help
438+
- default
439+
- default_str
440+
- required
441+
- ispassword
442+
- sensitive
443+
- advanced
444+
- exclusive
445+
- type
427446
Ulid:
428447
description: ULID identifier
429448
type: string

0 commit comments

Comments
 (0)