Skip to content

Commit b79c583

Browse files
JR-1991JR-1991
committed
Merge branch 'main' into add-native-progress
2 parents e53b0a3 + d18b6f7 commit b79c583

5 files changed

Lines changed: 145 additions & 33 deletions

File tree

‎dvuploader/dvuploader.py‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -311,7 +311,15 @@ def _get_file_id(
311311
# Find the file that matches label and directory_label
312312
for ds_file in ds_files:
313313
dspath = os.path.join(ds_file.get("directoryLabel", ""), ds_file["label"])
314-
fpath = os.path.join(file.directory_label, file.file_name) # type: ignore
314+
315+
if file.directory_label:
316+
fpath = os.path.join(file.directory_label, file.file_name) # type: ignore
317+
elif file.file_name:
318+
fpath = file.file_name
319+
else:
320+
raise ValueError(
321+
f"File {file.file_name} has no directory label or file name."
322+
)
315323

316324
if dspath == fpath:
317325
return ds_file["dataFile"]["id"]

‎dvuploader/file.py‎

Lines changed: 66 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -44,20 +44,72 @@ class File(BaseModel):
4444
arbitrary_types_allowed=True,
4545
)
4646

47-
filepath: str = Field(..., exclude=True)
48-
handler: Union[BytesIO, StringIO, IO, None] = Field(default=None, exclude=True)
49-
description: str = ""
50-
directory_label: str = Field(default="", alias="directoryLabel")
51-
mimeType: str = "application/octet-stream"
52-
categories: Optional[List[str]] = ["DATA"]
53-
restrict: bool = False
54-
checksum_type: ChecksumTypes = Field(default=ChecksumTypes.MD5, exclude=True)
55-
storageIdentifier: Optional[str] = None
56-
file_name: Optional[str] = Field(default=None, alias="fileName")
57-
checksum: Optional[Checksum] = None
58-
to_replace: bool = False
59-
file_id: Optional[Union[str, int]] = Field(default=None, alias="fileToReplaceId")
60-
tab_ingest: bool = Field(default=True, alias="tabIngest")
47+
filepath: str = Field(
48+
...,
49+
exclude=True,
50+
description="The path to the file",
51+
)
52+
handler: Union[BytesIO, StringIO, IO, None] = Field(
53+
default=None,
54+
exclude=True,
55+
description="File handler for reading the file contents",
56+
)
57+
description: Optional[str] = Field(
58+
default=None,
59+
alias="description",
60+
description="The description of the file",
61+
)
62+
directory_label: Optional[str] = Field(
63+
default=None,
64+
alias="directoryLabel",
65+
description="The label of the directory where the file is stored",
66+
)
67+
mimeType: str = Field(
68+
default="application/octet-stream",
69+
description="The MIME type of the file",
70+
)
71+
categories: Optional[List[str]] = Field(
72+
default=["DATA"],
73+
alias="categories",
74+
description="The categories associated with the file",
75+
)
76+
restrict: bool = Field(
77+
default=False,
78+
alias="restrict",
79+
description="Indicates if the file is restricted",
80+
)
81+
checksum_type: ChecksumTypes = Field(
82+
default=ChecksumTypes.MD5,
83+
exclude=True,
84+
description="The type of checksum used for the file",
85+
)
86+
storageIdentifier: Optional[str] = Field(
87+
default=None,
88+
description="The identifier of the storage where the file is stored",
89+
)
90+
file_name: Optional[str] = Field(
91+
default=None,
92+
alias="fileName",
93+
description="The name of the file",
94+
)
95+
checksum: Optional[Checksum] = Field(
96+
default=None,
97+
description="The checksum of the file",
98+
)
99+
file_id: Optional[Union[str, int]] = Field(
100+
default=None,
101+
alias="fileToReplaceId",
102+
description="The ID of the file to replace",
103+
)
104+
tab_ingest: bool = Field(
105+
default=True,
106+
alias="tabIngest",
107+
description="Indicates if tabular ingest should be performed",
108+
)
109+
to_replace: bool = Field(
110+
default=False,
111+
description="Indicates if the file should be replaced",
112+
)
61113

62114
_size: int = PrivateAttr(default=0)
63115
_unchanged_data: bool = PrivateAttr(default=False)
@@ -126,7 +178,6 @@ def apply_checksum(self):
126178

127179
self.checksum.apply_checksum()
128180

129-
130181
def update_checksum_chunked(self, blocksize=2**20):
131182
"""Updates the checksum with data read from a file-like object in chunks.
132183
@@ -155,7 +206,6 @@ def update_checksum_chunked(self, blocksize=2**20):
155206

156207
self.handler.seek(0)
157208

158-
159209
def __del__(self):
160210
if self.handler is not None:
161211
self.handler.close()

‎dvuploader/nativeupload.py‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -364,17 +364,13 @@ def _get_json_data(file: File) -> Dict:
364364
Dict: Dictionary containing file metadata for the upload request.
365365
"""
366366

367-
metadata = {
368-
"description": file.description,
369-
"categories": file.categories,
370-
"restrict": file.restrict,
371-
"forceReplace": True,
367+
include = {
368+
"description",
369+
"categories",
370+
"restrict",
371+
"tabIngest",
372372
}
373-
374-
if file.directory_label:
375-
metadata["directoryLabel"] = file.directory_label
376-
377-
return metadata
373+
return file.model_dump(by_alias=True, exclude_none=True, include=include)
378374

379375

380376
async def _update_metadata(
@@ -409,7 +405,14 @@ async def _update_metadata(
409405
tasks = []
410406

411407
for file in files:
412-
dv_path = os.path.join(file.directory_label, file.file_name) # type: ignore
408+
if file.directory_label:
409+
dv_path = os.path.join(file.directory_label, file.file_name) # type: ignore
410+
elif file.file_name:
411+
dv_path = file.file_name
412+
else:
413+
raise ValueError(
414+
f"File {file.file_name} has no directory label or file name."
415+
)
413416

414417
try:
415418
if _tab_extension(dv_path) in file_mapping:
@@ -469,8 +472,6 @@ async def _update_single_metadata(
469472

470473
json_data = _get_json_data(file)
471474

472-
del json_data["forceReplace"]
473-
474475
# Send metadata as a readable byte stream
475476
# This is a workaround since "data" and "json"
476477
# does not work

‎tests/unit/test_cli.py‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,15 @@ def test_full_input(self):
2626
assert cli_input.dataverse_url == "https://demo.dataverse.org/"
2727
assert cli_input.persistent_id == "doi:10.70122/XXX/XXXXX"
2828

29+
actual_files = []
30+
for file in cli_input.files:
31+
if file.directory_label:
32+
actual_files.append((file.directory_label, file.file_name))
33+
else:
34+
actual_files.append(("", file.file_name))
35+
2936
assert len(cli_input.files) == 2
30-
assert sorted(
31-
[(file.directory_label, file.file_name) for file in cli_input.files]
32-
) == sorted(expected_files)
37+
assert sorted(actual_files) == sorted(expected_files)
3338

3439

3540
class TestCLIMain:

‎tests/unit/test_directupload.py‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
from dvuploader.directupload import (
55
_add_files_to_ds,
66
_validate_ticket_response,
7+
_prepare_registration,
78
)
89

910
from dvuploader.file import File
@@ -128,3 +129,50 @@ def test_raises_assertion_error_when_abort_field_missing(self):
128129
}
129130
with pytest.raises(AssertionError):
130131
_validate_ticket_response(response)
132+
133+
134+
class TestPrepareRegistration:
135+
def test_tab_ingest_is_set_correctly(self):
136+
files = [
137+
File(filepath="tests/fixtures/add_dir_files/somefile.txt"),
138+
File(
139+
filepath="tests/fixtures/add_dir_files/somefile.txt",
140+
tab_ingest=False, # type: ignore
141+
),
142+
File(
143+
filepath="tests/fixtures/add_dir_files/somefile.txt",
144+
restrict=True,
145+
),
146+
File(
147+
filepath="tests/fixtures/add_dir_files/somefile.txt",
148+
categories=["Test file"],
149+
),
150+
]
151+
registration = _prepare_registration(files, use_replace=False)
152+
expected_registration = [
153+
{
154+
"categories": ["DATA"],
155+
"mimeType": "application/octet-stream",
156+
"restrict": False,
157+
"tabIngest": True,
158+
},
159+
{
160+
"categories": ["DATA"],
161+
"mimeType": "application/octet-stream",
162+
"restrict": False,
163+
"tabIngest": False,
164+
},
165+
{
166+
"categories": ["DATA"],
167+
"mimeType": "application/octet-stream",
168+
"restrict": True,
169+
"tabIngest": True,
170+
},
171+
{
172+
"categories": ["Test file"],
173+
"mimeType": "application/octet-stream",
174+
"restrict": False,
175+
"tabIngest": True,
176+
},
177+
]
178+
assert registration == expected_registration

0 commit comments

Comments
 (0)