Merge pull request #40 from rdenham/type_error

Handle server API differences in file response fields

Author: multimeric

filesender/api.py

@@ -99,6 +99,18 @@ def iter_files(paths: Iterable[Path], root: Optional[Path] = None) -> Iterable[T
                 # If this is a nested file, use the relative path from the root directory as the name
                 yield str(path.relative_to(root)), path
 
+def _file_key(file_info: response.File) -> str:
+    """
+    Returns the per-file upload key, checking both ``uid`` (older FileSender servers)
+    and ``puid`` (newer FileSender servers). The field was renamed in
+    https://github.com/filesender/filesender/pull/2429.
+    """
+    if "uid" in file_info:
+        return file_info["uid"]
+    if "puid" in file_info:
+        return file_info["puid"]
+    raise Exception(f"File response for {file_info['name']!r} has neither 'uid' nor 'puid' field")
+
 class EndpointHandler:
     base: str
 
@@ -292,7 +304,7 @@ async def update_file(
             self.http_client.build_request(
                 "PUT",
                 self.urls.file(file_info['id']),
-                params={"key": file_info["uid"]},
+                params={"key": _file_key(file_info)},
                 json=body,
             )
         )
@@ -322,7 +334,7 @@ async def _task_generator(chunk_size: int) -> AsyncIterator[Tuple[response.File,
             task_limit=self.concurrent_chunks
         )
        ).stream() as streamer:
-            async for _ in tqdm(streamer, total=math.ceil(file_info["size"] / self.chunk_size), desc=file_info["name"]): # type: ignore
+            async for _ in tqdm(streamer, total=math.ceil(int(file_info["size"]) / self.chunk_size), desc=file_info["name"]):
                 pass
 
     async def _upload_chunk(
@@ -338,7 +350,7 @@ async def _upload_chunk(
             self.http_client.build_request(
                 "PUT",
                 self.urls.chunk(file_info["id"], offset),
-                params={"key": file_info["uid"]},
+                params={"key": _file_key(file_info)},
                 content=chunk,
                 headers={
                     "Content-Type": "application/octet-stream",

filesender/download.py

@@ -1,50 +1,53 @@
-from typing import Iterable, TypedDict
+from typing import Iterable, Optional, TypedDict
 
 from bs4 import BeautifulSoup
 
 
 class DownloadFile(TypedDict):
     client_entropy: str
     encrypted: str
-    encrypted_size: int
+    encrypted_size: Optional[int]
     fileaead: str
     fileiv: str
     id: int
     key_salt: str
-    key_version: int
+    key_version: Optional[int]
     mime: str
     #: filename
     name: str
     password_encoding: str
-    password_hash_iterations: int
-    password_version: int
+    password_hash_iterations: Optional[int]
+    password_version: Optional[int]
     size: int
     transferid: int
 
+def _opt_int(value: str) -> Optional[int]:
+    return int(value) if value else None
+
 def files_from_page(content: bytes) -> Iterable[DownloadFile]:
     """
     Yields dictionaries describing the files listed on a FileSender web page
 
     Params:
-        content: The HTML content of the FileSender download page 
+        content: The HTML content of the FileSender download page
     """
     for file in BeautifulSoup(content, "html.parser").find_all(
         class_="file"
     ):
         yield {
             "client_entropy": file.attrs[f"data-client-entropy"],
             "encrypted": file.attrs["data-encrypted"],
-            "encrypted_size": int(file.attrs["data-encrypted-size"]),
+            "encrypted_size": _opt_int(file.attrs["data-encrypted-size"]),
             "fileaead": file.attrs["data-fileaead"],
             "fileiv": file.attrs["data-fileiv"],
             "id": int(file.attrs["data-id"]),
             "key_salt": file.attrs["data-key-salt"],
-            "key_version": int(file.attrs["data-key-version"]),
+            "key_version": _opt_int(file.attrs["data-key-version"]),
             "mime": file.attrs["data-mime"],
             "name": file.attrs["data-name"],
             "password_encoding": file.attrs["data-password-encoding"],
-            "password_hash_iterations": int(file.attrs["data-password-hash-iterations"]),
-            "password_version": int(file.attrs["data-password-version"]),
+            "password_hash_iterations": _opt_int(file.attrs["data-password-hash-iterations"]),
+            "password_version": _opt_int(file.attrs["data-password-version"]),
             "size": int(file.attrs["data-size"]),
             "transferid": int(file.attrs["data-transferid"]),
         }

filesender/response_types.py

@@ -1,14 +1,22 @@
+from __future__ import annotations
 from typing import List
 from typing_extensions import TypedDict
 
-class File(TypedDict):
+class _FileBase(TypedDict):
     id: int
     transfer_id: int
-    uid: str
     name: str
-    size: int
+    size: int | str  # some FileSender servers return this as a string
     sha1: str
 
+class _FileWithUid(_FileBase):
+    uid: str
+
+class _FileWithPuid(_FileBase):
+    puid: str
+
+File = _FileWithUid | _FileWithPuid
+
 class Date(TypedDict):
     raw: int
     formatted: str

pyproject.toml

@@ -25,7 +25,8 @@ dependencies = [
     "cryptography",
     "typing_extensions",
     "tenacity",
-    "tqdm"
+    "tqdm",
+    "click>=8.1.8",
 ]
 
 [tool.setuptools.packages.find]

test/conftest.py

@@ -1,20 +1,22 @@
-from pytest import Parser, Metafunc
+from pytest import Parser, Metafunc, skip
 
 def pytest_addoption(parser: Parser):
-    # Require the user to provide these arguments
-    parser.addoption("--base-url", required=True)
-    parser.addoption("--apikey", required=True)
-    parser.addoption("--username", required=True)
+    parser.addoption("--base-url", required=False)
+    parser.addoption("--apikey", required=False)
+    parser.addoption("--username", required=False)
     parser.addoption("--delay", required=False, default="0")
-    parser.addoption("--recipient", help="Email address that will be used as the recipient of the invitations", required=True)
+    parser.addoption("--recipient", help="Email address that will be used as the recipient of the invitations", required=False)
 
 def pytest_generate_tests(metafunc: Metafunc):
     argnames = []
     argvalues = []
 
     for fixture in metafunc.fixturenames:
         if hasattr(metafunc.config.option, fixture):
+            value = getattr(metafunc.config.option, fixture)
+            if value is None:
+                skip(f"--{fixture} not provided")
             argnames.append(fixture)
-            argvalues.append(getattr(metafunc.config.option, fixture))
+            argvalues.append(value)
 
     metafunc.parametrize(argnames, [argvalues])

test/test_utils.py

@@ -1,6 +1,120 @@
-from filesender.api import iter_files
+from filesender.api import iter_files, _file_key
+from filesender.response_types import _FileWithUid, _FileWithPuid
+from filesender.download import files_from_page
 from pathlib import Path
 import tempfile
+import pytest
+
+
+# Minimal file dicts matching the structure of the real API response
+
+FILE_WITH_UID: _FileWithUid = {
+    "id": 28000001,
+    "transfer_id": 3500001,
+    "uid": "abc123def456",
+    "name": "example.txt",
+    "size": 1024,
+    "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+}
+
+FILE_WITH_PUID: _FileWithPuid = {
+    "id": 28622026,
+    "transfer_id": 3573766,
+    "puid": "91dc452a-7d5c-4e34-9bee-6bc44fa5e012",
+    "name": "final_train.h5",
+    "size": "5243079740",  # AARNet server returns size as a string
+    "sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709",
+}
+
+# Minimal HTML matching the structure of the FileSender download page
+
+DOWNLOAD_PAGE_ENCRYPTED = b"""
+<html><body>
+<div class="file"
+    data-id="1"
+    data-transfer-id="1"
+    data-name="secret.txt"
+    data-size="1024"
+    data-mime="text/plain"
+    data-encrypted="1"
+    data-encrypted-size="1040"
+    data-fileiv="aabbccdd"
+    data-fileaead="eeff0011"
+    data-key-version="1"
+    data-key-salt="saltsalt"
+    data-client-entropy="entropydata"
+    data-password-version="1"
+    data-password-encoding="base64"
+    data-password-hash-iterations="10000"
+    data-transferid="1">
+</div>
+</body></html>
+"""
+
+DOWNLOAD_PAGE_UNENCRYPTED = b"""
+<html><body>
+<div class="file"
+    data-id="2"
+    data-transfer-id="2"
+    data-name="plain.txt"
+    data-size="2048"
+    data-mime="text/plain"
+    data-encrypted="0"
+    data-encrypted-size=""
+    data-fileiv=""
+    data-fileaead=""
+    data-key-version=""
+    data-key-salt=""
+    data-client-entropy=""
+    data-password-version=""
+    data-password-encoding=""
+    data-password-hash-iterations=""
+    data-transferid="2">
+</div>
+</body></html>
+"""
+
+
+def test_file_key_uid():
+    assert _file_key(FILE_WITH_UID) == "abc123def456"
+
+
+def test_file_key_puid():
+    assert _file_key(FILE_WITH_PUID) == "91dc452a-7d5c-4e34-9bee-6bc44fa5e012"
+
+
+def test_file_key_neither():
+    file: _FileWithUid = {
+        "id": 1,
+        "transfer_id": 1,
+        "uid": "x",
+        "name": "test.txt",
+        "size": 0,
+        "sha1": "",
+    }
+    # Strip the uid at runtime to simulate a response with neither field
+    d = dict(file)
+    del d["uid"]
+    with pytest.raises(Exception, match="neither 'uid' nor 'puid'"):
+        _file_key(d)  # type: ignore[arg-type]
+
+
+def test_files_from_page_encrypted():
+    (file,) = files_from_page(DOWNLOAD_PAGE_ENCRYPTED)
+    assert file["name"] == "secret.txt"
+    assert file["size"] == 1024
+    assert file["encrypted_size"] == 1040
+    assert file["key_version"] == 1
+    assert file["password_hash_iterations"] == 10000
+
+
+def test_files_from_page_unencrypted():
+    (file,) = files_from_page(DOWNLOAD_PAGE_UNENCRYPTED)
+    assert file["name"] == "plain.txt"
+    assert file["size"] == 2048
+    assert file["encrypted_size"] is None
+    assert file["key_version"] is None
+    assert file["password_hash_iterations"] is None
 
 def test_iter_files():
     with tempfile.TemporaryDirectory() as _tempdir: