added tar filter for upload_from_dir

Author: james-rl

src/runloop_api_client/lib/context_loader.py

@@ -2,11 +2,13 @@
 
 import io
 import tarfile
-from typing import Iterable, Optional, Sequence
+from typing import Callable, Iterable, Optional, Sequence
 from pathlib import Path
 
 from ._ignore import IgnoreMatcher, IgnorePattern, DockerIgnoreMatcher, iter_included_files
 
+TarFilter = Callable[[tarfile.TarInfo], tarfile.TarInfo | None]
+
 
 def build_docker_context_tar(
     context_root: Path,
@@ -35,6 +37,28 @@ def build_docker_context_tar(
     return buf.getvalue()
 
 
+def build_directory_tar(
+    root: Path,
+    *,
+    tar_filter: TarFilter | None = None,
+) -> bytes:
+    """Create a .tar.gz archive containing all files under ``root``.
+
+    No ignore semantics are applied by default; callers may pass a tar filter
+    compatible with :meth:`tarfile.TarFile.add` to modify or exclude members.
+    """
+
+    root = root.resolve()
+    buf = io.BytesIO()
+    with tarfile.open(mode="w:gz", fileobj=buf) as tf:
+        for file_path in root.rglob("*"):
+            if not file_path.is_file():
+                continue
+            rel = file_path.relative_to(root)
+            tf.add(file_path, arcname=rel.as_posix(), filter=tar_filter)
+    return buf.getvalue()
+
+
 def _iter_build_context_files(
     context_root: Path,
     *,

src/runloop_api_client/sdk/async_.py

@@ -24,11 +24,10 @@
 from .._types import Timeout, NotGiven, not_given
 from .._client import DEFAULT_MAX_RETRIES, AsyncRunloop
 from ._helpers import detect_content_type
-from ..lib._ignore import IgnoreMatcher
 from .async_devbox import AsyncDevbox
 from .async_snapshot import AsyncSnapshot
 from .async_blueprint import AsyncBlueprint
-from ..lib.context_loader import build_docker_context_tar
+from ..lib.context_loader import TarFilter, build_directory_tar
 from .async_storage_object import AsyncStorageObject
 from ..types.object_create_params import ContentType
 
@@ -376,7 +375,7 @@ async def upload_from_dir(
         name: Optional[str] = None,
         metadata: Optional[Dict[str, str]] = None,
         ttl: Optional[timedelta] = None,
-        ignore: IgnoreMatcher | None = None,
+        ignore: TarFilter | None = None,
         **options: Unpack[LongRequestOptions],
     ) -> AsyncStorageObject:
         """Create and upload an object from a local directory.
@@ -391,11 +390,10 @@ async def upload_from_dir(
         :type metadata: Optional[Dict[str, str]]
         :param ttl: Optional Time-To-Live, after which the object is automatically deleted
         :type ttl: Optional[timedelta]
-        :param ignore: Optional ignore matcher. When provided it controls which
-            files under ``dir_path`` are included in the archived build
-            context. When omitted, a default Docker-style matcher that honors
-            ``.dockerignore`` under ``dir_path`` is used.
-        :type ignore: Optional[IgnoreMatcher]
+        :param ignore: Optional tar filter function compatible with
+            :meth:`tarfile.TarFile.add`. If provided, it will be called for each
+            member to allow modification or exclusion (by returning ``None``).
+        :type ignore: Optional[TarFilter]
         :param options: See :typeddict:`~runloop_api_client.sdk._types.LongRequestOptions`
             for available options
         :return: Wrapper for the uploaded object
@@ -410,7 +408,7 @@ async def upload_from_dir(
         ttl_ms = int(ttl.total_seconds()) * 1000 if ttl else None
 
         def synchronous_io() -> bytes:
-            return build_docker_context_tar(path, ignore=ignore)
+            return build_directory_tar(path, tar_filter=ignore)
 
         tar_bytes = await asyncio.to_thread(synchronous_io)
 

src/runloop_api_client/sdk/sync.py

@@ -26,9 +26,8 @@
 from ._helpers import detect_content_type
 from .snapshot import Snapshot
 from .blueprint import Blueprint
-from ..lib._ignore import IgnoreMatcher
 from .storage_object import StorageObject
-from ..lib.context_loader import build_docker_context_tar
+from ..lib.context_loader import TarFilter, build_directory_tar
 from ..types.object_create_params import ContentType
 
 
@@ -375,7 +374,7 @@ def upload_from_dir(
         name: Optional[str] = None,
         metadata: Optional[Dict[str, str]] = None,
         ttl: Optional[timedelta] = None,
-        ignore: IgnoreMatcher | None = None,
+        ignore: TarFilter | None = None,
         **options: Unpack[LongRequestOptions],
     ) -> StorageObject:
         """Create and upload an object from a local directory.
@@ -390,11 +389,10 @@ def upload_from_dir(
         :type metadata: Optional[Dict[str, str]]
         :param ttl: Optional Time-To-Live, after which the object is automatically deleted
         :type ttl: Optional[timedelta]
-        :param ignore: Optional ignore matcher. When provided it controls which
-            files under ``dir_path`` are included in the archived build
-            context. When omitted, a default Docker-style matcher that honors
-            ``.dockerignore`` under ``dir_path`` is used.
-        :type ignore: Optional[IgnoreMatcher]
+        :param ignore: Optional tar filter function compatible with
+            :meth:`tarfile.TarFile.add`. If provided, it will be called for each
+            member to allow modification or exclusion (by returning ``None``).
+        :type ignore: Optional[TarFilter]
         :param options: See :typeddict:`~runloop_api_client.sdk._types.LongRequestOptions`
             for available options
         :return: Wrapper for the uploaded object
@@ -408,7 +406,7 @@ def upload_from_dir(
         name = name or f"{path.name}.tar.gz"
         ttl_ms = int(ttl.total_seconds()) * 1000 if ttl else None
 
-        tar_bytes = build_docker_context_tar(path, ignore=ignore)
+        tar_bytes = build_directory_tar(path, tar_filter=ignore)
 
         obj = self.create(name=name, content_type="tgz", metadata=metadata, ttl_ms=ttl_ms, **options)
         obj.upload_content(tar_bytes)

tests/sdk/test_async_clients.py

@@ -25,7 +25,6 @@
     AsyncBlueprintOps,
     AsyncStorageObjectOps,
 )
-from runloop_api_client.lib._ignore import DockerIgnoreMatcher
 from runloop_api_client.lib.polling import PollingConfig
 
 
@@ -403,43 +402,6 @@ async def test_upload_from_dir(
 
         mock_async_client.objects.complete.assert_awaited_once()
 
-    @pytest.mark.asyncio
-    async def test_upload_from_dir_respects_dockerignore(
-        self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path
-    ) -> None:
-        """upload_from_dir should respect .dockerignore patterns by default."""
-        mock_async_client.objects.create = AsyncMock(return_value=object_view)
-        mock_async_client.objects.complete = AsyncMock(return_value=object_view)
-
-        test_dir = tmp_path / "ctx"
-        test_dir.mkdir()
-        (test_dir / "keep.txt").write_text("keep", encoding="utf-8")
-        (test_dir / "ignore.log").write_text("ignore", encoding="utf-8")
-        build_dir = test_dir / "build"
-        build_dir.mkdir()
-        (build_dir / "ignored.txt").write_text("ignored", encoding="utf-8")
-
-        dockerignore = test_dir / ".dockerignore"
-        dockerignore.write_text("*.log\nbuild/\n", encoding="utf-8")
-
-        http_client = AsyncMock()
-        mock_response = create_mock_httpx_response()
-        http_client.put = AsyncMock(return_value=mock_response)
-        mock_async_client._client = http_client
-
-        client = AsyncStorageObjectOps(mock_async_client)
-        obj = await client.upload_from_dir(test_dir)
-
-        assert isinstance(obj, AsyncStorageObject)
-        uploaded_content = http_client.put.call_args[1]["content"]
-
-        with tarfile.open(fileobj=io.BytesIO(uploaded_content), mode="r:gz") as tar:
-            names = {m.name for m in tar.getmembers()}
-
-        assert "keep.txt" in names
-        assert "ignore.log" not in names
-        assert not any(name.startswith("build/") for name in names)
-
     @pytest.mark.asyncio
     async def test_upload_from_dir_with_inline_ignore_patterns(
         self, mock_async_client: AsyncMock, object_view: MockObjectView, tmp_path: Path
@@ -462,8 +424,14 @@ async def test_upload_from_dir_with_inline_ignore_patterns(
         mock_async_client._client = http_client
 
         client = AsyncStorageObjectOps(mock_async_client)
-        matcher = DockerIgnoreMatcher(patterns=["*.log", "build/"])
-        obj = await client.upload_from_dir(test_dir, ignore=matcher)
+
+        # Tar filter: drop logs and anything under build/
+        def ignore_logs_and_build(ti: tarfile.TarInfo) -> tarfile.TarInfo | None:
+            if ti.name.endswith(".log") or ti.name.startswith("build/"):
+                return None
+            return ti
+
+        obj = await client.upload_from_dir(test_dir, ignore=ignore_logs_and_build)
 
         assert isinstance(obj, AsyncStorageObject)
         uploaded_content = http_client.put.call_args[1]["content"]

tests/sdk/test_clients.py

@@ -25,7 +25,6 @@
     BlueprintOps,
     StorageObjectOps,
 )
-from runloop_api_client.lib._ignore import DockerIgnoreMatcher
 from runloop_api_client.lib.polling import PollingConfig
 
 
@@ -483,10 +482,10 @@ def test_upload_from_dir_with_string_path(
         http_client.put.assert_called_once()
         mock_client.objects.complete.assert_called_once()
 
-    def test_upload_from_dir_respects_dockerignore(
+    def test_upload_from_dir_respects_filter(
         self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path
     ) -> None:
-        """upload_from_dir should respect .dockerignore patterns by default."""
+        """upload_from_dir should respect a tar filter when provided."""
         mock_client.objects.create.return_value = object_view
 
         test_dir = tmp_path / "ctx"
@@ -497,90 +496,20 @@ def test_upload_from_dir_respects_dockerignore(
         build_dir.mkdir()
         (build_dir / "ignored.txt").write_text("ignored", encoding="utf-8")
 
-        dockerignore = test_dir / ".dockerignore"
-        dockerignore.write_text("*.log\nbuild/\n", encoding="utf-8")
-
         http_client = Mock()
         mock_response = create_mock_httpx_response()
         http_client.put.return_value = mock_response
         mock_client._client = http_client
 
         client = StorageObjectOps(mock_client)
-        obj = client.upload_from_dir(test_dir)
-
-        assert isinstance(obj, StorageObject)
-        http_client.put.assert_called_once()
-        uploaded_content = http_client.put.call_args[1]["content"]
-
-        with tarfile.open(fileobj=io.BytesIO(uploaded_content), mode="r:gz") as tar:
-            names = {m.name for m in tar.getmembers()}
-
-        assert "keep.txt" in names
-        assert "ignore.log" not in names
-        assert not any(name.startswith("build/") for name in names)
 
-    def test_upload_from_dir_with_extra_ignore_file(
-        self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path
-    ) -> None:
-        """upload_from_dir should merge .dockerignore and an extra ignore file."""
-        mock_client.objects.create.return_value = object_view
-
-        test_dir = tmp_path / "ctx"
-        test_dir.mkdir()
-        (test_dir / "keep.txt").write_text("keep", encoding="utf-8")
-        (test_dir / "ignore.log").write_text("ignore", encoding="utf-8")
-        build_dir = test_dir / "build"
-        build_dir.mkdir()
-        (build_dir / "ignored.txt").write_text("ignored", encoding="utf-8")
-
-        # Only ignore logs in .dockerignore
-        dockerignore = test_dir / ".dockerignore"
-        dockerignore.write_text("*.log\n", encoding="utf-8")
-
-        extra_ignore = tmp_path / "extra.ignore"
-        extra_ignore.write_text("build/\n", encoding="utf-8")
-
-        http_client = Mock()
-        mock_response = create_mock_httpx_response()
-        http_client.put.return_value = mock_response
-        mock_client._client = http_client
+        # Tar filter: drop logs and anything under build/
+        def ignore_logs_and_build(ti: tarfile.TarInfo) -> tarfile.TarInfo | None:
+            if ti.name.endswith(".log") or ti.name.startswith("build/"):
+                return None
+            return ti
 
-        client = StorageObjectOps(mock_client)
-        matcher = DockerIgnoreMatcher(extra_ignorefile=extra_ignore)
-        obj = client.upload_from_dir(test_dir, ignore=matcher)
-
-        assert isinstance(obj, StorageObject)
-        uploaded_content = http_client.put.call_args[1]["content"]
-
-        with tarfile.open(fileobj=io.BytesIO(uploaded_content), mode="r:gz") as tar:
-            names = {m.name for m in tar.getmembers()}
-
-        assert "keep.txt" in names
-        assert "ignore.log" not in names
-        assert not any(name.startswith("build/") for name in names)
-
-    def test_upload_from_dir_with_inline_ignore_patterns(
-        self, mock_client: Mock, object_view: MockObjectView, tmp_path: Path
-    ) -> None:
-        """upload_from_dir should respect inline ignore patterns."""
-        mock_client.objects.create.return_value = object_view
-
-        test_dir = tmp_path / "ctx"
-        test_dir.mkdir()
-        (test_dir / "keep.txt").write_text("keep", encoding="utf-8")
-        (test_dir / "ignore.log").write_text("ignore", encoding="utf-8")
-        build_dir = test_dir / "build"
-        build_dir.mkdir()
-        (build_dir / "ignored.txt").write_text("ignored", encoding="utf-8")
-
-        http_client = Mock()
-        mock_response = create_mock_httpx_response()
-        http_client.put.return_value = mock_response
-        mock_client._client = http_client
-
-        client = StorageObjectOps(mock_client)
-        matcher = DockerIgnoreMatcher(patterns=["*.log", "build/"])
-        obj = client.upload_from_dir(test_dir, ignore=matcher)
+        obj = client.upload_from_dir(test_dir, ignore=ignore_logs_and_build)
 
         assert isinstance(obj, StorageObject)
         uploaded_content = http_client.put.call_args[1]["content"]