Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
137 changes: 137 additions & 0 deletions src/pyinfra/operations/files.py
Original file line number Diff line number Diff line change
Expand Up @@ -2120,3 +2120,140 @@ def block(
else:
cmd = StringCommand(f"awk '/{mark_1}/,/{mark_2}/ {{next}} 1'")
yield StringCommand(out_prep, cmd, q_path, "> $OUT", real_out)


_TAR_FORMATS = {
".tar": ["-x"],
".tar.gz": ["-xz"],
".tgz": ["-xz"],
".tar.bz2": ["-xj"],
".tbz2": ["-xj"],
".tar.xz": ["-xJ"],
".txz": ["-xJ"],
".tar.zst": ["-x", "--zstd"],
}
_ZIP_FORMATS = (".zip",)
_ARCHIVE_EXTENSIONS = tuple(_TAR_FORMATS.keys()) + _ZIP_FORMATS


def _get_archive_format(src: str) -> tuple[str, list[str]] | None:
lower = src.lower()
for ext, flags in _TAR_FORMATS.items():
if lower.endswith(ext):
return "tar", flags
for ext in _ZIP_FORMATS:
if lower.endswith(ext):
return "unzip", ["-o"]
return None


@operation()
def unarchive(
src: str,
dest: str,
remote_src: bool = False,
creates: str | None = None,
extra_opts: list[str] | None = None,
user: str | None = None,
group: str | None = None,
):
"""
Extract archive files on the remote system.

+ src: path to the archive file (local or remote depending on ``remote_src``)
+ dest: remote directory to extract into (must exist)
+ remote_src: set to ``True`` if the archive is already on the remote system
+ creates: if this path already exists, the operation is skipped (idempotency)
+ extra_opts: list of additional arguments to pass to the extract command
+ user: user to own the extracted files
+ group: group to own the extracted files

Supported formats:
``.tar``, ``.tar.gz``/``.tgz``, ``.tar.bz2``/``.tbz2``,
``.tar.xz``/``.txz``, ``.tar.zst``, ``.zip``

**Examples:**

.. code:: python

# Extract a remote archive
files.unarchive(
name="Extract app tarball",
src="/tmp/app.tar.gz",
dest="/opt/app",
remote_src=True,
)

# Upload and extract a local archive
files.unarchive(
name="Deploy release",
src="releases/app-v1.0.tar.gz",
dest="/opt/app",
creates="/opt/app/bin/start",
)
"""

# Idempotency: skip if creates path already exists
if creates:
if host.get_fact(File, path=creates) is not None:
host.noop("archive already extracted ({0} exists)".format(creates))
return

# Validate destination exists and is a directory
dest_info = host.get_fact(Directory, path=dest)
if not dest_info:
raise OperationError("Destination {0} is not an existing directory".format(dest))

archive_format = _get_archive_format(src)
if archive_format is None:
raise OperationValueError(
"Unsupported archive format for {0}. Supported: {1}".format(
src, ", ".join(_ARCHIVE_EXTENSIONS)
)
)

tool, flags = archive_format

if not remote_src:
# Upload the local archive to a temp location on the remote
temp_archive = host.get_temp_filename(src)
yield FileUploadCommand(src, temp_archive)
archive_path = temp_archive
else:
# Validate the remote archive exists
if host.get_fact(File, path=src) is None:
raise OperationError("Remote archive {0} does not exist".format(src))
archive_path = src

extras = list(extra_opts) if extra_opts else []

if tool == "tar":
# tar <flags> <extras> -f <archive> -C <dest>
# Keep -f adjacent to the archive path so extras never get mistaken for it.
yield StringCommand(
tool,
*flags,
*extras,
"-f",
QuoteString(archive_path),
"-C",
QuoteString(dest),
)
else:
# unzip <flags> <extras> <archive> -d <dest>
yield StringCommand(
tool,
*flags,
*extras,
QuoteString(archive_path),
"-d",
QuoteString(dest),
)

# Clean up uploaded temp file
if not remote_src:
yield StringCommand("rm", "-f", QuoteString(temp_archive))

# Set ownership if requested
if user or group:
yield file_utils.chown(dest, user, group, recursive=True)
16 changes: 16 additions & 0 deletions tests/operations/files.unarchive/dest_not_directory.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"kwargs": {
"src": "/tmp/app.tar.gz",
"dest": "/opt/missing",
"remote_src": true
},
"facts": {
"files.Directory": {
"path=/opt/missing": null
}
},
"exception": {
"name": "OperationError",
"message": "Destination /opt/missing is not an existing directory"
}
}
16 changes: 16 additions & 0 deletions tests/operations/files.unarchive/extract_local_tar_gz.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"kwargs": {
"src": "releases/app.tar.gz",
"dest": "/opt/app"
},
"facts": {
"files.Directory": {
"path=/opt/app": true
}
},
"commands": [
["upload", "releases/app.tar.gz", "_tempfile_"],
"tar -xz -f _tempfile_ -C /opt/app",
"rm -f _tempfile_"
]
}
18 changes: 18 additions & 0 deletions tests/operations/files.unarchive/extract_remote_tar_gz.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"kwargs": {
"src": "/tmp/app.tar.gz",
"dest": "/opt/app",
"remote_src": true
},
"facts": {
"files.Directory": {
"path=/opt/app": true
},
"files.File": {
"path=/tmp/app.tar.gz": true
}
},
"commands": [
"tar -xz -f /tmp/app.tar.gz -C /opt/app"
]
}
18 changes: 18 additions & 0 deletions tests/operations/files.unarchive/extract_remote_tar_zst.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"kwargs": {
"src": "/tmp/app.tar.zst",
"dest": "/opt/app",
"remote_src": true
},
"facts": {
"files.Directory": {
"path=/opt/app": true
},
"files.File": {
"path=/tmp/app.tar.zst": true
}
},
"commands": [
"tar -x --zstd -f /tmp/app.tar.zst -C /opt/app"
]
}
18 changes: 18 additions & 0 deletions tests/operations/files.unarchive/extract_remote_zip.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
{
"kwargs": {
"src": "/tmp/app.zip",
"dest": "/opt/app",
"remote_src": true
},
"facts": {
"files.Directory": {
"path=/opt/app": true
},
"files.File": {
"path=/tmp/app.zip": true
}
},
"commands": [
"unzip -o /tmp/app.zip -d /opt/app"
]
}
21 changes: 21 additions & 0 deletions tests/operations/files.unarchive/extract_with_chown.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
{
"kwargs": {
"src": "/tmp/app.tar.gz",
"dest": "/opt/app",
"remote_src": true,
"user": "www-data",
"group": "www-data"
},
"facts": {
"files.Directory": {
"path=/opt/app": true
},
"files.File": {
"path=/tmp/app.tar.gz": true
}
},
"commands": [
"tar -xz -f /tmp/app.tar.gz -C /opt/app",
"chown -R www-data:www-data /opt/app"
]
}
19 changes: 19 additions & 0 deletions tests/operations/files.unarchive/extract_with_extra_opts.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"kwargs": {
"src": "/tmp/app.tar.gz",
"dest": "/opt/app",
"remote_src": true,
"extra_opts": ["--strip-components=1"]
},
"facts": {
"files.Directory": {
"path=/opt/app": true
},
"files.File": {
"path=/tmp/app.tar.gz": true
}
},
"commands": [
"tar -xz --strip-components=1 -f /tmp/app.tar.gz -C /opt/app"
]
}
19 changes: 19 additions & 0 deletions tests/operations/files.unarchive/extract_zip_with_extra_opts.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"kwargs": {
"src": "/tmp/app.zip",
"dest": "/opt/app",
"remote_src": true,
"extra_opts": ["-q"]
},
"facts": {
"files.Directory": {
"path=/opt/app": true
},
"files.File": {
"path=/tmp/app.zip": true
}
},
"commands": [
"unzip -o -q /tmp/app.zip -d /opt/app"
]
}
19 changes: 19 additions & 0 deletions tests/operations/files.unarchive/remote_archive_missing.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"kwargs": {
"src": "/tmp/missing.tar.gz",
"dest": "/opt/app",
"remote_src": true
},
"facts": {
"files.Directory": {
"path=/opt/app": true
},
"files.File": {
"path=/tmp/missing.tar.gz": null
}
},
"exception": {
"name": "OperationError",
"message": "Remote archive /tmp/missing.tar.gz does not exist"
}
}
15 changes: 15 additions & 0 deletions tests/operations/files.unarchive/skip_creates_exists.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
"kwargs": {
"src": "/tmp/app.tar.gz",
"dest": "/opt/app",
"remote_src": true,
"creates": "/opt/app/bin/start"
},
"facts": {
"files.File": {
"path=/opt/app/bin/start": true
}
},
"commands": [],
"noop_description": "archive already extracted (/opt/app/bin/start exists)"
}
16 changes: 16 additions & 0 deletions tests/operations/files.unarchive/unsupported_format.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
{
"kwargs": {
"src": "/tmp/app.rar",
"dest": "/opt/app",
"remote_src": true
},
"facts": {
"files.Directory": {
"path=/opt/app": true
}
},
"exception": {
"name": "OperationValueError",
"message": "Unsupported archive format for /tmp/app.rar. Supported: .tar, .tar.gz, .tgz, .tar.bz2, .tbz2, .tar.xz, .txz, .tar.zst, .zip"
}
}
Loading