Adds zstd decompression for layer content blobs. (#508)

- Closes apple/container#988.
- macOS libarchive is not built with zstd support, so the workaround in
ArchiveReader is to attempt to decompress every archive as zstd. If
decompression fails, we pass the original archive to libarchive. If it
succeeds, we pass the uncompressed archive.
- Adds blob media type recognition for zstd to EXT4Unpacker. Tested zstd
blob unpack using `image pull tonistiigi/hello-world:zstd-docker`.

Co-authored-by: Aditya Ramani <a_ramani@apple.com>

Author: jglogan

Package.resolved

@@ -47,6 +47,7 @@ let package = Package(
         .package(url: "https://github.com/apple/swift-system.git", from: "1.4.0"),
         .package(url: "https://github.com/swiftlang/swift-docc-plugin", from: "1.1.0"),
         .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.36.0"),
+        .package(url: "https://github.com/facebook/zstd.git", exact: "1.5.7"),
     ],
     targets: [
         .target(
@@ -143,17 +144,26 @@ let package = Package(
             name: "ContainerizationArchiveTests",
             dependencies: [
                 "ContainerizationArchive"
+            ],
+            resources: [
+                .copy("Resources/test.tar.zst")
             ]
         ),
         .target(
             name: "CArchive",
-            dependencies: [],
+            dependencies: [
+                .product(name: "libzstd", package: "zstd")
+            ],
             path: "Sources/ContainerizationArchive/CArchive",
+            sources: [
+                "archive_swift_bridge.c"
+            ],
             cSettings: [
                 .define(
                     "PLATFORM_CONFIG_H", to: "\"config_darwin.h\"",
                     .when(platforms: [.iOS, .macOS, .macCatalyst, .watchOS, .driverKit, .tvOS])),
                 .define("PLATFORM_CONFIG_H", to: "\"config_linux.h\"", .when(platforms: [.linux])),
+                .unsafeFlags(["-fno-modules"]),
             ],
             linkerSettings: [
                 .linkedLibrary("z"),

Package.swift

@@ -94,6 +94,8 @@ public struct EXT4Unpacker: Unpacker {
                 compression = .none
             case MediaTypes.imageLayerGzip, MediaTypes.dockerImageLayerGzip:
                 compression = .gzip
+            case MediaTypes.imageLayerZstd, MediaTypes.dockerImageLayerZstd:
+                compression = .zstd
             default:
                 throw ContainerizationError(.unsupported, message: "media type \(layer.mediaType) not supported.")
             }

Sources/Containerization/Image/Unpacker/EXT4Unpacker.swift

@@ -15,9 +15,11 @@
 //===----------------------------------------------------------------------===//
 
 import CArchive
+import ContainerizationError
 import ContainerizationOS
 import Foundation
 import SystemPackage
+import libzstd
 
 /// A protocol for reading data in chunks, compatible with both `InputStream` and zero-allocation archive readers.
 public protocol ReadableStream {
@@ -53,13 +55,32 @@ public final class ArchiveReader {
     var underlying: OpaquePointer?
     /// The file handle associated with the archive file being read.
     let fileHandle: FileHandle?
+    /// Temporary decompressed file URL if the input was zstd-compressed
+    private var tempDecompressedFile: URL?
 
     /// Initializes an `ArchiveReader` to read from a specified file URL with an explicit `Format` and `Filter`.
     /// Note: This method must be used when it is known that the archive at the specified URL follows the specified
     /// `Format` and `Filter`.
     public convenience init(format: Format, filter: Filter, file: URL) throws {
-        let fileHandle = try FileHandle(forReadingFrom: file)
-        try self.init(format: format, filter: filter, fileHandle: fileHandle)
+        // If filter is zstd, decompress it and use filter .none
+        let fileToRead: URL
+        let tempFile: URL?
+        let actualFilter: Filter
+
+        if filter == .zstd {
+            let decompressed = try Self.decompressZstd(file)
+            tempFile = decompressed
+            fileToRead = decompressed
+            actualFilter = .none
+        } else {
+            tempFile = nil
+            fileToRead = file
+            actualFilter = filter
+        }
+
+        let fileHandle = try FileHandle(forReadingFrom: fileToRead)
+        try self.init(format: format, filter: actualFilter, fileHandle: fileHandle)
+        self.tempDecompressedFile = tempFile
     }
 
     /// Initializes an `ArchiveReader` to read from the provided file descriptor with an explicit `Format` and `Filter`.
@@ -82,8 +103,19 @@ public final class ArchiveReader {
     /// Initialize the `ArchiveReader` to read from a specified file URL
     /// by trying to auto determine the archives `Format` and `Filter`.
     public init(file: URL) throws {
+
         self.underlying = archive_read_new()
-        let fileHandle = try FileHandle(forReadingFrom: file)
+
+        // Try to decompress as zstd first, fall back to original if it fails
+        let fileToRead: URL
+        if let decompressed = try? Self.decompressZstd(file) {
+            self.tempDecompressedFile = decompressed
+            fileToRead = decompressed
+        } else {
+            fileToRead = file
+        }
+
+        let fileHandle = try FileHandle(forReadingFrom: fileToRead)
         self.fileHandle = fileHandle
         try archive_read_support_filter_all(underlying)
             .checkOk(elseThrow: .failedToDetectFilter)
@@ -94,9 +126,84 @@ public final class ArchiveReader {
             .checkOk(elseThrow: { .unableToOpenArchive($0) })
     }
 
+    /// Decompress a zstd file to a temporary location
+    private static func decompressZstd(_ source: URL) throws -> URL {
+        guard let inputStream = InputStream(url: source) else {
+            throw ArchiveError.noUnderlyingArchive
+        }
+        inputStream.open()
+        defer { inputStream.close() }
+
+        // Create temp file into which the source zstd archived is decompressed
+        guard let tempDir = createTemporaryDirectory(baseName: "zstd-decompress") else {
+            throw ArchiveError.failedToDetectFormat
+        }
+
+        let tempFile = tempDir.appendingPathComponent(
+            source.deletingPathExtension().lastPathComponent
+        )
+
+        guard let outputStream = OutputStream(url: tempFile, append: false) else {
+            throw ArchiveError.noUnderlyingArchive
+        }
+        outputStream.open()
+        defer { outputStream.close() }
+
+        // Use streaming decompression since content size may be unknown
+        guard let dstream = ZSTD_createDStream() else {
+            throw ArchiveError.failedToDetectFormat
+        }
+        defer { ZSTD_freeDStream(dstream) }
+
+        let initResult = ZSTD_initDStream(dstream)
+        guard ZSTD_isError(initResult) == 0 else {
+            throw ArchiveError.failedToDetectFormat
+        }
+
+        let inputBufferSize = ZSTD_DStreamInSize()
+        let outputBufferSize = ZSTD_DStreamOutSize()
+
+        var inputBuffer = [UInt8](repeating: 0, count: inputBufferSize)
+
+        while case let amount = inputStream.read(&inputBuffer, maxLength: inputBufferSize), amount > 0 {
+            try inputBuffer.withUnsafeBufferPointer { ptr in
+                var input = ZSTD_inBuffer(
+                    src: ptr.baseAddress,
+                    size: amount,
+                    pos: 0
+                )
+                while input.pos < input.size {
+                    var outputBuffer = [UInt8](repeating: 0, count: outputBufferSize)
+                    var decompressedBytes = 0
+                    try outputBuffer.withUnsafeMutableBytes { outputBytes in
+                        var output = ZSTD_outBuffer(
+                            dst: outputBytes.baseAddress,
+                            size: outputBufferSize,
+                            pos: 0
+                        )
+                        let result = ZSTD_decompressStream(dstream, &output, &input)
+                        guard ZSTD_isError(result) == 0 else {
+                            throw ArchiveError.failedToDetectFormat
+                        }
+                        decompressedBytes = output.pos
+                    }
+                    if decompressedBytes > 0 {
+                        outputStream.write(outputBuffer, maxLength: decompressedBytes)
+                    }
+                }
+            }
+        }
+        return tempFile
+    }
+
     deinit {
         archive_read_free(underlying)
         try? fileHandle?.close()
+
+        // Clean up temp decompressed file
+        if let tempFile = tempDecompressedFile {
+            try? FileManager.default.removeItem(at: tempFile.deletingLastPathComponent())
+        }
     }
 }
 

Sources/ContainerizationArchive/ArchiveReader.swift

@@ -174,6 +174,7 @@ public enum Filter: String, Sendable {
     case lzop
     case grzip
     case lz4
+    case zstd
 
     internal var code: CInt {
         switch self {
@@ -190,6 +191,7 @@ public enum Filter: String, Sendable {
         case .lzop: return ARCHIVE_FILTER_LZOP
         case .grzip: return ARCHIVE_FILTER_GRZIP
         case .lz4: return ARCHIVE_FILTER_LZ4
+        case .zstd: return ARCHIVE_FILTER_ZSTD
         }
     }
 }

Sources/ContainerizationArchive/ArchiveWriterConfiguration.swift

@@ -657,4 +657,58 @@ struct ArchiveReaderTests {
             _ = try reader.extractContents(to: extractDir)
         }
     }
+
+    // MARK: - Zstd Compression Tests
+
+    @Test func readZstdCompressedArchive() throws {
+        guard let resourceURL = Bundle.module.url(forResource: "test", withExtension: "tar.zst") else {
+            Issue.record("Test resource test.tar.zst not found")
+            return
+        }
+
+        let extractDir = try createExtractionDirectory(name: "zstd-test")
+        defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) }
+
+        // Test with explicit filter
+        let reader = try ArchiveReader(format: .paxRestricted, filter: .zstd, file: resourceURL)
+        let rejectedPaths = try reader.extractContents(to: extractDir)
+
+        #expect(rejectedPaths.isEmpty, "No paths should be rejected")
+
+        // Check extracted files
+        let testFile = extractDir.appendingPathComponent("test.txt")
+        let file2 = extractDir.appendingPathComponent("file2.txt")
+
+        #expect(FileManager.default.fileExists(atPath: testFile.path), "test.txt should exist")
+        #expect(FileManager.default.fileExists(atPath: file2.path), "file2.txt should exist")
+
+        let testContent = try String(contentsOf: testFile, encoding: .utf8)
+        #expect(testContent == "Hello from zstd compressed archive", "Content should match")
+
+        let file2Content = try String(contentsOf: file2, encoding: .utf8)
+        #expect(file2Content == "Another file", "Content should match")
+    }
+
+    @Test func readZstdCompressedArchiveAutoDetect() throws {
+        guard let resourceURL = Bundle.module.url(forResource: "test", withExtension: "tar.zst") else {
+            Issue.record("Test resource test.tar.zst not found")
+            return
+        }
+
+        let extractDir = try createExtractionDirectory(name: "zstd-auto-test")
+        defer { try? FileManager.default.removeItem(at: extractDir.deletingLastPathComponent()) }
+
+        // Test with auto-detect
+        let reader = try ArchiveReader(file: resourceURL)
+        let rejectedPaths = try reader.extractContents(to: extractDir)
+
+        #expect(rejectedPaths.isEmpty, "No paths should be rejected")
+
+        // Check extracted files
+        let testFile = extractDir.appendingPathComponent("test.txt")
+        #expect(FileManager.default.fileExists(atPath: testFile.path), "test.txt should exist")
+
+        let testContent = try String(contentsOf: testFile, encoding: .utf8)
+        #expect(testContent == "Hello from zstd compressed archive", "Content should match")
+    }
 }