Stream tar extraction to disk and add file-based unpack (#167)

When extracting tar entries to disk, `hex_erl_tar` previously read each file entry fully into memory before writing it to disk. This change makes the disk extraction path stream file entries in chunks (default 64KB) directly to the output file.

- `hex_tarball:unpack/2,3` - Added `{file, Path}` as first argument to read tarballs from disk without loading into memory
- `hex_tarball:unpack/2,3` - Added `none` as output mode to extract only metadata and checksums, skipping contents
- `hex_tarball:unpack/2,3` - Refactored so `Output` drives the outer extraction strategy: `memory` extracts to memory, path/none extracts to a temp dir
- `hex_tarball:unpack_docs/2,3` - Added `{file, Path}` as first argument to read doc tarballs from disk without loading into memory
- `hex_erl_tar:extract/2` - Added `{chunks, N}` option to control chunk size for streaming extraction to disk

Author: ericmj

src/hex_erl_tar.erl

@@ -4,6 +4,10 @@
 %% 3. -doc and -moduledoc attributes removed for OTP 24 compatibility
 %% 4. safe_link_name/2 fixed to validate symlink targets relative to symlink's
 %%    parent directory instead of in isolation
+%% 5. When extracting to disk (cwd option), stream file entries in chunks
+%%    instead of loading them fully into memory
+%% 6. Default chunk_size to 65536 in add_opts instead of 0 with special case
+%% 7. Use compressed instead of compressed_one for file:open for OTP 24 compat
 %%
 %% OTP commit: 013041bd68c2547848e88963739edea7f0a1a90f
 %%
@@ -147,12 +151,18 @@ extract1(eof, Reader, _, Acc) ->
 extract1(#tar_header{name=Name,size=Size}=Header, Reader0, Opts, Acc0) ->
     case check_extract(Name, Opts) of
         true ->
-            case do_read(Reader0, Size) of
-                {ok, Bin, Reader1} ->
-                    Acc = extract2(Header, Bin, Opts, Acc0),
-                    {ok, Acc, Reader1};
-                {error, _} = Err ->
-                    throw(Err)
+            case Opts#read_opts.output of
+                memory ->
+                    case do_read(Reader0, Size) of
+                        {ok, Bin, Reader1} ->
+                            Acc = extract2(Header, Bin, Opts, Acc0),
+                            {ok, Acc, Reader1};
+                        {error, _} = Err ->
+                            throw(Err)
+                    end;
+                file ->
+                    Reader1 = extract_to_file(Header, Reader0, Opts),
+                    {ok, Acc0, Reader1}
             end;
         false ->
             {ok, Acc0, skip_file(Reader0)}
@@ -173,6 +183,79 @@ extract2(Header, Bin, Opts, Acc) ->
             throw(Err)
     end.
 
+extract_to_file(#tar_header{name=Name0}=Header, Reader0, Opts) ->
+    case typeflag(Header#tar_header.typeflag) of
+        regular ->
+            Name1 = make_safe_path(Name0, Opts),
+            case stream_to_file(Name1, Reader0, Opts) of
+                {ok, Reader1} ->
+                    read_verbose(Opts, "x ~ts~n", [Name0]),
+                    _ = set_extracted_file_info(Name1, Header),
+                    Reader1;
+                {error, _} = Err ->
+                    throw(Err)
+            end;
+        _ ->
+            Reader1 = skip_file(Reader0),
+            _ = write_extracted_element(Header, <<>>, Opts),
+            Reader1
+    end.
+
+stream_to_file(Name, Reader0, Opts) ->
+    Write =
+        case Opts#read_opts.keep_old_files of
+            true ->
+                case file:read_file_info(Name) of
+                    {ok, _} -> false;
+                    _ -> true
+                end;
+            false -> true
+        end,
+    case Write of
+        true ->
+            ChunkSize = Opts#read_opts.chunk_size,
+            case open_output_file(Name) of
+                {ok, Fd} ->
+                    try
+                        stream_to_file_loop(Fd, Reader0, ChunkSize)
+                    after
+                        file:close(Fd)
+                    end;
+                {error, _} = Err ->
+                    Err
+            end;
+        false ->
+            {ok, skip_file(Reader0)}
+    end.
+
+open_output_file(Name) ->
+    case file:open(Name, [write, raw, binary]) of
+        {ok, _} = Ok ->
+            Ok;
+        {error, enoent} ->
+            ok = make_dirs(Name, file),
+            file:open(Name, [write, raw, binary]);
+        {error, _} = Err ->
+            Err
+    end.
+
+stream_to_file_loop(_Fd, #reg_file_reader{num_bytes=0}=Reader, _ChunkSize) ->
+    {ok, Reader};
+stream_to_file_loop(_Fd, #sparse_file_reader{num_bytes=0}=Reader, _ChunkSize) ->
+    {ok, Reader};
+stream_to_file_loop(Fd, Reader, ChunkSize) ->
+    case do_read(Reader, ChunkSize) of
+        {ok, Bin, Reader1} ->
+            case file:write(Fd, Bin) of
+                ok ->
+                    stream_to_file_loop(Fd, Reader1, ChunkSize);
+                {error, _} = Err ->
+                    Err
+            end;
+        {error, _} = Err ->
+            Err
+    end.
+
 %% Checks if the file Name should be extracted.
 check_extract(_, #read_opts{files=all}) ->
     true;
@@ -382,13 +465,7 @@ open1({file, Fd}=Handle, read, [raw], Opts) ->
     end;
 open1({file, _Fd}=Handle, read, [], _Opts) ->
     {error, {Handle, {incompatible_option, cooked}}};
-open1(Name, Access, Raw, Opts0) when is_list(Name); is_binary(Name) ->
-    Opts = case lists:member(compressed, Opts0) andalso Access == read of
-               true ->
-                   [compressed_one | (Opts0 -- [compressed])];
-               false ->
-                   Opts0
-           end,
+open1(Name, Access, Raw, Opts) when is_list(Name); is_binary(Name) ->
     case file:open(Name, Raw ++ [binary, Access|Opts]) of
         {ok, File} ->
             {ok, #reader{handle=File,access=Access,func=fun file_op/2}};
@@ -1833,9 +1910,6 @@ do_write(#reader{handle=Handle,func=Fun}=Reader0, Data)
             Err
     end.
 
-do_copy(#reader{func=Fun}=Reader, Source, #add_opts{chunk_size=0}=Opts)
-  when is_function(Fun, 2) ->
-    do_copy(Reader, Source, Opts#add_opts{chunk_size=65536});
 do_copy(#reader{func=Fun}=Reader, Source, #add_opts{chunk_size=ChunkSize})
     when is_function(Fun, 2) ->
     case file:open(Source, [read, binary]) of
@@ -2009,6 +2083,8 @@ extract_opts([cooked|Rest], Opts=#read_opts{open_mode=OpenMode}) ->
     extract_opts(Rest, Opts#read_opts{open_mode=[cooked|OpenMode]});
 extract_opts([verbose|Rest], Opts) ->
     extract_opts(Rest, Opts#read_opts{verbose=true});
+extract_opts([{chunks,N}|Rest], Opts) ->
+    extract_opts(Rest, Opts#read_opts{chunk_size=N});
 extract_opts([Other|Rest], Opts) ->
     extract_opts(Rest, read_opts([Other], Opts));
 extract_opts([], Opts) ->

src/hex_erl_tar.hrl

@@ -1,4 +1,7 @@
-%% This file is a copy of erl_tar.hrl from OTP with no modifications.
+%% This file is a copy of erl_tar.hrl from OTP with the following modifications:
+%% 1. Added chunk_size field to #read_opts{} for streaming extraction to disk
+%% 2. Added {chunks, pos_integer()} to extract_opt() type
+%% 3. Default chunk_size to 65536 in #add_opts{} instead of 0
 %%
 %% OTP commit: 013041bd68c2547848e88963739edea7f0a1a90f
 %%
@@ -25,7 +28,7 @@
 %% Options used when adding files to a tar archive.
 -record(add_opts, {
 	 read_info,          %% Fun to use for read file/link info.
-	 chunk_size = 0,     %% For file reading when sending to sftp. 0=do not chunk
+	 chunk_size = 65536, %% Chunk size for reading files.
          verbose = false,    %% Verbose on/off.
          atime = undefined,
          mtime = undefined,
@@ -42,7 +45,8 @@
           files = all,                         %% Set of files to extract (or all)
           output = file :: 'file' | 'memory',
           open_mode = [],                      %% Open mode options.
-          verbose = false :: boolean()}).      %% Verbose on/off.
+          verbose = false :: boolean(),        %% Verbose on/off.
+          chunk_size = 65536}).                 %% Chunk size for streaming to disk.
 -type read_opts() :: #read_opts{}.
 
 -type add_opt() :: dereference |
@@ -59,6 +63,7 @@
 
 -type extract_opt() :: {cwd, string()} |
                        {files, [name_in_archive()]} |
+                       {chunks, pos_integer()} |
                        compressed |
                        cooked |
                        memory |