Skip to content

Commit f63a6d1

Browse files
CopilotbytemainCopilot
authored
feat: Add zst support to archiver library (#663)
* Add zst archive decompression support Agent-Logs-Url: https://github.com/version-fox/vfox/sessions/abcce404-2e68-4df1-aad4-2d6d0295b4f0 Co-authored-by: bytemain <13938334+bytemain@users.noreply.github.com> * Secure zst archive extraction Agent-Logs-Url: https://github.com/version-fox/vfox/sessions/abcce404-2e68-4df1-aad4-2d6d0295b4f0 Co-authored-by: bytemain <13938334+bytemain@users.noreply.github.com> * Tighten zst archive target normalization Agent-Logs-Url: https://github.com/version-fox/vfox/sessions/abcce404-2e68-4df1-aad4-2d6d0295b4f0 Co-authored-by: bytemain <13938334+bytemain@users.noreply.github.com> * Make zst root extraction explicit Agent-Logs-Url: https://github.com/version-fox/vfox/sessions/abcce404-2e68-4df1-aad4-2d6d0295b4f0 Co-authored-by: bytemain <13938334+bytemain@users.noreply.github.com> * Handle zst extraction file close errors Agent-Logs-Url: https://github.com/version-fox/vfox/sessions/abcce404-2e68-4df1-aad4-2d6d0295b4f0 Co-authored-by: bytemain <13938334+bytemain@users.noreply.github.com> * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Update archiver documentation with usage example --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: bytemain <13938334+bytemain@users.noreply.github.com> Co-authored-by: Jiacheng <jiacheng.li@bytedance.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
1 parent 8182390 commit f63a6d1

5 files changed

Lines changed: 279 additions & 6 deletions

File tree

docs/plugins/library/archiver.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
11
# Archiver Library
22

3-
`vfox` provides a decompression tool that supports `tar.gz`, `tgz`, `tar.xz`, `zip`, and `7z`. In Lua scripts, you can
3+
`vfox` provides a decompression tool that supports `tar.gz`, `tgz`, `tar.xz`, `tar.zst`, `tzst`, `zip`, and `7z`. In Lua scripts, you can
44
use `require("vfox.archiver")` to access it.
55

66
**Usage**
77

88
```lua
99
local archiver = require("vfox.archiver")
1010
local err = archiver.decompress("testdata/test.zip", "testdata/test")
11-
```
11+
```
Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,11 @@
11
# Archiver 标准库
22

3-
`vfox` 提供了解压工具, 支持`tar.gz``tgz``tar.xz``zip``7z`。在Lua脚本中,你可以使用`require("vfox.archiver")`来访问它。
3+
`vfox` 提供了解压工具, 支持`tar.gz``tgz``tar.xz``tar.zst``tzst``zip``7z`。在Lua脚本中,你可以使用`require("vfox.archiver")`来访问它。
44
例如:
55

66
**Usage**
7-
```shell
7+
8+
```lua
89
local archiver = require("vfox.archiver")
910
local err = archiver.decompress("testdata/test.zip", "testdata/test")
10-
```
11+
```

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ require (
99
github.com/Microsoft/go-winio v0.6.2
1010
github.com/PuerkitoBio/goquery v1.9.3
1111
github.com/bodgit/sevenzip v1.5.1
12+
github.com/klauspost/compress v1.17.7
1213
github.com/lithammer/fuzzysearch v1.1.8
1314
github.com/pterm/pterm v0.12.79
1415
github.com/schollz/progressbar/v3 v3.14.2
@@ -36,7 +37,6 @@ require (
3637
github.com/hashicorp/errwrap v1.0.0 // indirect
3738
github.com/hashicorp/go-multierror v1.1.1 // indirect
3839
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
39-
github.com/klauspost/compress v1.17.7 // indirect
4040
github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
4141
github.com/mattn/go-runewidth v0.0.15 // indirect
4242
github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db // indirect

internal/shared/util/decompressor.go

Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929
"strings"
3030

3131
"github.com/bodgit/sevenzip"
32+
"github.com/klauspost/compress/zstd"
3233
"github.com/ulikunitz/xz"
3334
)
3435

@@ -262,6 +263,159 @@ loop:
262263
return nil
263264
}
264265

266+
type ZstdTarDecompressor struct {
267+
src string
268+
}
269+
270+
func (z *ZstdTarDecompressor) Decompress(dest string) error {
271+
rootFolderInTar := findRootFolderInZstdTar(z.src)
272+
file, err := os.Open(z.src)
273+
if err != nil {
274+
return err
275+
}
276+
defer file.Close()
277+
278+
zr, err := zstd.NewReader(file)
279+
if err != nil {
280+
return err
281+
}
282+
defer zr.Close()
283+
284+
tr := tar.NewReader(zr)
285+
var symlinks []symlink
286+
287+
loop:
288+
for {
289+
header, err := tr.Next()
290+
switch {
291+
case err == io.EOF:
292+
break loop
293+
case err != nil:
294+
return err
295+
case header == nil:
296+
continue
297+
}
298+
299+
target, err := safeZstdTarTarget(dest, header.Name, rootFolderInTar)
300+
if err != nil {
301+
return err
302+
}
303+
304+
switch header.Typeflag {
305+
case tar.TypeDir:
306+
if _, err := os.Stat(target); err != nil {
307+
if err := os.MkdirAll(target, 0755); err != nil {
308+
return err
309+
}
310+
}
311+
case tar.TypeReg:
312+
if err := os.MkdirAll(filepath.Dir(target), 0755); err != nil {
313+
return err
314+
}
315+
f, err := os.OpenFile(target, os.O_CREATE|os.O_RDWR, os.FileMode(header.Mode))
316+
if err != nil {
317+
return err
318+
}
319+
if _, err := io.Copy(f, tr); err != nil {
320+
_ = f.Close()
321+
return err
322+
}
323+
if err := f.Close(); err != nil {
324+
return err
325+
}
326+
case tar.TypeSymlink:
327+
symlinks = append(symlinks, symlink{header.Linkname, target})
328+
}
329+
}
330+
331+
for _, s := range symlinks {
332+
dir := filepath.Dir(s.newname)
333+
if _, err := os.Stat(dir); os.IsNotExist(err) {
334+
if err := os.MkdirAll(dir, 0755); err != nil {
335+
return err
336+
}
337+
}
338+
if err = os.Symlink(s.oldname, s.newname); err != nil {
339+
return err
340+
}
341+
}
342+
return nil
343+
}
344+
345+
func findRootFolderInZstdTar(tarFilePath string) string {
346+
file, err := os.Open(tarFilePath)
347+
if err != nil {
348+
return ""
349+
}
350+
defer file.Close()
351+
352+
zr, err := zstd.NewReader(file)
353+
if err != nil {
354+
return ""
355+
}
356+
defer zr.Close()
357+
358+
tr := tar.NewReader(zr)
359+
var firstElement string
360+
361+
for {
362+
header, err := tr.Next()
363+
if err == io.EOF {
364+
break
365+
}
366+
if err != nil || header == nil {
367+
return ""
368+
}
369+
370+
normalizedPath := strings.Trim(strings.ReplaceAll(header.Name, "\\", "/"), "/")
371+
if normalizedPath == "" || strings.HasPrefix(normalizedPath, ".DS_Store") || strings.HasPrefix(normalizedPath, "__MACOSX") {
372+
continue
373+
}
374+
375+
currentFirstElement := strings.Split(normalizedPath, "/")[0]
376+
if firstElement != "" && firstElement != currentFirstElement {
377+
return ""
378+
}
379+
if firstElement == "" {
380+
firstElement = currentFirstElement
381+
}
382+
}
383+
return firstElement
384+
}
385+
386+
func safeZstdTarTarget(dest string, name string, rootFolderInTar string) (string, error) {
387+
normalizedPath := strings.ReplaceAll(name, "\\", "/")
388+
if strings.HasPrefix(normalizedPath, "/") {
389+
return "", fmt.Errorf("archive entry %q is outside destination", name)
390+
}
391+
normalizedPath = strings.Trim(normalizedPath, "/")
392+
if normalizedPath == "" {
393+
return "", fmt.Errorf("archive entry %q is empty", name)
394+
}
395+
396+
parts := strings.Split(normalizedPath, "/")
397+
if len(parts) > 1 && rootFolderInTar != "" && parts[0] == rootFolderInTar {
398+
parts = parts[1:]
399+
}
400+
fname := filepath.Clean(strings.Join(parts, "/"))
401+
if fname == "." {
402+
return "", fmt.Errorf("archive entry %q is empty", name)
403+
}
404+
if !filepath.IsLocal(fname) {
405+
return "", fmt.Errorf("archive entry %q is outside destination", name)
406+
}
407+
408+
target := filepath.Join(dest, fname)
409+
rel, err := filepath.Rel(dest, target)
410+
if err != nil {
411+
return "", err
412+
}
413+
if rel == ".." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) {
414+
return "", fmt.Errorf("archive entry %q is outside destination", name)
415+
}
416+
return target, nil
417+
}
418+
265419
type ZipDecompressor struct {
266420
src string
267421
}
@@ -525,6 +679,11 @@ func NewDecompressor(src string) Decompressor {
525679
src: src,
526680
}
527681
}
682+
if strings.HasSuffix(filename, ".tar.zst") || strings.HasSuffix(filename, ".tzst") {
683+
return &ZstdTarDecompressor{
684+
src: src,
685+
}
686+
}
528687
if strings.HasSuffix(filename, ".zip") {
529688
return &ZipDecompressor{
530689
src: src,

internal/shared/util/decompressor_test.go

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,13 @@
1717
package util
1818

1919
import (
20+
"archive/tar"
21+
"os"
22+
"path/filepath"
23+
"strings"
2024
"testing"
25+
26+
"github.com/klauspost/compress/zstd"
2127
)
2228

2329
func TestNewDecompressor(t *testing.T) {
@@ -31,12 +37,119 @@ func TestNewDecompressor(t *testing.T) {
3137
t.Errorf("Expected ZipDecompressor, got %T", zipDecompressor)
3238
}
3339

40+
zstdTarDecompressor := NewDecompressor("test.tar.zst")
41+
if _, ok := zstdTarDecompressor.(*ZstdTarDecompressor); !ok {
42+
t.Errorf("Expected ZstdTarDecompressor, got %T", zstdTarDecompressor)
43+
}
44+
45+
tzstDecompressor := NewDecompressor("test.tzst")
46+
if _, ok := tzstDecompressor.(*ZstdTarDecompressor); !ok {
47+
t.Errorf("Expected ZstdTarDecompressor, got %T", tzstDecompressor)
48+
}
49+
3450
unknownDecompressor := NewDecompressor("test.unknown")
3551
if unknownDecompressor != nil {
3652
t.Errorf("Expected nil, got %T", unknownDecompressor)
3753
}
3854
}
3955

56+
func TestZstdTarDecompressor(t *testing.T) {
57+
tempDir := t.TempDir()
58+
archivePath := filepath.Join(tempDir, "test.tar.zst")
59+
dest := filepath.Join(tempDir, "dest")
60+
body := "Hello, zstd!"
61+
62+
writeZstdTar(t, archivePath, "test.txt", body)
63+
64+
decompressor := NewDecompressor(archivePath)
65+
if err := decompressor.Decompress(dest); err != nil {
66+
t.Fatalf("Failed to decompress: %v", err)
67+
}
68+
69+
decompressedFile, err := os.ReadFile(filepath.Join(dest, "test.txt"))
70+
if err != nil {
71+
t.Fatal(err)
72+
}
73+
if strings.TrimSpace(string(decompressedFile)) != body {
74+
t.Errorf("Expected %q, got %q", body, string(decompressedFile))
75+
}
76+
}
77+
78+
func TestZstdTarDecompressorStripsCommonRootFolder(t *testing.T) {
79+
tempDir := t.TempDir()
80+
archivePath := filepath.Join(tempDir, "test.tar.zst")
81+
dest := filepath.Join(tempDir, "dest")
82+
body := "Hello from root!"
83+
84+
writeZstdTar(t, archivePath, "root/test.txt", body)
85+
86+
decompressor := NewDecompressor(archivePath)
87+
if err := decompressor.Decompress(dest); err != nil {
88+
t.Fatalf("Failed to decompress: %v", err)
89+
}
90+
91+
decompressedFile, err := os.ReadFile(filepath.Join(dest, "test.txt"))
92+
if err != nil {
93+
t.Fatal(err)
94+
}
95+
if strings.TrimSpace(string(decompressedFile)) != body {
96+
t.Errorf("Expected %q, got %q", body, string(decompressedFile))
97+
}
98+
}
99+
100+
func TestZstdTarDecompressorRejectsPathTraversal(t *testing.T) {
101+
tempDir := t.TempDir()
102+
archivePath := filepath.Join(tempDir, "test.tar.zst")
103+
dest := filepath.Join(tempDir, "dest")
104+
105+
writeZstdTar(t, archivePath, "root/../../evil.txt", "evil")
106+
107+
decompressor := NewDecompressor(archivePath)
108+
if err := decompressor.Decompress(dest); err == nil {
109+
t.Fatal("Expected path traversal archive entry to fail")
110+
}
111+
if _, err := os.Stat(filepath.Join(tempDir, "evil.txt")); !os.IsNotExist(err) {
112+
t.Fatalf("Expected no file outside destination, got err %v", err)
113+
}
114+
}
115+
116+
func writeZstdTar(t *testing.T, archivePath string, name string, body string) {
117+
t.Helper()
118+
119+
file, err := os.Create(archivePath)
120+
if err != nil {
121+
t.Fatal(err)
122+
}
123+
124+
zw, err := zstd.NewWriter(file)
125+
if err != nil {
126+
t.Fatal(err)
127+
}
128+
tw := tar.NewWriter(zw)
129+
130+
err = tw.WriteHeader(&tar.Header{
131+
Name: name,
132+
Mode: 0600,
133+
Size: int64(len(body)),
134+
Typeflag: tar.TypeReg,
135+
})
136+
if err != nil {
137+
t.Fatal(err)
138+
}
139+
if _, err := tw.Write([]byte(body)); err != nil {
140+
t.Fatal(err)
141+
}
142+
if err := tw.Close(); err != nil {
143+
t.Fatal(err)
144+
}
145+
if err := zw.Close(); err != nil {
146+
t.Fatal(err)
147+
}
148+
if err := file.Close(); err != nil {
149+
t.Fatal(err)
150+
}
151+
}
152+
40153
//func TestDecompress(t *testing.T) {
41154
// // Create a temporary directory for testing
42155
// tempDir, err := os.MkdirTemp("", "decompress_test")

0 commit comments

Comments
 (0)