Skip to content

Commit ee0af93

Browse files
Cyan4973Cyan4973
authored
Merge pull request #4609 from Integral-Tech/memory-g-suffix
Accept G, GB and GiB suffixes for options
2 parents d7ee320 + 488b3a7 commit ee0af93

5 files changed

Lines changed: 94 additions & 30 deletions

File tree

‎programs/zstdcli.c‎

Lines changed: 37 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ static void errorOut(const char* msg)
347347

348348
/*! readU32FromCharChecked() :
349349
* @return 0 if success, and store the result in *value.
350-
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
350+
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
351351
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
352352
* @return 1 if an overflow error occurs */
353353
static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
@@ -362,15 +362,22 @@ static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
362362
if (result < last) return 1; /* overflow error */
363363
(*stringPtr)++ ;
364364
}
365-
if ((**stringPtr=='K') || (**stringPtr=='M')) {
366-
unsigned const maxK = ((unsigned)(-1)) >> 10;
367-
if (result > maxK) return 1; /* overflow error */
368-
result <<= 10;
369-
if (**stringPtr=='M') {
370-
if (result > maxK) return 1; /* overflow error */
371-
result <<= 10;
365+
if ((**stringPtr=='K') || (**stringPtr=='M') || (**stringPtr=='G')) {
366+
switch (**stringPtr) {
367+
case 'K':
368+
if (result > (((unsigned)-1) >> 10)) return 1; /* overflow error */
369+
result <<= 10;
370+
break;
371+
case 'M':
372+
if (result > (((unsigned)-1) >> 20)) return 1; /* overflow error */
373+
result <<= 20;
374+
break;
375+
case 'G':
376+
if (result > (((unsigned)-1) >> 30)) return 1; /* overflow error */
377+
result <<= 30;
378+
break;
372379
}
373-
(*stringPtr)++; /* skip `K` or `M` */
380+
(*stringPtr)++; /* skip `K`, `M` or `G` */
374381
if (**stringPtr=='i') (*stringPtr)++;
375382
if (**stringPtr=='B') (*stringPtr)++;
376383
}
@@ -380,7 +387,7 @@ static int readU32FromCharChecked(const char** stringPtr, unsigned* value)
380387

381388
/*! readU32FromChar() :
382389
* @return : unsigned integer value read from input in `char` format.
383-
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
390+
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
384391
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
385392
* Note : function will exit() program if digit sequence overflows */
386393
static unsigned readU32FromChar(const char** stringPtr) {
@@ -392,7 +399,7 @@ static unsigned readU32FromChar(const char** stringPtr) {
392399

393400
/*! readIntFromChar() :
394401
* @return : signed integer value read from input in `char` format.
395-
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
402+
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
396403
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
397404
* Note : function will exit() program if digit sequence overflows */
398405
static int readIntFromChar(const char** stringPtr) {
@@ -409,7 +416,7 @@ static int readIntFromChar(const char** stringPtr) {
409416

410417
/*! readSizeTFromCharChecked() :
411418
* @return 0 if success, and store the result in *value.
412-
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
419+
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
413420
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
414421
* @return 1 if an overflow error occurs */
415422
static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
@@ -424,15 +431,22 @@ static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
424431
if (result < last) return 1; /* overflow error */
425432
(*stringPtr)++ ;
426433
}
427-
if ((**stringPtr=='K') || (**stringPtr=='M')) {
428-
size_t const maxK = ((size_t)(-1)) >> 10;
429-
if (result > maxK) return 1; /* overflow error */
430-
result <<= 10;
431-
if (**stringPtr=='M') {
432-
if (result > maxK) return 1; /* overflow error */
433-
result <<= 10;
434+
if ((**stringPtr=='K') || (**stringPtr=='M') || (**stringPtr=='G')) {
435+
switch (**stringPtr) {
436+
case 'K':
437+
if (result > (((size_t)-1) >> 10)) return 1; /* overflow error */
438+
result <<= 10;
439+
break;
440+
case 'M':
441+
if (result > (((size_t)-1) >> 20)) return 1; /* overflow error */
442+
result <<= 20;
443+
break;
444+
case 'G':
445+
if (result > (((size_t)-1) >> 30)) return 1; /* overflow error */
446+
result <<= 30;
447+
break;
434448
}
435-
(*stringPtr)++; /* skip `K` or `M` */
449+
(*stringPtr)++; /* skip `K`, `M` or `G` */
436450
if (**stringPtr=='i') (*stringPtr)++;
437451
if (**stringPtr=='B') (*stringPtr)++;
438452
}
@@ -442,7 +456,7 @@ static int readSizeTFromCharChecked(const char** stringPtr, size_t* value)
442456

443457
/*! readSizeTFromChar() :
444458
* @return : size_t value read from input in `char` format.
445-
* allows and interprets K, KB, KiB, M, MB and MiB suffix.
459+
* allows and interprets K, KB, KiB, M, MB, MiB, G, GB and GiB suffix.
446460
* Will also modify `*stringPtr`, advancing it to position where it stopped reading.
447461
* Note : function will exit() program if digit sequence overflows */
448462
static size_t readSizeTFromChar(const char** stringPtr) {
@@ -830,7 +844,7 @@ static unsigned init_nbWorkers(unsigned defaultNbWorkers) {
830844
NEXT_FIELD(__nb); \
831845
_varu32 = readU32FromChar(&__nb); \
832846
if(*__nb != 0) { \
833-
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
847+
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed"); \
834848
} \
835849
}
836850

@@ -839,7 +853,7 @@ static unsigned init_nbWorkers(unsigned defaultNbWorkers) {
839853
NEXT_FIELD(__nb); \
840854
_varTsize = readSizeTFromChar(&__nb); \
841855
if(*__nb != 0) { \
842-
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed"); \
856+
errorOut("error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed"); \
843857
} \
844858
}
845859

‎tests/cli-tests/basic/memlimit.sh‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,30 @@ rm file.zst
3535
println "+ zstd --memory=1MiB file"
3636
zstd -q --memory=1MiB file && die "Should allow numeric parameter with expected suffix"
3737
rm file.zst
38+
println "+ zstd --memory=1G file"
39+
zstd -q --memory=1G file && die "Should allow numeric parameter with expected suffix"
40+
rm file.zst
41+
println "+ zstd --memory=1GB file"
42+
zstd -q --memory=1GB file && die "Should allow numeric parameter with expected suffix"
43+
rm file.zst
44+
println "+ zstd --memory=1GiB file"
45+
zstd -q --memory=1GiB file && die "Should allow numeric parameter with expected suffix"
46+
rm file.zst
47+
println "+ zstd --memory=3G file"
48+
zstd -q --memory=3G file && die "Should allow numeric parameter with expected suffix"
49+
rm file.zst
50+
println "+ zstd --memory=3GB file"
51+
zstd -q --memory=3GB file && die "Should allow numeric parameter with expected suffix"
52+
rm file.zst
53+
println "+ zstd --memory=3GiB file"
54+
zstd -q --memory=3GiB file && die "Should allow numeric parameter with expected suffix"
55+
rm file.zst
56+
println "+ zstd --memory=4G file"
57+
zstd --memory=4G file && die "Should not allow out-of-bound numeric parameter"
58+
println "+ zstd --memory=4GB file"
59+
zstd --memory=4GB file && die "Should not allow out-of-bound numeric parameter"
60+
println "+ zstd --memory=4GiB file"
61+
zstd --memory=4GiB file && die "Should not allow out-of-bound numeric parameter"
3862

3963
rm file
4064
exit 0

‎tests/cli-tests/basic/memlimit.sh.stderr.exact‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,22 @@
1-
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
2-
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
3-
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
4-
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
5-
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
6-
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB are allowed
1+
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
2+
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
3+
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
4+
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
5+
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
6+
error: only numeric values with optional suffixes K, KB, KiB, M, MB, MiB, G, GB, GiB are allowed
77
Should allow numeric parameter without suffix
88
Should allow numeric parameter with expected suffix
99
Should allow numeric parameter with expected suffix
1010
Should allow numeric parameter with expected suffix
1111
Should allow numeric parameter with expected suffix
1212
Should allow numeric parameter with expected suffix
1313
Should allow numeric parameter with expected suffix
14+
Should allow numeric parameter with expected suffix
15+
Should allow numeric parameter with expected suffix
16+
Should allow numeric parameter with expected suffix
17+
Should allow numeric parameter with expected suffix
18+
Should allow numeric parameter with expected suffix
19+
Should allow numeric parameter with expected suffix
20+
error: numeric value overflows 32-bit unsigned int
21+
error: numeric value overflows 32-bit unsigned int
22+
error: numeric value overflows 32-bit unsigned int

‎tests/cli-tests/basic/memlimit.sh.stdout.exact‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,12 @@
1111
+ zstd --memory=1M file
1212
+ zstd --memory=1MB file
1313
+ zstd --memory=1MiB file
14+
+ zstd --memory=1G file
15+
+ zstd --memory=1GB file
16+
+ zstd --memory=1GiB file
17+
+ zstd --memory=3G file
18+
+ zstd --memory=3GB file
19+
+ zstd --memory=3GiB file
20+
+ zstd --memory=4G file
21+
+ zstd --memory=4GB file
22+
+ zstd --memory=4GiB file

‎tests/playTests.sh‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1075,13 +1075,19 @@ cat tmp | zstd -14 -f --size-hint=11050 | zstd -t # slightly too high
10751075
cat tmp | zstd -14 -f --size-hint=10950 | zstd -t # slightly too low
10761076
cat tmp | zstd -14 -f --size-hint=22000 | zstd -t # considerably too high
10771077
cat tmp | zstd -14 -f --size-hint=5500 | zstd -t # considerably too low
1078-
println "test : allows and interprets K,KB,KiB,M,MB and MiB suffix"
1078+
println "test : allows and interprets K,KB,KiB,M,MB,MiB,G,GB and GiB suffix"
10791079
cat tmp | zstd -14 -f --size-hint=11K | zstd -t
10801080
cat tmp | zstd -14 -f --size-hint=11KB | zstd -t
10811081
cat tmp | zstd -14 -f --size-hint=11KiB | zstd -t
10821082
cat tmp | zstd -14 -f --size-hint=1M | zstd -t
10831083
cat tmp | zstd -14 -f --size-hint=1MB | zstd -t
10841084
cat tmp | zstd -14 -f --size-hint=1MiB | zstd -t
1085+
cat tmp | zstd -14 -f --size-hint=1G | zstd -t
1086+
cat tmp | zstd -14 -f --size-hint=1GB | zstd -t
1087+
cat tmp | zstd -14 -f --size-hint=1GiB | zstd -t
1088+
cat tmp | zstd -14 -f --size-hint=3G | zstd -t
1089+
cat tmp | zstd -14 -f --size-hint=3GB | zstd -t
1090+
cat tmp | zstd -14 -f --size-hint=3GiB | zstd -t
10851091

10861092

10871093
println "\n===> dictionary tests "
@@ -1684,6 +1690,8 @@ roundTripTest -g1M -P50 "1 --single-thread --long=29" " --memory=512MB"
16841690
roundTripTest -g1M -P50 "1 --single-thread --long=29 --zstd=wlog=28" " --memory=256MB"
16851691
roundTripTest -g1M -P50 "1 --single-thread --long=29" " --long=28 --memory=512MB"
16861692
roundTripTest -g1M -P50 "1 --single-thread --long=29" " --zstd=wlog=28 --memory=512MB"
1693+
roundTripTest -g1M -P50 "1 --single-thread --long=30" " --memory=1GB"
1694+
roundTripTest -g1M -P50 "1 --single-thread --long=30" " --zstd=wlog=29 --memory=1GB"
16871695

16881696

16891697
if [ "$ZSTD_LIB_EXCLUDE_COMPRESSORS_DFAST_AND_UP" -ne "1" ]; then

0 commit comments

Comments
 (0)