Add --auto compression level

lib/compress/zstd_spectral_gap.c

@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * Spectral Gap Auto-Compression Level Detection.
+ *
+ * Method: compress incrementally at levels 1..22, compute fractional
+ * gain at each step. Stop when gain < epsilon for K consecutive levels.
+ * Epsilon calibrated from inter-level gains — transfers to all files.
+ */
+
+#include "zstd_spectral_gap.h"
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+#include "../zstd.h"
+
+/* M_PI not guaranteed on all C standards */
+#ifndef M_PI
+#define M_PI 3.14159265358979323846
+#endif
+
+/* Skip auto-detection for files larger than this (use default level) */
+#define AUTO_MAX_FILE_SIZE ((size_t)100 * 1024 * 1024)  /* 100 MB */
+
+static double calibrate_epsilon(const void* src, size_t srcSize) {
+    double epsilon;
+    size_t bound;
+    void* out;
+    double max_gain;
+    size_t prev_size;
+    int first;
+    int sample_levels[9];
+    int n_samples;
+    int i;
+
+    epsilon = M_PI / sqrt((double)SPECTRAL_GAP_MAX_LEVEL);
+    if (epsilon > 0.02) epsilon = 0.02;
+    if (epsilon < 0.0005) epsilon = 0.0005;
+
+    bound = ZSTD_compressBound(srcSize);
+    out = malloc(bound);
+    if (!out) return epsilon;
+
+    max_gain = 0.0;
+    prev_size = 0;
+    first = 1;
+    sample_levels[0] = 1; sample_levels[1] = 3; sample_levels[2] = 5;
+    sample_levels[3] = 7; sample_levels[4] = 9; sample_levels[5] = 11;
+    sample_levels[6] = 15; sample_levels[7] = 19; sample_levels[8] = 22;
+    n_samples = 9;
+
+    for (i = 0; i < n_samples; i++) {
+        size_t csize = ZSTD_compress(out, bound, src, srcSize,
+                                      sample_levels[i]);
+        if (ZSTD_isError(csize)) continue;
+
+        if (first) {
+            first = 0;
+        } else {
+            double gain = (double)((long long)prev_size - (long long)csize)
+                        / (double)prev_size;
+            if (gain > max_gain) {
+                max_gain = gain;
+                epsilon = max_gain * 0.15;
+                if (epsilon > 0.02) epsilon = 0.02;
+                if (epsilon < 0.0005) epsilon = 0.0005;
+            }
+        }
+        prev_size = csize;
+    }
+
+    free(out);
+    return epsilon;
+}
+
+int ZSTD_autoDetectLevel(const void* src, size_t srcSize,
+                          double* out_epsilon) {
+    double epsilon;
+    size_t bound;
+    void* out;
+    size_t prev_size;
+    int optimal_level;
+    int converged_streak;
+    int first;
+    int level;
+
+    if (!src || srcSize == 0) return 1;
+    /* For very large files, auto-detection would be too expensive.
+     * Fall back to default level — the spectral gap still applies
+     * at the block level inside ZSTD_compress. */
+    if (srcSize > AUTO_MAX_FILE_SIZE) return 3;
+
+    epsilon = calibrate_epsilon(src, srcSize);
+    if (out_epsilon) *out_epsilon = epsilon;
+
+    bound = ZSTD_compressBound(srcSize);
+    out = malloc(bound);
+    if (!out) return 3;
+
+    prev_size = 0;
+    optimal_level = 1;
+    converged_streak = 0;
+    first = 1;
+
+    for (level = 1; level <= SPECTRAL_GAP_MAX_LEVEL; level++) {
+        size_t csize = ZSTD_compress(out, bound, src, srcSize, level);
+        if (ZSTD_isError(csize)) break;
+
+        if (first) {
+            first = 0;
+        } else {
+            double gain = (double)((long long)prev_size - (long long)csize)
+                        / (double)prev_size;
+
+            if (gain >= epsilon) {
+                optimal_level = level;
+                converged_streak = 0;
+            } else {
+                converged_streak++;
+                if (converged_streak >= SPECTRAL_GAP_CONVERGE_K) {
+                    break;
+                }
+            }
+        }
+        prev_size = csize;
+    }
+
+    free(out);
+    return optimal_level;
+}

lib/compress/zstd_spectral_gap.h

@@ -0,0 +1,20 @@
+/*
+ * Copyright (c) Meta Platforms, Inc. and affiliates.
+ * All rights reserved.
+ *
+ * Spectral Gap Auto-Compression Level Detection.
+ * Detects optimal compression level by monitoring fractional gain convergence.
+ */
+#ifndef ZSTD_SPECTRAL_GAP_H
+#define ZSTD_SPECTRAL_GAP_H
+
+#include <stddef.h>
+
+#define SPECTRAL_GAP_MAX_LEVEL     22
+#define SPECTRAL_GAP_CONVERGE_K     3
+#define SPECTRAL_GAP_DEFAULT_EPSILON 0.005
+
+int ZSTD_autoDetectLevel(const void* src, size_t srcSize,
+                          double* out_epsilon);
+
+#endif /* ZSTD_SPECTRAL_GAP_H */

programs/zstdcli.c

@@ -29,6 +29,7 @@
 #  include "zstdcli_trace.h"
 #endif
 #include "../lib/zstd.h"  /* ZSTD_VERSION_STRING, ZSTD_minCLevel, ZSTD_maxCLevel */
+#include "../lib/compress/zstd_spectral_gap.h"  /* ZSTD_autoDetectLevel */
 #include "fileio_asyncio.h"
 #include "fileio_common.h"
 
@@ -225,6 +226,7 @@ static void usageAdvanced(const char* programName)
     DISPLAYOUT("\n");
     DISPLAYOUT("Advanced compression options:\n");
     DISPLAYOUT("  --ultra                       Enable levels beyond %i, up to %i; requires more memory.\n", ZSTDCLI_CLEVEL_MAX, ZSTD_maxCLevel());
+    DISPLAYOUT("  --auto                        Auto-detect optimal compression level (spectral gap analysis).\n");
     DISPLAYOUT("  --fast[=#]                    Use to very fast compression levels. [Default: %u]\n", 1);
 #ifdef ZSTD_GZCOMPRESS
     if (exeNameMatch(programName, ZSTD_GZ)) {     /* behave like gzip */
@@ -897,6 +899,7 @@ int main(int argCount, const char* argv[])
         removeSrcFile = 0,
         cLevel = init_cLevel(),
         ultra = 0,
+        autoMode = 0,
         cLevelLast = MINCLEVEL - 1, /* for benchmark range */
         setThreads_non1 = 0;
     unsigned nbWorkers = init_nbWorkers(NBWORKERS_UNSET);
@@ -1005,6 +1008,7 @@ int main(int argCount, const char* argv[])
                 if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; }
                 if (!strcmp(argument, "--stdout")) { forceStdout=1; outFileName=stdoutmark; continue; }
                 if (!strcmp(argument, "--ultra")) { ultra=1; continue; }
+                if (!strcmp(argument, "--auto")) { autoMode=1; continue; }
                 if (!strcmp(argument, "--check")) { FIO_setChecksumFlag(prefs, 2); continue; }
                 if (!strcmp(argument, "--no-check")) { FIO_setChecksumFlag(prefs, 0); continue; }
                 if (!strcmp(argument, "--sparse")) { FIO_setSparseWrite(prefs, 2); continue; }
@@ -1615,6 +1619,39 @@ int main(int argCount, const char* argv[])
     if (patchFromDictFileName != NULL)
         dictFileName = patchFromDictFileName;
     FIO_setMemLimit(prefs, memLimit);
+    /* Auto-detect optimal compression level using spectral gap analysis */
+    if (autoMode && operation==zom_compress && filenames->tableSize > 0) {
+        U64 fileSize;
+        FILE* f;
+        size_t srcSize;
+        void* src;
+        double epsilon;
+        int autoLevel;
+        DISPLAYLEVEL(2, "Auto-detecting optimal compression level...\n");
+        /* Use 64-bit file size to avoid ftell() overflow on large files */
+        fileSize = UTIL_getFileSize(filenames->fileNames[0]);
+        if (fileSize == UTIL_FILESIZE_UNKNOWN) {
+            DISPLAYLEVEL(2, "Cannot determine file size, using default level %d\n", cLevel);
+        } else if (fileSize > (U64)100 * 1024 * 1024) {
+            DISPLAYLEVEL(2, "File too large for auto-detection, using default level %d\n", cLevel);
+        } else {
+            /* Read source file into memory for analysis */
+            f = fopen(filenames->fileNames[0], "rb");
+            if (f) {
+                srcSize = (size_t)fileSize;
+                src = malloc(srcSize);
+                if (src && fread(src, 1, srcSize, f) == srcSize) {
+                    epsilon = 0;
+                    autoLevel = ZSTD_autoDetectLevel(src, srcSize, &epsilon);
+                    cLevel = autoLevel;
+                    DISPLAYLEVEL(2, "Auto-detected level %d (epsilon=%.4f) for %s (%zu bytes)\n",
+                                autoLevel, epsilon, filenames->fileNames[0], srcSize);
+                }
+                free(src);
+                fclose(f);
+            }
+        }
+    }
     if (operation==zom_compress) {
 #ifndef ZSTD_NOCOMPRESS
         FIO_setCompressionType(prefs, cType);