setup: bundle and reference additional private zstd headers

This restores the prior behavior in 0.14. I'm not a fan of the behavior.
But it does unblock building/linking against the system libzstd.

Closes #129. But the solution is not ideal. CC #106.

Author: indygreg

c-ext/compressor.c

@@ -8,6 +8,7 @@
 
 #include "python-zstandard.h"
 
+/* TODO pool.h is a private header and we shouldn't rely on it. */
 #ifndef ZSTD_SINGLE_FILE
 #include "pool.h"
 #endif

c-ext/decompressor.c

@@ -7,6 +7,8 @@
  */
 
 #include "python-zstandard.h"
+
+/* TODO pool.h is a private header and we shouldn't rely on it. */
 #ifndef ZSTD_SINGLE_FILE
 #include "pool.h"
 #endif

docs/news.rst

@@ -65,6 +65,7 @@ Actions Blocking Release
 * Support ``ZSTD_threadPool`` APIs for managing a thread pool.
 * Utilize ``ZSTD_getDictID_fromCDict()``?
 * Utilize ``ZSTD_DCtx_getParameter()``.
+* Stop relying on private libzstd headers and symbols (namely ``pool.h``).
 
 Other Actions Not Blocking Release
 ---------------------------------------
@@ -84,6 +85,10 @@ Bug Fixes
   the case in releases prior to 0.15.0 and the include order was reversed
   as part of running ``clang-format``. The old/working order has been
   restored. (#128)
+* Include some private zstd C headers so we can build the C extension against
+  a system library. The previous behavior of referencing these headers is
+  restored. That behave is rather questionable and undermines the desire to
+  use the system zstd.
 
 0.15.0 (released 2020-12-29)
 ============================

setup_zstd.py

@@ -22,6 +22,12 @@
     "c-ext/backend_c.c",
 ]
 
+zstd_includes = [
+    "zstd",
+    "zstd/common",
+    "zstd/dictBuilder",
+]
+
 
 def get_c_extension(
     support_legacy=False,
@@ -52,11 +58,22 @@ def get_c_extension(
 
     sources = sorted(set([os.path.join(actual_root, p) for p in ext_sources]))
     local_include_dirs = [os.path.join(actual_root, d) for d in ext_includes]
-    depends = []
 
-    if not system_zstd:
+    if system_zstd:
+        # TODO remove this once pool.h dependency goes away.
+        #
+        # This effectively causes system zstd mode to pull in our
+        # local headers instead of the system's. Then we link with the
+        # system library. This is super sketchy and could result in link
+        # time errors due to symbol mismatch or even run-time errors if
+        # APIs behave differently.
+        local_include_dirs.extend(
+            [os.path.join(actual_root, d) for d in zstd_includes]
+        )
+    else:
         local_include_dirs.append(os.path.join(actual_root, "zstd"))
-        depends = sorted(glob.glob(os.path.join(actual_root, "c-ext", "*")))
+
+    depends = sorted(glob.glob(os.path.join(actual_root, "c-ext", "*")))
 
     compiler = distutils.ccompiler.new_compiler()
 

zstd/common/pool.h

@@ -0,0 +1,84 @@
+/*
+ * Copyright (c) 2016-2020, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#ifndef POOL_H
+#define POOL_H
+
+#if defined (__cplusplus)
+extern "C" {
+#endif
+
+
+#include "zstd_deps.h"
+#define ZSTD_STATIC_LINKING_ONLY   /* ZSTD_customMem */
+#include "../zstd.h"
+
+typedef struct POOL_ctx_s POOL_ctx;
+
+/*! POOL_create() :
+ *  Create a thread pool with at most `numThreads` threads.
+ * `numThreads` must be at least 1.
+ *  The maximum number of queued jobs before blocking is `queueSize`.
+ * @return : POOL_ctx pointer on success, else NULL.
+*/
+POOL_ctx* POOL_create(size_t numThreads, size_t queueSize);
+
+POOL_ctx* POOL_create_advanced(size_t numThreads, size_t queueSize,
+                               ZSTD_customMem customMem);
+
+/*! POOL_free() :
+ *  Free a thread pool returned by POOL_create().
+ */
+void POOL_free(POOL_ctx* ctx);
+
+/*! POOL_resize() :
+ *  Expands or shrinks pool's number of threads.
+ *  This is more efficient than releasing + creating a new context,
+ *  since it tries to preserve and re-use existing threads.
+ * `numThreads` must be at least 1.
+ * @return : 0 when resize was successful,
+ *           !0 (typically 1) if there is an error.
+ *    note : only numThreads can be resized, queueSize remains unchanged.
+ */
+int POOL_resize(POOL_ctx* ctx, size_t numThreads);
+
+/*! POOL_sizeof() :
+ * @return threadpool memory usage
+ *  note : compatible with NULL (returns 0 in this case)
+ */
+size_t POOL_sizeof(POOL_ctx* ctx);
+
+/*! POOL_function :
+ *  The function type that can be added to a thread pool.
+ */
+typedef void (*POOL_function)(void*);
+
+/*! POOL_add() :
+ *  Add the job `function(opaque)` to the thread pool. `ctx` must be valid.
+ *  Possibly blocks until there is room in the queue.
+ *  Note : The function may be executed asynchronously,
+ *         therefore, `opaque` must live until function has been completed.
+ */
+void POOL_add(POOL_ctx* ctx, POOL_function function, void* opaque);
+
+
+/*! POOL_tryAdd() :
+ *  Add the job `function(opaque)` to thread pool _if_ a worker is available.
+ *  Returns immediately even if not (does not block).
+ * @return : 1 if successful, 0 if not.
+ */
+int POOL_tryAdd(POOL_ctx* ctx, POOL_function function, void* opaque);
+
+
+#if defined (__cplusplus)
+}
+#endif
+
+#endif

zstd/common/zstd_deps.h

@@ -0,0 +1,111 @@
+/*
+ * Copyright (c) 2016-2020, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+/* This file provides common libc dependencies that zstd requires.
+ * The purpose is to allow replacing this file with a custom implementation
+ * to compile zstd without libc support.
+ */
+
+/* Need:
+ * NULL
+ * INT_MAX
+ * UINT_MAX
+ * ZSTD_memcpy()
+ * ZSTD_memset()
+ * ZSTD_memmove()
+ */
+#ifndef ZSTD_DEPS_COMMON
+#define ZSTD_DEPS_COMMON
+
+#include <limits.h>
+#include <stddef.h>
+#include <string.h>
+
+#if defined(__GNUC__) && __GNUC__ >= 4
+# define ZSTD_memcpy(d,s,l) __builtin_memcpy((d),(s),(l))
+# define ZSTD_memmove(d,s,l) __builtin_memmove((d),(s),(l))
+# define ZSTD_memset(p,v,l) __builtin_memset((p),(v),(l))
+#else
+# define ZSTD_memcpy(d,s,l) memcpy((d),(s),(l))
+# define ZSTD_memmove(d,s,l) memmove((d),(s),(l))
+# define ZSTD_memset(p,v,l) memset((p),(v),(l))
+#endif
+
+#endif /* ZSTD_DEPS_COMMON */
+
+/* Need:
+ * ZSTD_malloc()
+ * ZSTD_free()
+ * ZSTD_calloc()
+ */
+#ifdef ZSTD_DEPS_NEED_MALLOC
+#ifndef ZSTD_DEPS_MALLOC
+#define ZSTD_DEPS_MALLOC
+
+#include <stdlib.h>
+
+#define ZSTD_malloc(s) malloc(s)
+#define ZSTD_calloc(n,s) calloc((n), (s))
+#define ZSTD_free(p) free((p))
+
+#endif /* ZSTD_DEPS_MALLOC */
+#endif /* ZSTD_DEPS_NEED_MALLOC */
+
+/*
+ * Provides 64-bit math support.
+ * Need:
+ * U64 ZSTD_div64(U64 dividend, U32 divisor)
+ */
+#ifdef ZSTD_DEPS_NEED_MATH64
+#ifndef ZSTD_DEPS_MATH64
+#define ZSTD_DEPS_MATH64
+
+#define ZSTD_div64(dividend, divisor) ((dividend) / (divisor))
+
+#endif /* ZSTD_DEPS_MATH64 */
+#endif /* ZSTD_DEPS_NEED_MATH64 */
+
+/* Need:
+ * assert()
+ */
+#ifdef ZSTD_DEPS_NEED_ASSERT
+#ifndef ZSTD_DEPS_ASSERT
+#define ZSTD_DEPS_ASSERT
+
+#include <assert.h>
+
+#endif /* ZSTD_DEPS_ASSERT */
+#endif /* ZSTD_DEPS_NEED_ASSERT */
+
+/* Need:
+ * ZSTD_DEBUG_PRINT()
+ */
+#ifdef ZSTD_DEPS_NEED_IO
+#ifndef ZSTD_DEPS_IO
+#define ZSTD_DEPS_IO
+
+#include <stdio.h>
+#define ZSTD_DEBUG_PRINT(...) fprintf(stderr, __VA_ARGS__)
+
+#endif /* ZSTD_DEPS_IO */
+#endif /* ZSTD_DEPS_NEED_IO */
+
+/* Only requested when <stdint.h> is known to be present.
+ * Need:
+ * intptr_t
+ */
+#ifdef ZSTD_DEPS_NEED_STDINT
+#ifndef ZSTD_DEPS_STDINT
+#define ZSTD_DEPS_STDINT
+
+#include <stdint.h>
+
+#endif /* ZSTD_DEPS_STDINT */
+#endif /* ZSTD_DEPS_NEED_STDINT */