7.4 initial commit and 7.4.1 prerelease (#165)

* XDB-382 Merge upstream 7.4 initial commit
* XDB-382 Merge upstream 7.4.1
* XDB-382 cherrypick XDB-360 fix tests from ow-fork-main to release-7.4
* XDB-364 added getting jemalloc version (#158)
* XDB-368 create container with jemalloc (#159)
* XDB-382 create container for 7.4.1
* XDB-382 added thread lib and pytest
* XDB-382 fixed tests after upstream merge

Author: MarkSh1

.flake8

@@ -3,4 +3,4 @@ ignore = E203, E266, E402, E501, W503, F403, F401, E711, C901, E721, W605
 max-line-length = 88
 max-complexity = 18
 select = B,C,E,F,W,T4,B9
-exclude = bindings/python/fdb/six.py,contrib/Implib.so/implib-gen.py,documentation/sphinx/extensions/rubydomain.py
+exclude = contrib/Implib.so/implib-gen.py,documentation/sphinx/extensions/rubydomain.py

.github/workflows/build.yml

@@ -50,8 +50,8 @@ jobs:
             prepare: "debian-based"
             build_on: linux
             parallel: 5
-            image: foundationdb-build:7.4.0-1.ow.build
-            owner: owtech
+            image: foundationdb-build:7.4.1-0.ow.build
+            owner: ${GITHUB_REPOSITORY_OWNER@L}
     runs-on: ${{ matrix.run_on }}
     steps:
       - name: Checkout
@@ -180,12 +180,17 @@ jobs:
       JOSHUA_AGENT_TAG: "rhel8-20220816"
 
     steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          path: ${{github.workspace}}/src
+
       - name: Install dependencies
         shell: bash
         run: |
           sudo apt-get update
           sudo apt-get install -y sudo wget crudini git python3 python3-pip
-          sudo pip3 install wheel setuptools python-dateutil lxml boto3
+          sudo pip3 install wheel setuptools python-dateutil lxml boto3 pytest
 
       - name: Install FoundationDb
         shell: bash
@@ -227,12 +232,7 @@ jobs:
           podman pull ${{ env.JOSHUA_AGENT_URL }}/joshua-agent:${{ env.JOSHUA_AGENT_TAG }}
           podman create --name joshua-agent -v /etc/foundationdb:/etc/foundationdb -it joshua-agent:${{ env.JOSHUA_AGENT_TAG }}
           podman start joshua-agent
-
-      - name: Checkout
-        uses: actions/checkout@v4
-        with:
-          path: ${{github.workspace}}/src
-  
+ 
       - name: run tests
         shell: bash
         working-directory: ${{github.workspace}}/fdb-joshua

.pre-commit-config.yaml

@@ -5,7 +5,6 @@ repos:
     - id: black
       exclude: |
           (?x)^(
-              bindings/python/fdb/six.py|
               contrib/Implib.so/implib-gen.py|
               documentation/sphinx/extensions/rubydomain.py
           )$

CMakeLists.txt

@@ -24,7 +24,7 @@ if(POLICY CMP0135)
 endif()
 
 project(foundationdb
-  VERSION 7.4.0
+  VERSION 7.4.1
   DESCRIPTION "FoundationDB is a scalable, fault-tolerant, ordered key-value store with full ACID transactions."
   HOMEPAGE_URL "http://www.foundationdb.org/"
   LANGUAGES C CXX ASM Swift)
@@ -138,10 +138,26 @@ message(STATUS "FDB version is ${FDB_VERSION}")
 message(STATUS "FDB_PATCH is ${FDB_PATCH}")
 
 set(FDB_CURRENT_VERSION ${PROJECT_VERSION})
-set(FDB_FUTURE_VERSION "7.5.0")
-set(FDB_PREV_RELEASE_VERSION "7.3.41")
-set(FDB_PREV2_RELEASE_VERSION "7.1.33")
-set(FDB_PREV3_RELEASE_VERSION "7.0.0")
+set(FDB_FUTURE_VERSION "8.0.0")
+set(FDB_PREV_RELEASE_VERSION "7.3.59")
+set(FDB_PREV2_RELEASE_VERSION "7.1.61")
+
+set(MULTIREGION_TEST TRUE)
+option(NO_MULTIREGION_TEST "Disable multiregion simulation tests" OFF)
+
+if (NO_MULTIREGION_TEST)
+    set(MULTIREGION_TEST FALSE)
+    add_definitions(-DNO_MULTIREGION_TEST)
+    message(STATUS "NO_MULTIREGION_TEST is ON")
+endif()
+
+set(RESTART_TEST TRUE)
+option(NO_RESTART_TEST "Disable restart simulation tests" OFF)
+if (NO_RESTART_TEST)
+    set(RESTART_TEST FALSE)
+    add_definitions(-DNO_RESTART_TEST)
+    message(STATUS "NO_RESTART_TEST is ON")
+endif()
 
 ################################################################################
 # Flow
@@ -276,5 +292,3 @@ endif()
 print_components()
 
 message(STATUS "CPACK_COMPONENTS_ALL ${CPACK_COMPONENTS_ALL}")
-
-

README.md

@@ -20,6 +20,25 @@ Contributing to FoundationDB can be in contributions to the code base, sharing y
 
 ## Getting Started
 
+### Latest Stable Releases
+
+The latest stable releases are (were) versions that are recommended for production use, which have been extensively validated via simulation and real cluster tests and used in our production environment.
+
+| Branch   |     Latest Production Release      |  Notes |
+|:--------:|:-------------:|------:|
+| 7.3      |  [7.3.63](https://github.com/apple/foundationdb/releases/tag/7.3.63) | Supported |
+| 7.2      |   | Experimental |
+| 7.1      |  [7.1.57](https://github.com/apple/foundationdb/releases/tag/7.1.57)   |   Bug fixes |
+| 7.0      |           |   Experimental |
+| 6.3      |  [6.3.25](https://github.com/apple/foundationdb/releases/tag/6.3.25) |   Unsupported |
+
+- `Supported` branches are those we actively maintain and will publish new patch releases.
+- `Bug fixes` are branches we still accept bug fixes into the branch, but may not publish newer patch releases. The community can build the latest release binaries if needed and is encouraged to upgrade to the `Supported` branches.
+- `Experimental` branches are those used for internal feature testing. They are not recommended for production use.
+- `Unsupported` branches are those which will no longer receive any updates.
+
+If you are running on old production releases, we recommend always upgrading to the next major release's latest version, and then continue to the next major version, e.g., 6.2.X -> 6.3.25 -> 7.1.57 -> 7.3.63. These upgrade paths have been well tested in production (skipping a major release, not marked as `Experimental`, for an upgrade is only tested in simulation).
+
 ### Binary downloads
 
 Developers interested in using FoundationDB can get started by downloading and installing a binary package. Please see the [downloads page](https://github.com/apple/foundationdb/releases) for a list of available packages.
@@ -34,7 +53,7 @@ The official docker image for building is [`foundationdb/build`](https://hub.doc
 
 To build outside of the official docker image, you'll need at least these dependencies:
 
-1. Install cmake Version 3.13 or higher [CMake](https://cmake.org/)
+1. Install cmake Version 3.24.2 or higher [CMake](https://cmake.org/)
 1. Install [Mono](https://www.mono-project.com/download/stable/)
 1. Install [Ninja](https://ninja-build.org/) (optional, but recommended)
 
@@ -170,8 +189,8 @@ $SRCDIR/packaging/osx/buildpkg.sh . $SRCDIR
 Under Windows, only Visual Studio with ClangCl is supported
 
 1. Install Visual Studio 2019 (IDE or Build Tools), and enable llvm support
-1. Install  [CMake 3.15](https://cmake.org/) or higher
-1. Download [Boost 1.77.0](https://boostorg.jfrog.io/artifactory/main/release/1.77.0/source/boost_1_77_0.7z)
+1. Install  [CMake 3.24.2](https://cmake.org/) or higher
+1. Download [Boost 1.86.0](https://archives.boost.io/release/1.86.0/source/boost_1_86_0.tar.bz2)
 1. Unpack boost to C:\boost, or use `-DBOOST_ROOT=<PATH_TO_BOOST>` with `cmake` if unpacked elsewhere
 1. Install [Python](https://www.python.org/downloads/) if is not already installed by Visual Studio
 1. (Optional) Install [OpenJDK 11](https://developers.redhat.com/products/openjdk/download) to build Java bindings

bindings/bindingtester/known_testers.py

@@ -86,15 +86,6 @@ def _absolute_path(path):
 testers = {
     "python": Tester(
         "python",
-        "python " + _absolute_path("python/tests/tester.py"),
-        2040,
-        23,
-        MAX_API_VERSION,
-        types=ALL_TYPES,
-        tenants_enabled=True,
-    ),
-    "python3": Tester(
-        "python3",
         "python3 " + _absolute_path("python/tests/tester.py"),
         2040,
         23,

bindings/bindingtester/run_tester_loop.sh

@@ -20,7 +20,6 @@ function scripted() {
 
 function run_scripted() {
 	scripted python
-	scripted python3
 	scripted ruby
 	scripted java
 	scripted java_async
@@ -35,7 +34,6 @@ while `true`; do
 	echo "Pass $i"
 	i=$((i+1))
 	run python
-	run python3
 	run ruby
 	run java
 	run java_async

bindings/bindingtester/tests/test_util.py

@@ -150,11 +150,11 @@ def random_tuple_list(self, max_size, max_list_size):
 
     def random_range_params(self):
         if random.random() < 0.75:
-            limit = random.randint(1, 1e3)
+            limit = random.randint(1, int(1e3))
         elif random.random() < 0.75:
             limit = 0
         else:
-            limit = random.randint(1e8, (1 << 31) - 1)
+            limit = random.randint(int(1e8), (1 << 31) - 1)
 
         return (limit, random.randint(0, 1), random.randint(-2, 4))
 

bindings/bindingtester/util.py

@@ -27,18 +27,25 @@
 
 
 def initialize_logger_level(logging_level):
+  
     logger = get_logger()
 
-    assert logging_level in ["DEBUG", "INFO", "WARNING", "ERROR"]
+    # Map logging levels to their corresponding constants
+    LOGGING_LEVELS = {
+        "DEBUG": logging.DEBUG,
+        "INFO": logging.INFO,
+        "WARNING": logging.WARNING,
+        "ERROR": logging.ERROR,
+    }
+
+    if logging_level not in LOGGING_LEVELS:
+        raise ValueError(f"Invalid logging level: {logging_level}")
+    
+    logger.setLevel(LOGGING_LEVELS[logging_level])
+
+
+
 
-    if logging_level == "DEBUG":
-        logger.setLevel(logging.DEBUG)
-    elif logging_level == "INFO":
-        logger.setLevel(logging.INFO)
-    elif logging_level == "WARNING":
-        logger.setLevel(logging.WARNING)
-    elif logging_level == "ERROR":
-        logger.setLevel(logging.ERROR)
 
 
 def get_logger():

bindings/c/CMakeLists.txt

@@ -295,9 +295,29 @@ if(NOT WIN32)
     )
 
     file(GLOB API_TEST_FILES "${CMAKE_CURRENT_SOURCE_DIR}/test/apitester/tests/*.toml")
+    set(PATTERNS_TO_SKIP "XXXX")
+    if (USE_ASAN)
+      set(PATTERNS_TO_SKIP
+        ".*BlobGranule.*"  # skip BlobGranule tests because we don't use
+        ".*Tenant.*"       # skip Tenant tests because we don't use
+      )
+    endif()
     foreach(test_file ${API_TEST_FILES})
       get_filename_component(file_name "${test_file}" NAME_WE)
       set(test_name "fdb_c_api_test_${file_name}")
+
+      # skip tests that match the patterns in PATTERNS_TO_SKIP
+      set(SKIP_FILE FALSE)
+      foreach(PATTERN ${PATTERNS_TO_SKIP})
+          if(file_name MATCHES "${PATTERN}")
+              set(SKIP_FILE TRUE)
+              break()
+          endif()
+      endforeach()
+      if(SKIP_FILE)
+        continue()
+      endif()
+
       add_python_venv_test(NAME "${test_name}"
         COMMAND python ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/run_c_api_tests.py
           --build-dir ${CMAKE_BINARY_DIR}
@@ -333,22 +353,6 @@ if(NOT WIN32)
         --client-config-tester-bin $<TARGET_FILE:fdb_c_client_config_tester>
         )
 
-      add_python_venv_test(NAME fdb_c_upgrade_from_prev3_gradual
-        COMMAND python ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-        --build-dir ${CMAKE_BINARY_DIR}
-        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-        --upgrade-path "${FDB_PREV3_RELEASE_VERSION}" "${FDB_PREV2_RELEASE_VERSION}" "${FDB_PREV_RELEASE_VERSION}" "${FDB_CURRENT_VERSION}" "${FDB_PREV_RELEASE_VERSION}"
-        --process-number 3
-        )
-
-      add_python_venv_test(NAME fdb_c_upgrade_from_prev3_direct
-        COMMAND python ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
-        --build-dir ${CMAKE_BINARY_DIR}
-        --test-file ${CMAKE_SOURCE_DIR}/bindings/c/test/apitester/tests/upgrade/MixedApiWorkloadMultiThr.toml
-        --upgrade-path "${FDB_PREV3_RELEASE_VERSION}" "${FDB_CURRENT_VERSION}" "${FDB_PREV_RELEASE_VERSION}"
-        --process-number 3
-        )
-
       add_python_venv_test(NAME fdb_c_upgrade_from_prev2_gradual
         COMMAND python ${CMAKE_SOURCE_DIR}/tests/TestRunner/upgrade_test.py
         --build-dir ${CMAKE_BINARY_DIR}
@@ -398,22 +402,53 @@ if(NOT WIN32)
 
 endif()
 
-set(c_workloads_srcs
+set(cpp_workloads_srcs
   test/workloads/workloads.cpp
   test/workloads/workloads.h
   test/workloads/SimpleWorkload.cpp)
 
+set(c_workloads_srcs
+  test/workloads/CWorkload.c)
+
 if(OPEN_FOR_IDE)
+  add_library(cpp_workloads OBJECT ${cpp_workloads_srcs})
   add_library(c_workloads OBJECT ${c_workloads_srcs})
 else()
+  add_library(cpp_workloads SHARED ${cpp_workloads_srcs})
   add_library(c_workloads SHARED ${c_workloads_srcs})
 endif()
+set_target_properties(cpp_workloads PROPERTIES
+  LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/share/foundationdb")
 set_target_properties(c_workloads PROPERTIES
   LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/share/foundationdb")
-target_link_libraries(c_workloads PUBLIC fdb_c)
+target_link_libraries(cpp_workloads PUBLIC fdb_c)
+target_include_directories(c_workloads PUBLIC ${CMAKE_SOURCE_DIR}/bindings/c)
 
 if(NOT WIN32 AND NOT APPLE AND NOT OPEN_FOR_IDE)
-  target_link_options(c_workloads PRIVATE "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/external_workload.map,-z,nodelete")
+  # Add the -Wl,--undefined-version flag to the linker command to allow
+  # undefined symbols in version scripts. Clang 19 doesn't allow this and would
+  # complain with "symbol not defined" errors.
+  if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_VERSION VERSION_GREATER_EQUAL "19.0.0")
+    target_link_options(cpp_workloads PRIVATE
+      "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/external_workload.map"
+      "LINKER:-z,nodelete"
+      "LINKER:--undefined-version"
+    )
+    target_link_options(c_workloads PRIVATE
+      "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/external_workload.map"
+      "LINKER:-z,nodelete"
+      "LINKER:--undefined-version"
+    )
+  else()
+    target_link_options(cpp_workloads PRIVATE
+      "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/external_workload.map"
+      "LINKER:-z,nodelete"
+    )
+    target_link_options(c_workloads PRIVATE
+      "LINKER:--version-script=${CMAKE_CURRENT_SOURCE_DIR}/external_workload.map"
+      "LINKER:-z,nodelete"
+    )
+  endif()
 endif()
 
 # Generate shim library in Linux builds