restore non-performance files to develop; merge B-category overlaps

The squash carried implicit reverts of every develop commit landed between
the old branch base (e3cdf87) and current HEAD (2a864c5) — typo fixes
(21988e7), the get_timeseries_metadata N+1 optimization (324acba), the
TS_2DIFF float/double overflow-page fix (2a864c5), the 2.3.1 release-prep
poms, and several regression tests in develop's reader/writer test suites.

This commit:
  - restores 16 files the optimization branch never touched
  - 3-way merges 15 files both sides modified, keeping develop's typo
    fixes / N+1 optimization / TS_2DIFF overflow bitmaps / regression tests
    on top of the read/write batch optimization changes
  - keeps the small Windows-only compile fix in
    query_by_row_performance_test.cc and the zlib-1.2.13 -> 1.3.1 bump
  - restores the cpp/CMakeLists.txt TSFILE_OPTIMIZATION_FLAGS knob while
    keeping -O3 -march=native -flto as the Linux/macOS Release default

Author: ColinLeeo

RELEASE_NOTES.md

@@ -18,6 +18,19 @@
     under the License.
 
 -->
+# Apache TsFile 2.3.1
+
+## New Features
+
+- Added scripts to convert CSV, Parquet and Arrow formats to TsFile.
+- Adapted TsFile for the MSVC compiler.
+
+## Bugs
+
+- Fixed the issue that the format conversion scripts did not support date and timestamp data types.
+- Fixed garbled characters when using Chinese table names in the conversion scripts.
+- Fixed the issue where TsFile displayed empty when converting with uppercase column names.
+
 # Apache TsFile 2.3.0
 
 ## New Features
@@ -187,7 +200,7 @@
 * Added accountable function to measurementSchema by @Caideyipi in #509
 * Correct the retained size calculation for BinaryColumn and BinaryColumnBuilder by @JackieTien97 in #514
 * add switch to disable native lz4 (#480) by @jt2594838 in #515
-* Correct the memroy calculation of BinaryColumnBuilder by @JackieTien97 in #530
+* Correct the memory calculation of BinaryColumnBuilder by @JackieTien97 in #530
 * Fetch max tsblock line number each time from TSFileConfig by @JackieTien97 in #535
 * Support set default compression by data type & Bump org.apache.commons:commons-lang3 from 3.15.0 to 3.18.0 by @jt2594838 in #547
 * Avoid calculating shallow size of map by @shuwenwei in #566

cpp/CLAUDE.md

@@ -92,6 +92,7 @@ cpp/src/
 ## Code Style
 
 - **Formatter**: clang-format (Google style), configured in `.clang-format`
+- After modifying C++ code, run from the repo root to format: `./mvnw spotless:apply -P with-cpp`
 
 ## Testing
 

cpp/CMakeLists.txt

@@ -32,10 +32,15 @@ endif ()
 set(TsFile_CPP_VERSION 2.2.1.dev)
 
 if (MSVC)
-    # MSVC has no /std:c++11 flag; pin the closest supported standard mode.
+    # MSVC does not provide a /std:c++11 flag; C++11 is its implicit baseline.
+    # The lowest explicitly settable standard is /std:c++14. Without this flag,
+    # the default varies by VS version (VS2017+ defaults to C++14 mode with some
+    # C++17 extensions), so we pin it explicitly for reproducibility.
     set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} /W3 /utf-8 /EHsc /bigobj /Zc:__cplusplus /std:c++14")
     add_definitions(-DNOMINMAX -D_CRT_SECURE_NO_WARNINGS -D_CRT_NONSTDC_NO_WARNINGS
                     -D_SCL_SECURE_NO_WARNINGS -D_WINSOCK_DEPRECATED_NO_WARNINGS)
+    # Export all symbols of the tsfile shared library automatically so that
+    # consumers do not need __declspec(dllexport) annotations.
     set(CMAKE_WINDOWS_EXPORT_ALL_SYMBOLS ON)
 else ()
     set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall")
@@ -46,6 +51,8 @@ if (CMAKE_CXX_COMPILER_ID MATCHES "GNU")
 endif ()
 
 message("cmake using: USE_CPP11=${USE_CPP11}")
+# MSVC has no /std:c++11; CMake maps this to the closest supported standard
+# (C++14 default on MSVC), which compiles the C++11 codebase fine.
 set(CMAKE_CXX_STANDARD 11)
 set(CMAKE_CXX_STANDARD_REQUIRED OFF)
 if (NOT MSVC)
@@ -73,6 +80,13 @@ if (${COV_ENABLED})
     message("add_definitions -DCOV_ENABLED=1")
 endif ()
 
+option(ENABLE_MEM_STAT "Enable memory status" ON)
+
+if (ENABLE_MEM_STAT)
+    add_definitions(-DENABLE_MEM_STAT)
+    message("add_definitions -DENABLE_MEM_STAT")
+endif ()
+
 
 if (NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE "Release" CACHE STRING "Choose the type of build." FORCE)
@@ -91,25 +105,46 @@ else ()
 endif ()
 
 message("CMAKE BUILD TYPE " ${CMAKE_BUILD_TYPE})
-if (NOT MSVC)
-    if (CMAKE_BUILD_TYPE STREQUAL "Debug")
-        set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g")
-    elseif (CMAKE_BUILD_TYPE STREQUAL "Release")
-        # -flto + MinGW gcc + statically-linked antlr4_static produces
-        # unresolved-reference errors at link time (LTO intermediate objects
-        # can't see the .a's vtable thunks). -march=native is also a poor
-        # default for CI binaries shipped to other machines. Keep both on
-        # Linux/macOS where the optimization actually pays off.
-        if (MINGW OR WIN32)
-            set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
-        else ()
-            set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -march=native -flto")
+# Keep optimization policy external by default (caller/toolchain/CMake defaults).
+set(TSFILE_OPTIMIZATION_FLAGS ""
+        CACHE STRING
+        "Optional extra optimization flags for tsfile-cpp (e.g. -O3). Empty means inherit caller defaults.")
+if (TSFILE_OPTIMIZATION_FLAGS)
+    # Apply after CMake defaults for each config so explicit optimization can
+    # override default -O flags in Release/RelWithDebInfo/Debug/MinSizeRel.
+    set(CMAKE_CXX_FLAGS_DEBUG
+            "${CMAKE_CXX_FLAGS_DEBUG} ${TSFILE_OPTIMIZATION_FLAGS}")
+    set(CMAKE_CXX_FLAGS_RELEASE
+            "${CMAKE_CXX_FLAGS_RELEASE} ${TSFILE_OPTIMIZATION_FLAGS}")
+    set(CMAKE_CXX_FLAGS_RELWITHDEBINFO
+            "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} ${TSFILE_OPTIMIZATION_FLAGS}")
+    set(CMAKE_CXX_FLAGS_MINSIZEREL
+            "${CMAKE_CXX_FLAGS_MINSIZEREL} ${TSFILE_OPTIMIZATION_FLAGS}")
+    message("cmake using: TSFILE_OPTIMIZATION_FLAGS=${TSFILE_OPTIMIZATION_FLAGS}")
+else ()
+    message("cmake using: TSFILE_OPTIMIZATION_FLAGS=<inherit>")
+    # MSVC provides sensible per-configuration optimization flags by default; the
+    # GCC-style flags below would be rejected by cl.exe, so skip them on MSVC.
+    if (NOT MSVC)
+        if (CMAKE_BUILD_TYPE STREQUAL "Debug")
+            set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -O0 -g")
+        elseif (CMAKE_BUILD_TYPE STREQUAL "Release")
+            # -flto + MinGW gcc + statically-linked antlr4_static produces
+            # unresolved-reference errors at link time (LTO intermediate objects
+            # can't see the .a's vtable thunks). -march=native is also a poor
+            # default for CI binaries shipped to other machines. Keep both on
+            # Linux/macOS where the optimization actually pays off.
+            if (MINGW OR WIN32)
+                set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3")
+            else ()
+                set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -O3 -march=native -flto")
+            endif ()
+        elseif (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
+            set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O2 -g")
+        elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")
+            set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} -ffunction-sections -fdata-sections -Os")
+            set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
         endif ()
-    elseif (CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
-        set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${CMAKE_CXX_FLAGS_RELWITHDEBINFO} -O2 -g")
-    elseif (CMAKE_BUILD_TYPE STREQUAL "MinSizeRel")
-        set(CMAKE_CXX_FLAGS_MINSIZEREL "${CMAKE_CXX_FLAGS_MINSIZEREL} -ffunction-sections -fdata-sections -Os")
-        set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -Wl,--gc-sections")
     endif ()
 endif ()
 message("CMAKE DEBUG: CMAKE_CXX_FLAGS=${CMAKE_CXX_FLAGS}")
@@ -120,11 +155,22 @@ option(ENABLE_ASAN "Enable Address Sanitizer" OFF)
 if (ENABLE_ASAN)
     message("Address Sanitizer is enabled.")
     if (MSVC)
+        # MSVC ships AddressSanitizer; it requires Visual Studio 2019 16.9 or
+        # newer (MSVC_VERSION >= 1928). Only the address sanitizer is available
+        # (there is no UndefinedBehaviorSanitizer for MSVC).
         if (MSVC_VERSION LESS 1928)
             message(FATAL_ERROR
                 "ENABLE_ASAN requires MSVC 19.28+ (Visual Studio 2019 16.9); "
                 "detected MSVC_VERSION=${MSVC_VERSION}.")
         endif ()
+        # /fsanitize=address is incompatible with the /RTC* runtime checks that
+        # CMake injects into Debug builds, and with incremental linking. Strip
+        # /RTC* from the per-config flags and force non-incremental linking.
+        #
+        # ASan also needs debug info: /Zi (compile) + /DEBUG (link). Without it
+        # MSVC emits warning C5072 ("ASAN enabled without debug information
+        # emission"), which the bundled googletest build promotes to an error
+        # via /WX in Release builds, and ASan reports lose symbol/line info.
         add_compile_options(/fsanitize=address /Zi)
         foreach (flagsVar
                  CMAKE_C_FLAGS_DEBUG CMAKE_CXX_FLAGS_DEBUG
@@ -135,19 +181,6 @@ if (ENABLE_ASAN)
     elseif (NOT WIN32)
         set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fsanitize=address,undefined -fno-omit-frame-pointer")
 
-        # -flto + libstdc++ <regex> produces spurious ODR-violation reports
-        # under ASan (globals like __classnames / __collatenames in
-        # bits/regex.tcc show up once per LTO partition).
-        #
-        # -march=native lets gcc autovectorize tight byte-stride loops
-        # (e.g. Int64Packer::unpack_8values) into AVX2 32-byte gathers
-        # that overread by up to one SIMD lane past the end of the input
-        # buffer; the read sits inside ASan's redzone and ASan traps it
-        # as SEGV. The non-vectorized scalar code is correct, so just
-        # drop the aggressive flags whenever ASan is on.
-        string(REGEX REPLACE "(^| )-flto( |$)" " " CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
-        string(REGEX REPLACE "(^| )-march=native( |$)" " " CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE}")
-
         if (NOT APPLE)
             set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -static-libasan")
             set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fsanitize=address,undefined -static-libasan -static-libubsan")
@@ -198,10 +231,6 @@ if (ENABLE_ZLIB)
     add_definitions(-DENABLE_GZIP)
 endif()
 
-option(ENABLE_SIMD "Enable SIMD acceleration via SIMDe" ON)
-message("cmake using: ENABLE_SIMD=${ENABLE_SIMD}")
-set(ENABLE_SIMDE ${ENABLE_SIMD} CACHE BOOL "Enable SIMDe (SIMD Everywhere)" FORCE)
-
 option(ENABLE_THREADS "Enable multi-threaded read/write (requires pthreads)" ON)
 message("cmake using: ENABLE_THREADS=${ENABLE_THREADS}")
 
@@ -211,11 +240,11 @@ if (ENABLE_THREADS)
     link_libraries(Threads::Threads)
 endif()
 
-option(ENABLE_MEM_STAT "Enable per-module memory allocation statistics" ON)
-message("cmake using: ENABLE_MEM_STAT=${ENABLE_MEM_STAT}")
+option(ENABLE_SIMDE "Enable SIMDe (SIMD Everywhere)" OFF)
+message("cmake using: ENABLE_SIMDE=${ENABLE_SIMDE}")
 
-if (ENABLE_MEM_STAT)
-    add_definitions(-DENABLE_MEM_STAT)
+if (ENABLE_SIMDE)
+    add_definitions(-DENABLE_SIMDE)
 endif()
 
 # All libs will be stored here, including libtsfile, compress-encoding lib.
@@ -231,12 +260,15 @@ set(THIRD_PARTY_INCLUDE ${PROJECT_BINARY_DIR}/third_party)
 
 set(SAVED_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
 if (MSVC)
+    # MSVC does not provide a /std:c++11 flag; C++11 is its implicit baseline.
+    # The lowest explicitly settable standard is /std:c++14. Without this flag,
+    # the default varies by VS version (VS2017+ defaults to C++14 mode with some
+    # C++17 extensions), so we pin it explicitly for reproducibility.
     set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} /W3 /utf-8 /EHsc /bigobj /Zc:__cplusplus /std:c++14")
 else ()
     set(CMAKE_CXX_FLAGS "$ENV{CXXFLAGS} -Wall -std=c++11")
 endif ()
 add_subdirectory(third_party)
-set(CMAKE_CXX_FLAGS "${SAVED_CXX_FLAGS}")
 
 add_subdirectory(src)
 if (BUILD_TEST)
@@ -248,11 +280,5 @@ else()
     message("BUILD_TEST is OFF, skipping test directory")
 endif ()
 
-option(BUILD_EXAMPLES "Build examples (requires Arrow/Parquet)" OFF)
-if (BUILD_EXAMPLES)
-    add_subdirectory(examples)
-endif()
+add_subdirectory(examples)
 
-if (EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/experiment/CMakeLists.txt")
-    add_subdirectory(experiment)
-endif()

cpp/build.sh

@@ -149,7 +149,7 @@ then
   cd build/minsizerel
 else
   echo ""
-  echo "unknow build type: ${build_type}, valid build types(case intensive): Debug, Release, RelWithDebInfo, MinSizeRel"
+  echo "unknown build type: ${build_type}, valid build types(case insensitive): Debug, Release, RelWithDebInfo, MinSizeRel"
   echo ""
   exit 1
 fi

cpp/examples/CMakeLists.txt

@@ -22,15 +22,33 @@ message("Running in examples directory")
 
 if (NOT MSVC)
     set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
 endif ()
 
-# TsFile include dirs
+# TsFile include dir
 set(SDK_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/../src/)
-include_directories(${SDK_INCLUDE_DIR})
+message("SDK_INCLUDE_DIR: ${SDK_INCLUDE_DIR}")
+
+# TsFile shared object dir
+set(SDK_LIB_DIR_RELEASE ${PROJECT_SOURCE_DIR}/../build/Release/lib)
+message("SDK_LIB_DIR_RELEASE: ${SDK_LIB_DIR_RELEASE}")
+
+set(SDK_LIB_DIR_DEBUG ${PROJECT_SOURCE_DIR}/../build/Debug/lib)
+message("SDK_LIB_DIR_DEBUG: ${SDK_LIB_DIR_DEBUG}")
 include_directories(${PROJECT_SOURCE_DIR}/../third_party/antlr4-cpp-runtime-4/runtime/src)
 
-if (NOT MSVC)
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -DNDEBUG")
+set(BUILD_TYPE "Release")
+include_directories(${SDK_INCLUDE_DIR})
+
+if (DEFINED TSFILE_OPTIMIZATION_FLAGS AND NOT "${TSFILE_OPTIMIZATION_FLAGS}" STREQUAL "")
+    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${TSFILE_OPTIMIZATION_FLAGS}")
+    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${TSFILE_OPTIMIZATION_FLAGS}")
+    message("examples using: TSFILE_OPTIMIZATION_FLAGS=${TSFILE_OPTIMIZATION_FLAGS}")
+else ()
+    message("examples using: TSFILE_OPTIMIZATION_FLAGS=<inherit>")
+    if (NOT MSVC)
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -DNDEBUG")
+    endif ()
 endif ()
 
 # Arrow + Parquet are required (for bench_read)

cpp/examples/README.md

@@ -55,6 +55,14 @@ target_link_libraries(your_target ${TSFILE_LIB})
 
 Note: Set ${SDK_LIB} to your TSFile library directory.
 
+### Optional Optimization Control
+
+By default, `tsfile-cpp` inherits optimization settings from the caller/toolchain.
+If you want to override optimization for `tsfile-cpp`, pass
+`TSFILE_OPTIMIZATION_FLAGS` during configure:
+
+Leave `TSFILE_OPTIMIZATION_FLAGS` empty to keep inherited behavior.
+
 ## 3. Implementation Examples
 
 ### Directory Structure

cpp/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <groupId>org.apache.tsfile</groupId>
         <artifactId>tsfile-parent</artifactId>
-        <version>2.2.1-SNAPSHOT</version>
+        <version>2.3.2-SNAPSHOT</version>
     </parent>
     <artifactId>tsfile-cpp</artifactId>
     <packaging>pom</packaging>
@@ -99,8 +99,8 @@
                                     plugin's generate goal throw an NPE.
                                 -->
                             </options>
-                            <sourcePath/>
-                            <targetPath/>
+                            <sourcePath />
+                            <targetPath />
                         </configuration>
                     </execution>
                     <!-- Compile the test code -->