embed the tokens XML file directly at build time, not run time.

Update CI to have newer compilers.

Author: adriweb

.github/workflows/build.yml

@@ -12,8 +12,22 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-22.04,ubuntu-latest,macOS-15-intel,macOS-latest]
+        os: [ubuntu-24.04,macos-15-intel,macos-26-intel,macos-26]
     steps:
-      - uses: actions/checkout@v5
-      - run: make -j4
+      - uses: actions/checkout@v6
+
+      - name: Install LLVM 19
+        if: startsWith(matrix.os, 'ubuntu')
+        run: sudo apt-get update && sudo apt-get install -y clang-19 lld-19
+
+      - name: Build
+        if: startsWith(matrix.os, 'ubuntu')
+        run: make -j4 CC=clang-19 CXX=clang++-19 LFLAGS=-fuse-ld=lld-19
+      - name: Build
+        if: matrix.os == 'macos-15-intel'
+        run: make -j4 CC=gcc-15 CXX=g++-15
+      - name: Build
+        if: startsWith(matrix.os, 'macos-26')
+        run: make -j4 CC=$(brew --prefix llvm@20)/bin/clang CXX=$(brew --prefix llvm@20)/bin/clang++
+
       - run: ./tivars_tests

CMakeLists.txt

@@ -1,4 +1,4 @@
-cmake_minimum_required(VERSION 3.5)
+cmake_minimum_required(VERSION 3.21)
 
 if(DEFINED TIVARS_ENABLE_FUZZING)
     set(TIVARS_ENABLE_FUZZING "${TIVARS_ENABLE_FUZZING}" CACHE BOOL "Build libFuzzer targets" FORCE)
@@ -22,7 +22,7 @@ endif()
 
 project(tivars_lib_cpp
         VERSION     1.3.0
-        LANGUAGES   CXX)
+        LANGUAGES   C CXX)
 
 include(CheckCXXCompilerFlag)
 if(MSVC)
@@ -49,6 +49,14 @@ endif()
 
 file(GLOB HEADER_FILES "src/*.h" "src/TypeHandlers/*.h")
 file(GLOB COMMON_SOURCE ${HEADER_FILES} "src/*.cpp" "src/TypeHandlers/*.cpp")
+set(TIVARS_BUILTIN_TOKENS_SOURCE src/TypeHandlers/BuiltinTokensXml.c)
+list(APPEND COMMON_SOURCE ${TIVARS_BUILTIN_TOKENS_SOURCE})
+set_source_files_properties(${TIVARS_BUILTIN_TOKENS_SOURCE}
+        PROPERTIES
+        C_STANDARD 23
+        C_STANDARD_REQUIRED ON
+        C_EXTENSIONS ON
+)
 
 # pugixml (vendored)
 include_directories(${CMAKE_CURRENT_SOURCE_DIR}/vendor/pugixml)
@@ -71,7 +79,6 @@ target_compile_definitions(tivars_lib_cpp_cli PRIVATE CXXOPTS_NO_REGEX=1)
 
 if(TIVARS_ENABLE_FUZZING)
     set(TIVARS_FUZZ_CORPUS_DIR ${CMAKE_CURRENT_SOURCE_DIR}/fuzz/corpus)
-    set(TIVARS_TOKENS_XML_PATH ${CMAKE_CURRENT_SOURCE_DIR}/ti-toolkit-8x-tokens.xml)
 
     add_custom_target(tivars_fuzzer_seed_corpus
             COMMAND ${CMAKE_COMMAND} -E make_directory ${TIVARS_FUZZ_CORPUS_DIR}
@@ -83,7 +90,6 @@ if(TIVARS_ENABLE_FUZZING)
     add_dependencies(tivars_lib_cpp_fuzzer tivars_fuzzer_seed_corpus)
     target_compile_definitions(tivars_lib_cpp_fuzzer PRIVATE
             FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1
-            TIVARS_TOKENS_XML_PATH="${TIVARS_TOKENS_XML_PATH}"
     )
     target_compile_options(tivars_lib_cpp_fuzzer PRIVATE
             -O1
@@ -212,9 +218,7 @@ if(APPLE)
             $<TARGET_OBJECTS:tivars_quicklook_support>
             quicklook/PreviewProvider.mm
             quicklook/ExtensionMain.m
-            ti-toolkit-8x-tokens.xml
     )
-    set_source_files_properties(ti-toolkit-8x-tokens.xml PROPERTIES MACOSX_PACKAGE_LOCATION Resources)
     set_target_properties(tivars_quicklook_preview_extension PROPERTIES
             BUNDLE_EXTENSION appex
             OUTPUT_NAME TIVarsQuickLookPreview
@@ -232,7 +236,6 @@ if(APPLE)
             $<TARGET_OBJECTS:tivars_quicklook_support>
             quicklook/ThumbnailProvider.mm
             quicklook/ExtensionMain.m
-            ti-toolkit-8x-tokens.xml
     )
     set_target_properties(tivars_quicklook_thumbnail_extension PROPERTIES
             BUNDLE_EXTENSION appex

Makefile

@@ -1,12 +1,14 @@
+CC ?= cc
 CXXFLAGS := -O3 -std=c++2a -DTH_GDB_SUPPORT=1 -Ivendor/pugixml -W -Wall -Wextra -Werror=write-strings -Werror=redundant-decls -Werror=format -Werror=format-security -Werror=date-time -Werror=return-type -Werror=pointer-arith -Winit-self
+CFLAGS := -O3 -std=c2x -W -Wall -Wextra
 
-SOURCES_COMMON := $(wildcard src/*.cpp) $(wildcard src/TypeHandlers/*.cpp) vendor/pugixml/pugixml.cpp
+SOURCES_COMMON := $(wildcard src/*.cpp) $(wildcard src/TypeHandlers/*.cpp) $(wildcard src/TypeHandlers/*.c) vendor/pugixml/pugixml.cpp
 
 SOURCES_TESTS := $(SOURCES_COMMON) tests.cpp
 SOURCES_CLI   := $(SOURCES_COMMON) cli/cli.cpp
 
-OBJS_TESTS = $(patsubst %.cpp, %.o, $(SOURCES_TESTS))
-OBJS_CLI   = $(patsubst %.cpp, %.o, $(SOURCES_CLI))
+OBJS_TESTS = $(patsubst %.c, %.o, $(patsubst %.cpp, %.o, $(SOURCES_TESTS)))
+OBJS_CLI   = $(patsubst %.c, %.o, $(patsubst %.cpp, %.o, $(SOURCES_CLI)))
 
 OUTPUT := tivars_tests tivars_cli
 
@@ -15,6 +17,9 @@ all: $(OUTPUT)
 %.o: %.cpp
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
+%.o: %.c
+	$(CC) $(CFLAGS) -c $< -o $@
+
 tivars_tests: $(OBJS_TESTS)
 	$(CXX) $(CXXFLAGS) $(LFLAGS) $^ -o $@
 

Makefile.emscripten

@@ -1,3 +1,4 @@
+CC := emcc
 CXX := em++
 DEBUG ?= 0
 
@@ -12,14 +13,15 @@ else
 endif
 
 # Emscripten stuff
-EMFLAGS := --bind -s WASM=1 -s MODULARIZE=1 -s EXPORT_ES6=1 -s EXPORT_NAME="'TIVarsLib'" -s NO_EXIT_RUNTIME=1 -s ASSERTIONS=$(ASSERTIONS) -s DISABLE_EXCEPTION_CATCHING=0 -s EXPORTED_RUNTIME_METHODS="['FS','getExceptionMessage','decrementExceptionRefcount']" $(DEBUG_EMFLAGS) --embed-file ti-toolkit-8x-tokens.xml
+EMFLAGS := --bind -s WASM=1 -s MODULARIZE=1 -s EXPORT_ES6=1 -s EXPORT_NAME="'TIVarsLib'" -s NO_EXIT_RUNTIME=1 -s ASSERTIONS=$(ASSERTIONS) -s DISABLE_EXCEPTION_CATCHING=0 -s EXPORTED_RUNTIME_METHODS="['FS','getExceptionMessage','decrementExceptionRefcount']" $(DEBUG_EMFLAGS)
 
 CXXFLAGS := $(OPTFLAGS) -std=c++2a -DTH_GDB_SUPPORT=1 -Ivendor/pugixml -W -Wall -Wextra
+CFLAGS := $(OPTFLAGS) -std=c2x -W -Wall -Wextra
 LFLAGS := $(OPTFLAGS) $(EMFLAGS)
 
-SOURCES := $(wildcard src/*.cpp) $(wildcard src/TypeHandlers/*.cpp) vendor/pugixml/pugixml.cpp
+SOURCES := $(wildcard src/*.cpp) $(wildcard src/TypeHandlers/*.cpp) $(wildcard src/TypeHandlers/*.c) vendor/pugixml/pugixml.cpp
 
-OBJS = $(patsubst %.cpp, %.bc, $(SOURCES))
+OBJS = $(patsubst %.c, %.bc, $(patsubst %.cpp, %.bc, $(SOURCES)))
 
 OUTPUT := TIVarsLib
 
@@ -30,6 +32,9 @@ all: wasm
 %.bc: %.cpp
 	$(CXX) $(CXXFLAGS) -c $< -o $@
 
+%.bc: %.c
+	$(CC) $(CFLAGS) -c $< -o $@
+
 $(OUTPUT).js: $(OBJS)
 	$(CXX) $(CXXFLAGS) $(LFLAGS) $^ -o $@
 

cli/cli.cpp

@@ -39,7 +39,6 @@ int main(int argc, char** argv)
             ("n,name", "Variable name", cxxopts::value<string>())
             ("t,type", "Variable type", cxxopts::value<string>())
             ("m,calc", "Calc. model", cxxopts::value<string>())
-            ("x,xml", "Path to tokens XML file (overrides default ti-toolkit-8x-tokens.xml)", cxxopts::value<string>())
             ("l,lang", "Language", cxxopts::value<string>()->default_value("en"))
             ("a,archive", "Archive status", cxxopts::value<bool>())
             ("r,reindent", "Re-indent", cxxopts::value<bool>())
@@ -214,14 +213,6 @@ int main(int argc, char** argv)
             }
             else
             {
-                if (result.count("xml"))
-                {
-                    string xmlFilePath = result["xml"].as<string>();
-                    TH_Tokenized::initTokensFromXMLFilePath(xmlFilePath);
-                } else {
-                    TH_Tokenized::initTokens();
-                }
-
                 string requestedName;
                 if (result.count("name"))
                 {

fuzz/tivars_file_fuzzer.cpp

@@ -21,10 +21,6 @@
 #include "../src/TypeHandlers/TypeHandlers.h"
 #include "../src/tivarslib_utils.h"
 
-#ifndef TIVARS_TOKENS_XML_PATH
-#define TIVARS_TOKENS_XML_PATH "ti-toolkit-8x-tokens.xml"
-#endif
-
 namespace
 {
     constexpr size_t maxInputSize = 2U * 1024U * 1024U;
@@ -47,7 +43,6 @@ namespace
         std::call_once(initFlag, []()
         {
             std::setlocale(LC_ALL, ".UTF-8");
-            tivars::TypeHandlers::TH_Tokenized::initTokensFromXMLFilePath(TIVARS_TOKENS_XML_PATH);
         });
     }
 

quicklook/TIVarsQuickLookSupport.mm

@@ -693,15 +693,6 @@ void initialize_library()
     {
         std::call_once(initFlag, []() {
             std::setlocale(LC_ALL, ".UTF-8");
-            NSString* xmlPath = [NSBundle.mainBundle pathForResource:@"ti-toolkit-8x-tokens" ofType:@"xml"];
-            if (xmlPath)
-            {
-                tivars::TypeHandlers::TH_Tokenized::initTokensFromXMLFilePath(xmlPath.UTF8String);
-            }
-            else
-            {
-                tivars::TypeHandlers::TH_Tokenized::initTokens();
-            }
         });
     }
 

scripts/api_smoke_tests.sh

@@ -27,6 +27,7 @@ int main()
 }
 CPP
 
+cc -std=c2x -c "$ROOT_DIR/src/TypeHandlers/BuiltinTokensXml.c" -o "$TMP_DIR/BuiltinTokensXml.o"
 COMMON_SOURCES=("$ROOT_DIR"/src/*.cpp "$ROOT_DIR"/src/TypeHandlers/*.cpp "$ROOT_DIR"/vendor/pugixml/pugixml.cpp)
-c++ -std=c++2a -DTH_GDB_SUPPORT=1 -I"$ROOT_DIR" -I"$ROOT_DIR/vendor/pugixml" "${COMMON_SOURCES[@]}" "$TMP_DIR/api_init_smoke.cpp" -o "$TMP_DIR/api_init_smoke"
+c++ -std=c++2a -DTH_GDB_SUPPORT=1 -I"$ROOT_DIR" -I"$ROOT_DIR/vendor/pugixml" "${COMMON_SOURCES[@]}" "$TMP_DIR/BuiltinTokensXml.o" "$TMP_DIR/api_init_smoke.cpp" -o "$TMP_DIR/api_init_smoke"
 "$TMP_DIR/api_init_smoke"

src/TypeHandlers/BuiltinTokensXml.c

@@ -0,0 +1,22 @@
+/*
+ * Part of tivars_lib_cpp
+ * (C) 2015-2026 Adrien "Adriweb" Bertrand
+ * https://github.com/adriweb/tivars_lib_cpp
+ * License: MIT
+ */
+
+#include <stddef.h>
+
+const unsigned char tivars_builtin_tokens_xml[] = {
+#if defined(__has_embed)
+#  if __has_embed("../../ti-toolkit-8x-tokens.xml")
+#    embed "../../ti-toolkit-8x-tokens.xml"
+#  else
+#    error "ti-toolkit-8x-tokens.xml not found"
+#  endif
+#else
+#  error "This compiler does not support #embed"
+#endif
+};
+
+const size_t tivars_builtin_tokens_xml_size = sizeof(tivars_builtin_tokens_xml);