Skip to content

Commit d25906d

Browse files
committed
+SUSUWU_IS_VIRTUAL, +SUSUWU_INSTALL_TENSORFLOW
@`build.sh`: +`SUSUWU_IS_VIRTUAL`, +`SUSUWU_INSTALL_TENSORFLOW`; installs prerequisites + TensorFlow. @`README.md#optionssetup`; document this. @`.gitignore`, @`sh/make.sh:SUSUWU_BUILD_CTAGS()`; exclude "./tensorflow/", "./ml_dtypes/". Is followup to: commit HEAD~1 (@`build.sh`; use SUSUWU_DEPENDENCY_INCLUDE()`), which gives TensorFlow paths to `${CXXFLAGS}` (if found).
1 parent d39b327 commit d25906d

4 files changed

Lines changed: 41 additions & 2 deletions

File tree

.gitignore

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,3 +71,7 @@ index.xhtml
7171
robots.txt
7272
compile_commands.json
7373

74+
# Vendored
75+
ml_dtypes/
76+
tensorflow/
77+
libtensorflow/

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,8 @@ Usage: [`./build.sh [OPTIONS]`](./build.sh) produces objects (`./obj/*.o`, for d
143143
- `--abort-on-first-error`: sets [`SUSUWU_ABORT_ON_FIRST_ERROR=true`](./sh/Macros.sh), which causes [`SUSUWU_BUILD_OBJECTS`](./sh/make.sh) to `exit 1` if a subbuild ([`${CC}`](./build.sh) or [`${CXX}`](./build.sh)) fails.
144144
- Environment flags: as [_GNU_ `make`'s](https://www.gnu.org/software/make/manual/make.html#Implicit-Variables), plus;
145145
- `export SUSUWU_SH_TPUT_COMMAND=<path>`; replaces calls to `tput` with `<path>` (for instance, with `no-such-command` to test that `SUSUWU_SH_COLOR_COUNT()` does not require `ncurses-utils`).
146+
- `SUSUWU_IS_VIRTUAL` is set (`=true`) if [`./build.sh`](./build.sh) is executed through [_GitHub Workflows_](https://docs.github.com/en/actions/writing-workflows/about-workflows). TODO: test for other amnesiac environments (such as Docker).
147+
- `SUSUWU_INSTALL_TENSORFLOW=true` to install [`libtensorflow`](https://github.com/tensorflow/tensorflow/) (+ prerequisites), `=false` to skip; default is `=${SUSUWU_IS_VIRTUAL}`.
146148
- Macro flags (use `vim build.sh` to put into `FLAGS_USER`). If `=true`, most use more resources, except `SUSUWU*PREFER_*` or `SUSUWU*SKIP_*`. "default is `=!defined(NDEBUG)`" is short for; "if `--debug`, default `=true`, but if `--release`, default `=false`".
147149
- `-DSUSUWU_UNIT_TESTS[=true|=false]` with `=true` to build + execute unit tests. Default is `=true`, but more stable future version could have default `=!defined(NDEBUG)`. If set to `=false`; compilation time, object size, execuable size reduced (to around half).
148150
- `-DSUSUWU_HEX_DOES_PREFIX=true` to have `classIoHex*()` insert/remove "0x". Default is `=false` (caller must do).

build.sh

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,26 @@ export FLAGS_TENSORFLOW="-std=c++17 -DUSE_TENSORFLOW_CNS" #/* `./cxx/*` uses `#i
3333
C_SOURCE_PATH="./c/" #/* Usage: replace with directory root for _C_ source code */
3434
CXX_SOURCE_PATH="./cxx/" #/* Usage: replace with directory root for _C++_ source code */
3535

36+
test -n "${GITHUB_ACTIONS}" #/* If `build.sh` is executed through [GitHub Workflows](https://docs.github.com/en/actions/writing-workflows/about-workflows). TODO: `|| <test for other amnesiac environments, such as Docker>` */
37+
SUSUWU_IS_VIRTUAL=$? #/* Set to last command (`test`)'s return value */
38+
SUSUWU_INSTALL_TENSORFLOW=${SUSUWU_IS_VIRTUAL} #/* If virtual, install prerequisites for `cxx/ClassTensorFlowCns.hxx`; use `export SUSUWU_INSTALL_TENSORFLOW=false` to reduce resource use, or `export SUSUWU_INSTALL_TENSORFLOW=true` to install prerequisites on all computers (default is to avoid changes to system unless virtual) */
39+
if [ true = "${SUSUWU_INSTALL_TENSORFLOW}" ]; then
40+
SUSUWU_PRINT "$0" "$(SUSUWU_SH_NOTICE)" "Was executed through one of GitHub's Workflows (or user set $(SUSUWU_SH_QUOTE "VAR" "SUSUWU_INSTALL_TENSORFLOW")), will auto-install $(SUSUWU_SH_QUOTE "CODE" "libeigen3-dev") and $(SUSUWU_SH_QUOTE "CODE" "libtensorflow")."
41+
sudo apt -y install libeigen3-dev || sudo apt -y install eigen
42+
if ! (sudo apt -y install libtensorflow #|| git clone https://github.com/tensorflow/tensorflow.git --depth 1
43+
); then # If normal `apt` is not sufficient to install `libtensorflow`, ...
44+
# ... then download and extract TensorFlow C++ library.
45+
LIBTENSORFLOW_TAR="libtensorflow-cpu-linux-x86_64-2.11.0.tar.gz"
46+
wget --no-verbose "https://storage.googleapis.com/tensorflow/libtensorflow/${LIBTENSORFLOW_TAR}" && \
47+
tar xzf "${LIBTENSORFLOW_TAR}" && \
48+
ls -a include && ls -a include/tensorflow && ls -a include/tensorflow/core && ls -a include/tensorflow/third-party && \
49+
sudo mv lib/* /usr/lib/ #&& \
50+
# sudo mv include/* /usr/include/ && \
51+
# ls /usr/include/tensorflow/
52+
git clone https://github.com/openxla/xla.git --depth 1 #`libtensorflow` does not include `xla`
53+
fi
54+
fi
55+
3656
if SUSUWU_DEPENDENCY_INCLUDE "-I" "libtensorflow" "./" "tensorflow/core/" "sudo apt install libtensorflow"; then
3757
TENSORFLOW_PATH_PREFIX=""
3858
elif SUSUWU_DEPENDENCY_INCLUDE "-I" "tensorflow" "tensorflow/" "tensorflow/core/" "git clone https://github.com/tensorflow/tensorflow.git --depth 1"; then
@@ -41,13 +61,25 @@ fi
4161
TENSORFLOW_INCLUDE_PATH="${SUSUWU_DEPENDENCY_INCLUDE_PATH}" #/* `TENSORFLOW_INCLUDE_PATH=$(SUSUWU_DEPENDENCY_INCLUDE ...)` discards `SUSUWU_DEPENDENCY_INCLUDE`'s changes to env vars */
4262
if [ -n "${TENSORFLOW_INCLUDE_PATH}" ]; then
4363
SUSUWU_DEPENDENCY_INCLUDE "-I" "tensorflow:xla" "${TENSORFLOW_PATH_PREFIX}xla/" "xla/" "https://github.com/openxla/xla.git --depth 1"
64+
XLA_SOURCE_PATH="${SUSUWU_DEPENDENCY_INCLUDE_PATH}"
4465
SUSUWU_DEPENDENCY_INCLUDE "-I" "tensorflow:tsl" "${TENSORFLOW_PATH_PREFIX}xla/third_party/tsl/" "tsl/" ""
4566

4667
ML_DTYPES_PREFIX="ml_dtypes/include/"
4768
ML_DTYPES_ROOT="xla/third_party/py"
4869
if ! SUSUWU_DEPENDENCY_INCLUDE "-F" "tensorflow:ml_dtypes" "${TENSORFLOW_INCLUDE_PATH}${TENSORFLOW_PATH_PREFIX}" "${ML_DTYPES_PREFIX}float8.h" "cd ${TENSORFLOW_INCLUDE_PATH}${TENSORFLOW_PATH_PREFIX}${ML_DTYPES_ROOT}/ml_dtypes && bazel build"; then # If can't use `ml_dtypes` from `tensorflow`
4970
ML_DTYPES_GIT="https://github.com/jax-ml/ml_dtypes.git"
50-
SUSUWU_DEPENDENCY_INCLUDE "-F" "jax-ml:ml_dtypes" "ml_dtypes/" "${ML_DTYPES_PREFIX}float8.h" "git clone ${ML_DTYPES_GIT} --depth 1" # use `ml_dtypes` from GitHub
71+
if [ ! -d "ml_dtypes" ] && [ true = "${SUSUWU_INSTALL_TENSORFLOW}" ]; then
72+
git clone "${ML_DTYPES_GIT}" --depth 1
73+
fi
74+
if SUSUWU_DEPENDENCY_INCLUDE "-F" "jax-ml:ml_dtypes" "ml_dtypes/" "${ML_DTYPES_PREFIX}float8.h" "git clone ${ML_DTYPES_GIT} --depth 1"; then # If can use `ml_dtypes` from GitHub
75+
ML_DTYPES_FALLBACK_PREFIX="tensorflow/core/platform/"
76+
ML_DTYPES_FALLBACK="${TENSORFLOW_INCLUDE_PATH}${ML_DTYPES_FALLBACK_PREFIX}"
77+
if [ -e "${ML_DTYPES_FALLBACK}float8.h" ]; then # If fallback path has `float8.h`
78+
SUSUWU_PRINT "$0" "$(SUSUWU_SH_WARNING)" "As last resort, will use $(SUSUWU_SH_QUOTE "PATH" "${ML_DTYPES_FALLBACK}") (which has $(SUSUWU_SH_QUOTE "PATH" "float8.h")) as path for $(SUSUWU_SH_QUOTE "CODE" "ml_dtypes"). If this causes more $(SUSUWU_SH_QUOTE "CODE" "#include") errors, execute $(SUSUWU_SH_QUOTE "CODE" "cd ${XLA_SOURCE_PATH} && git reset --hard HEAD") or $(SUSUWU_SH_QUOTE "CODE" "find \"${XLA_SOURCE_PATH}\" -type f -exec sed \"s|\\\"${ML_DTYPES_PREFIX}|\\\"${ML_DTYPES_FALLBACK_PREFIX}|\" -i'' {} +") to undo."
79+
find "${XLA_SOURCE_PATH}" -type f -exec sed "s|\"${ML_DTYPES_PREFIX}|\"${ML_DTYPES_FALLBACK_PREFIX}|" -i'' {} + # [error: 'ml_dtypes/include/float8.h' file not found](https://github.com/tensorflow/tensorflow/issues/93130) fix. #TODO: filter with `grep "\.\(h\|cc\)$"`
80+
export SUSUWU_USED_ML_DTYPES_SED=true
81+
fi
82+
fi
5183
fi
5284
SUSUWU_DEPENDENCY_INCLUDE "-I" "eigen" "eigen3/" "Eigen/" "sudo apt install eigen || git clone https://github.com/PX4/eigen.git --depth 1"
5385
# SUSUWU_DEPENDENCY_INCLUDE "-I" "eigen" "eigen3/" "eigen3/Eigen/" "sudo apt install eigen || git clone https://github.com/PX4/eigen.git --depth 1"
@@ -76,6 +108,7 @@ if [ -n "${TENSORFLOW_INCLUDE_PATH}" ]; then #/* If `libtensorflow` was found */
76108
SUSUWU_PRINT "$0" "$(SUSUWU_SH_NOTICE)" "$(SUSUWU_SH_QUOTE "CODE" "${CXX} ${SUSUWU_TENSORFLOW_TEST_PATH}") passed, will enable $(SUSUWU_SH_QUOTE "CODE" "CXXFLAGS=\"\${CXXFLAGS} ${FLAGS_TENSORFLOW}\"")."
77109
else
78110
SUSUWU_PRINT "$0" "$(SUSUWU_SH_NOTICE)" "$(SUSUWU_SH_QUOTE "CODE" "${CXX} ${CXXFLAGS} ${SUSUWU_TENSORFLOW_TEST_PATH}") failed, will not enable $(SUSUWU_SH_QUOTE "CODE" "CXXFLAGS=\"\${CXXFLAGS} ${FLAGS_TENSORFLOW}\"") (skipped). If $(SUSUWU_SH_QUOTE "CODE" "libtensorflow") is installed, insert $(SUSUWU_SH_QUOTE "CODE" "${FLAGS_TENSORFLOW}") into $(SUSUWU_SH_QUOTE "CODE" "$0:FLAGS_USER"). To troubleshoot, use $(SUSUWU_SH_QUOTE "CODE" "cd ${TENSORFLOW_INCLUDE_PATH} && ./configure")"
111+
# ${SUSUWU_USED_ML_DTYPES_SED} && find "${XLA_SOURCE_PATH}" -type f -exec sed "s|\"${ML_DTYPES_FALLBACK_PREFIX}|\"${ML_DTYPES_PREFIX}|" -i'' {} + # [error: 'ml_dtypes/include/float8.h' file not found](https://github.com/tensorflow/tensorflow/issues/93130) fix. #TODO: exclude 'third_party/xla/xla/tsl/platform/resource_loader.h'
79112
fi
80113
fi
81114
SUSUWU_PROCESS_INCLUDES "${CXX_SOURCE_PATH}Class*.hxx" "${CXX_SOURCE_PATH}Macros.hxx"

sh/make.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,7 @@ SUSUWU_BUILD_CTAGS() ( #/* Usage: `SUSUWU_BUILD_CTAGS [-flags... --flags...] [SO
257257
SUSUWU_STATUS=1
258258
if command -v ctags >/dev/null; then
259259
if [ -z "${1}" ] || [ -z "${2}" ]; then
260-
CTAGS_DEFAULTS="-R --exclude=.git/ --exclude=*.html --exclude=compile_commands.json ."
260+
CTAGS_DEFAULTS="-R --exclude=.git/* --exclude=*.html --exclude=compile_commands.json --exclude=ml_dtypes/* --exclude=tensorflow/* ."
261261
SUSUWU_PRINT "SUSUWU_BUILD_CTAGS()" "$(SUSUWU_SH_NOTICE)" "Was called with less than 2 params; will default to $(SUSUWU_SH_QUOTE "CODE" "SUSUWU_BUILD_CTAGS ${CTAGS_DEFAULTS}")."
262262
#shellcheck disable=SC2086
263263
ctags ${CTAGS_DEFAULTS} && SUSUWU_STATUS=0

0 commit comments

Comments
 (0)