diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 8c3f433efb..d28fefa6eb 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -52,7 +52,7 @@ This github action builds and optionally publishes the documentation located in The [docs](./build.yml#L124) job: -- creates a `miniconda` environment from [requirements-test.yml](../../scripts/requirements-test.yml) and [docs_environment.yml](../../docs/docs_environment.yml) +- creates a `miniconda` environment from [cil_development.yml](../../scripts/cil_development.yml) and [docs_environment.yml](../../docs/docs_environment.yml) - `cmake` builds & installs CIL into the `miniconda` environment + builds the HTML documentation with `sphinx` - installs ruby dependencies from [`Gemfile`](../../docs/Gemfile) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 56e584701d..a50164215c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -37,11 +37,11 @@ jobs: run: | envname="${GITHUB_REPOSITORY##*/}-${GITHUB_RUN_ID}.${GITHUB_RUN_NUMBER}" echo "envname=$envname" >> $GITHUB_OUTPUT - sed -ri -e 's/^(name: ).*/\1$envname/' -e '/ python /d' -e 's/(.* numpy) .*/\1=${{ matrix.numpy-version }}/' scripts/requirements-test.yml + sed -ri -e 's/^(name: ).*/\1$envname/' -e '/ python /d' -e 's/(.* numpy) .*/\1=${{ matrix.numpy-version }}/' scripts/cil_development.yml - uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - environment-file: scripts/requirements-test.yml + environment-file: scripts/cil_development.yml activate-environment: ${{ steps.reqs.outputs.envname }} run-post: false - id: build @@ -72,11 +72,11 @@ jobs: - uses: actions/checkout@v4 with: {fetch-depth: 0, submodules: recursive} - name: set requirements - run: sed -ri -e '/ python /d' -e 's/(.* numpy) .*/\1=${{ matrix.numpy-version }}/' -e 's/=cuda*//' -e '/tigre/d' scripts/requirements-test.yml + run: sed -ri -e '/ python /d' -e 's/(.* numpy) .*/\1=${{ matrix.numpy-version }}/' -e 's/=cuda*//' -e '/tigre/d' scripts/cil_development.yml - uses: conda-incubator/setup-miniconda@v3 with: python-version: ${{ matrix.python-version }} - environment-file: scripts/requirements-test.yml + environment-file: scripts/cil_development.yml activate-environment: cil_dev - run: pip install . - name: test @@ -224,7 +224,7 @@ jobs: - name: install dependencies run: | mamba install -c conda-forge -yq conda-merge - conda-merge ../scripts/requirements-test.yml docs_environment.yml > environment.yml + conda-merge ../scripts/cil_development.yml docs_environment.yml > environment.yml mamba env update -n test --file environment.yml conda list - run: pip install .. diff --git a/CHANGELOG.md b/CHANGELOG.md index 570e43b5f8..bc213eccd2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,8 +17,10 @@ - cvxpy version set to !=1.8.2 to fix #2303 (#2306) - Update to TomoPhantom v3.0 (#2287) - Handle regularisation toolkit CPU only package error message (#2302) + - Update FindIPP.cmake to find IPP libraries in conda environments (#2286) - Documentation: - Render the user showcase notebooks in the documentation (#2189) + - Update on build instructions in README and developer guide for all OS (#2286) - Enhancements: - Add prefix argument to TIFFStackReader to load a subset of TIFF files in a folder (#2239) diff --git a/Dockerfile b/Dockerfile index 5c508e0d71..6708a0ec72 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,9 +11,9 @@ LABEL org.opencontainers.image.licenses="Apache-2.0 AND BSD-3-Clause AND GPL-3.0 # CUDA-specific packages ARG CIL_EXTRA_PACKAGES="tigre=2.6 astra-toolbox=2.1.0=cuda*" # build & runtime dependencies -# TODO: sync scripts/create_local_env_for_cil_development.sh, scripts/requirements-test.yml, recipe/meta.yaml (e.g. missing libstdcxx-ng _openmp_mutex pip)? +# TODO: sync scripts/create_local_env_for_cil_development.sh, scripts/cil_development.yml, recipe/meta.yaml (e.g. missing libstdcxx-ng _openmp_mutex pip)? # vis. https://github.com/TomographicImaging/CIL/pull/1590 -COPY --chown="${NB_USER}" scripts/requirements-test.yml environment.yml +COPY --chown="${NB_USER}" scripts/cil_development.yml environment.yml # channel_priority: https://stackoverflow.com/q/58555389 RUN sed -ri '/tigre|astra-toolbox| python /d' environment.yml \ && for pkg in 'jupyter-server-proxy>4.1.0' $CIL_EXTRA_PACKAGES; do echo " - $pkg" >> environment.yml; done \ diff --git a/README.md b/README.md index 92dcd1c392..1bfa24e52f 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ A number of additional dependencies are required for specific functionality in C #### Binary packages and dependencies -While building the CIL package we test with specific versions of dependencies. These are listed in the [build.yml](https://github.com/TomographicImaging/CIL/blob/master/.github/workflows/build.yml) GitHub workflow and [environment-test.yml](https://github.com/TomographicImaging/CIL/blob/master/scripts/requirements-test.yml). The following table tries to resume the tested versions of CIL and its required and optional dependencies. If you use these packages as a backend please remember to cite them in addition to CIL. +While building the CIL package we test with specific versions of dependencies. These are listed in the [build.yml](https://github.com/TomographicImaging/CIL/blob/master/.github/workflows/build.yml) GitHub workflow and [cil_development.yml](https://github.com/TomographicImaging/CIL/blob/master/scripts/cil_development.yml). The following table tries to resume the tested versions of CIL and its required and optional dependencies. If you use these packages as a backend please remember to cite them in addition to CIL. | Package | Tested Version | Conda install command | Description | License | |----|----|--------|--------|----| @@ -120,80 +120,7 @@ Jupyter Notebooks usage examples without any local installation are provided in ## Building CIL from source code -### Getting the code - -In case of development it is useful to be able to build the software directly. You should clone this repository as - -```sh -git clone --recurse-submodule git@github.com:TomographicImaging/CIL -``` - -The use of `--recurse-submodule` is necessary if the user wants the examples data to be fetched (they are needed by the unit tests). We have moved such data, previously hosted in this repo at `Wrappers/Python/data` to the [CIL-data](https://github.com/TomographicImaging/CIL-Data) repository and linked it to this one as submodule. If the data is not available it can be fetched in an already cloned repository as - -```sh -git submodule update --init --recursive -``` - -### Building with `pip` - -#### Install Dependencies - -To create a conda environment with all the dependencies for building CIL run the following shell script: - -```sh -bash ./scripts/create_local_env_for_cil_development.sh -``` - -Or with the CIL build and test dependencies: - -```sh -bash ./scripts/create_local_env_for_cil_development.sh -t -``` - -And then install CIL in to this environment using `pip`. - -Alternatively, one can use the `scripts/requirements-test.yml` to create a conda environment with all the -appropriate dependencies, using the following command: - -```sh -conda env create -f ./scripts/requirements-test.yml -``` -or, on windows: -```sh -conda env create -f ./scripts/requirements-test-windows.yml -``` - -#### Build CIL - -A C++ compiler is required to build the source code. Let's suppose that the user is in the source directory, then the following commands should work: - -```sh -pip install --no-deps . -``` - -If not installing inside a conda environment, then the user might need to set the locations of optional libraries: - -```sh -pip install . -Ccmake.define.IPP_ROOT="" -Ccmake.define.OpenMP_ROOT="" -``` - -### Building with Docker - -In the repository root, simply update submodules and run `docker build`: - -```sh -git submodule update --init --recursive -docker build . -t ghcr.io/tomographicimaging/cil -``` - -### Testing - -One installed, CIL functionality can be tested using the following command: - -```sh -export TESTS_FORCE_GPU=1 # optional, makes GPU test failures noisy -python -m unittest discover -v ./Wrappers/Python/test -``` +For instructions on how to build CIL from source code, please see our [Developers' Guide](https://tomographicimaging.github.io/CIL/nightly/developer_guide/) ## Citing CIL diff --git a/docs/docs_environment.yml b/docs/docs_environment.yml index c24e24d6e0..78ba3a5697 100644 --- a/docs/docs_environment.yml +++ b/docs/docs_environment.yml @@ -18,7 +18,7 @@ channels: - ccpi - defaults dependencies: - # in addition to CIL deps (see ../scripts/requirements-test.yml) + # in addition to CIL deps (see ../scripts/cil_development.yml) - jinja2 # - pydata-sphinx-theme - recommonmark diff --git a/docs/source/developer_guide.rst b/docs/source/developer_guide.rst index 0c65dae4d9..10667f9da0 100644 --- a/docs/source/developer_guide.rst +++ b/docs/source/developer_guide.rst @@ -17,7 +17,132 @@ Developers' Guide ***************** CIL is an Object Orientated software. It has evolved during the years and it currently does not fully adhere to the following conventions. New additions must comply with -the following. +the conventions and documentation guidelines described in this section. + +Building CIL from source code +============================== + +Getting the code +^^^^^^^^^^^^^^^^ + +In case of local development and testing it is useful to be able to build the software directly. +You should first clone this repository as: + +.. code:: sh + + git clone git@github.com:TomographicImaging/CIL + +The parameter ``--depth 1`` can be added to create a shallow clone with a history truncated to the specified number of commits reducing the size of the clone. +See `git documentation `_ + + +Building with ``pip`` +^^^^^^^^^^^^^^^^^^^^^ + +Install Dependencies +"""""""""""""""""" + +We suggest creating a conda environment with all the dependencies for building CIL using the appropriate command for your operating system: + +.. list-table:: + :header-rows: 1 + :widths: 20 65 15 + + * - OS + - Command + - Status + * - Linux + - ``conda env create -f ./scripts/cil_development.yml`` + - Tested + * - Windows + - ``conda env create -f ./scripts/cil_development.yml`` + - Tested + * - MacOS (ARM) + - ``conda env create -f ./scripts/cil_development_osx.yml`` + - Experimental + +.. note:: + Currently only Linux and Windows are tested and supported. The support on MacOS (ARM) is experimental and certain features are not available and not working, such as FFT filtering for FDK. + +This will create an environment called `cil_dev`. + +Activate the environment with: + +.. code:: sh + + conda activate cil_dev + +Build CIL +"""""""" + +A C++ compiler is required to build the source code. Let's suppose that the user is in the source directory, then the following commands should work: + + +.. code:: sh + + pip install -e . + +.. note:: + You need to have a **working compiler** on your system, such as Visual Studio on Windows, GCC on Linux and XCode on MacOS. + + +If not installing inside a conda environment, then the user might need to set the locations of optional libraries: + +.. code:: sh + + pip install -e . -Ccmake.define.IPP_ROOT="" -Ccmake.define.OpenMP_ROOT="" -Ccmake.define.CMAKE_BUILD_TYPE=RelWithDebInfo + + +Notes for Windows users +""""""""""""""""""""""" + +One option for development on Windows is using `Windows Subsystem for Linux (WSL) `_ +Launch WSL and install build-essential using: + +.. code:: sh + + apt install build-essential + +This will enable you to then follow the linux instructions for creating the environment and building CIL. + + +To build for windows you can install Visual Studio Community (or higher) and select the **Desktop development with C++** workload. + +If you are developing on Windows with conda, you need to have access to both the Visual Studio compiler and have created the conda environment using the command for Windows above. + +You can achieve this in two ways: + +1. by opening "x64 Native Tools Command Prompt for VS" and activating the conda environment from there. This requires you + to know the path to the ``conda.bat`` file, which is typically located in the ``condabin`` subdirectory of your conda installation. + Once located you need to run ``\conda.bat activate `` to activate the conda environment, and then you can run the build command from there. +2. by opening the conda prompt and running the ``vcvarsall.bat x64`` file from the Visual Studio installation (with ``x64`` argument), and then running the build command. + This requires you to know the path to the ``vcvarsall.bat`` file, + which is typically located in the ``VC/Auxiliary/Build`` subdirectory of your Visual Studio installation. + +Note: we tested these instructions with Visual Studio 2026 version 18.1.1 + + +Building with Docker +^^^^^^^^^^^^^^^^^^^^^ + +In the repository root, simply update submodules and run ``docker build``: + +.. code:: sh + + git submodule update --init --recursive + docker build . -t ghcr.io/tomographicimaging/cil + + +Testing +^^^^^^^ + +Once installed, CIL functionality can be tested using the following command: + +.. code:: sh + + export TESTS_FORCE_GPU=1 # optional, makes GPU test failures noisy + python -m unittest discover -v ./Wrappers/Python/test + Conventions on new CIL objects ============================== @@ -110,6 +235,7 @@ Rendered .. automethod:: cil.recon.FBP.FBP.run + Building documentation locally ------------------------------ @@ -133,11 +259,11 @@ a HTTP server to view the documentation. Example: :: - git clone --recurse-submodule git@github.com:TomographicImaging/CIL + git clone git@github.com:TomographicImaging/CIL cd CIL - sh scripts/create_local_env_for_cil_development_tests.sh -n NUMPY_VERSION -p PYTHON_VERSION -e ENVIRONMENT_NAME + conda env create -f ./scripts/cil_development.yml conda activate ENVIRONMENT_NAME - pip install . + pip install -e . cd docs conda update -n base -c defaults conda conda env update -f docs_environment.yml # with the name field set to ENVIRONMENT_NAME diff --git a/recipe/build.sh b/recipe/build.sh index 872bff219b..93f548655a 100755 --- a/recipe/build.sh +++ b/recipe/build.sh @@ -6,7 +6,7 @@ if test $(python -c "from __future__ import print_function; import platform; pri echo "Darwin" extra_args="$extra_args -DOPENMP_LIBRARIES=${CONDA_PREFIX}/lib -DOPENMP_INCLUDES=${CONDA_PREFIX}/include" else - echo "something else" + echo "Not Darwin" fi export SETUPTOOLS_SCM_PRETEND_VERSION_FOR_CIL="${PKG_VERSION}" diff --git a/scripts/requirements-test.yml b/scripts/cil_development.yml similarity index 100% rename from scripts/requirements-test.yml rename to scripts/cil_development.yml diff --git a/scripts/requirements-test-windows.yml b/scripts/cil_development_osx.yml similarity index 69% rename from scripts/requirements-test-windows.yml rename to scripts/cil_development_osx.yml index 1d1f08ef47..c530490633 100644 --- a/scripts/requirements-test-windows.yml +++ b/scripts/cil_development_osx.yml @@ -1,5 +1,5 @@ -# Copyright 2025 United Kingdom Research and Innovation -# Copyright 2025 The University of Manchester +# Copyright 2023 United Kingdom Research and Innovation +# Copyright 2023 The University of Manchester # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -14,28 +14,18 @@ name: cil_dev channels: - conda-forge - - nvidia - https://software.repos.intel.com/python/conda dependencies: - - python >=3.10 - - numpy >=1.23,<2 + - python=3.12 + - numpy=1.26 + - llvm-openmp - ccpi::cil-data >=22 - - ccpi::tigre 2.6 - - ccpi::ccpi-regulariser 24.0.1 - - astra-toolbox 2.1 cuda* + - numba + - astra-toolbox=2.1=py* - cvxpy!=1.8.2 - - ccpi::tomophantom 3.0.3 - python-wget - scikit-image - - packaging - - cmake >=3.16 - - setuptools >=64 - - ipp-include 2021.12 - - ipp-devel 2021.12 - - ipp 2021.12 - - ipywidgets - - scipy >=1.4 - - matplotlib-base >=3.3 + - matplotlib-base >=3.3.0 - h5py - pillow - dxchange >=0.2.1 @@ -47,3 +37,4 @@ dependencies: - pip - pip: - unittest-parametrize + \ No newline at end of file diff --git a/scripts/create_local_env_for_cil_development.sh b/scripts/create_local_env_for_cil_development.sh deleted file mode 100755 index 0808a0d074..0000000000 --- a/scripts/create_local_env_for_cil_development.sh +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env bash - -# Copyright 2020 United Kingdom Research and Innovation -# Copyright 2020 The University of Manchester -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# Authors: -# CIL Developers, listed at: https://github.com/TomographicImaging/CIL/blob/master/NOTICE.txt - -set -euxo pipefail -numpy='1.24' -python='3.10' -name=cil -test_deps=0 -cil_ver='' -pip_install_pkgs=() - -while getopts hn:p:e:tv: option ; do - case "${option}" in - n) numpy="${OPTARG}" ;; - p) python="${OPTARG}" ;; - e) name="${OPTARG}" ;; - t) test_deps=1 ;; - v) test_deps=1 ; cil_ver="${OPTARG}" ;; - h) - echo "Usage: $0 [-n numpy_version] [-p python_version] [-e environment_name] [-t] [-v cil_version]" - echo 'Where `-t` installs test dependencies, and `-v cil_version` implies `-t`' - exit - ;; - *) - echo "Wrong option passed. Use the -h option to get some help." >&2 - exit 1 - ;; - esac -done - -echo "Numpy $numpy" -echo "Python $python" -echo "Environment name $name" - -conda_args=(create --name="$name" - python="$python" - numpy="$numpy" - cmake'>=3.16' - dxchange'>=0.2.1' - h5py - llvm-openmp - ipp'>=2021.10' - ipp-devel'>=2021.10' - ipp-include'>=2021.10' - matplotlib-base - numba - olefile'>=0.46' - packaging - pillow - python-wget - pywavelets - scikit-image - scipy - tqdm - zenodo_get'>=1.6' -) - -if test "$(uname)" = Linux; then - conda_args+=(libgcc-ng libstdcxx-ng) -fi - -if test -n "$cil_ver"; then - echo "CIL version $cil_ver" - conda_args+=(cil="${cil_ver}") -fi - -if test $test_deps = 0; then - conda_args+=(-c conda-forge -c https://software.repos.intel.com/python/conda --override-channels) -else - conda_args+=( - astra-toolbox=2.1=cuda* - ccpi-regulariser=24.0.1 - cil-data - cvxpy!=1.8.2 - ipywidgets - packaging - python-wget - setuptools - scikit-image - tigre=2.6 - tomophantom=2.0.0 - -c conda-forge - -c https://software.repos.intel.com/python/conda - -c ccpi/label/dev - -c ccpi - --override-channels - ) - pip_install_pkgs+=( - unittest-parametrize - ) -fi - -conda "${conda_args[@]}" -if [[ -n "${pip_install_pkgs[@]}" ]]; then - env_path=$(conda info --base)/envs/"$name" - if [[ "$OSTYPE" =~ msys|win32|cygwin ]]; then - python_exec="$env_path/python.exe" - else - python_exec="$env_path/bin/python" - fi - "$python_exec" -m pip install "${pip_install_pkgs[@]}" -fi diff --git a/scripts/create_local_env_for_cil_development_tests.sh b/scripts/create_local_env_for_cil_development_tests.sh deleted file mode 100755 index f2fbc7c702..0000000000 --- a/scripts/create_local_env_for_cil_development_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/usr/bin/env bash -set -euxo pipefail -$(dirname "$0")/create_local_env_for_cil_development.sh -t "$@" diff --git a/src/Core/cmake/FindIPP.cmake b/src/Core/cmake/FindIPP.cmake index b5c402d202..070b7ad215 100644 --- a/src/Core/cmake/FindIPP.cmake +++ b/src/Core/cmake/FindIPP.cmake @@ -10,16 +10,18 @@ elseif(NOT WIN32) set(IPP_PRE "lib") set(IPP_POST ".so") endif() -find_library(IPP_CORE ${IPP_PRE}ippcore${IPP_POST} PATHS ${IPP_ROOT_DIR}) -find_library(IPP_S ${IPP_PRE}ipps${IPP_POST} PATHS ${IPP_ROOT_DIR}) -find_library(IPP_VM ${IPP_PRE}ippvm${IPP_POST} PATHS ${IPP_ROOT_DIR}) -find_library(IPP_I ${IPP_PRE}ippi${IPP_POST} PATHS ${IPP_ROOT_DIR}) +find_library(IPP_CORE ${IPP_PRE}ippcore${IPP_POST} PATHS ${IPP_ROOT_DIR} $ENV{CONDA_PREFIX} PATH_SUFFIXES lib Library/lib) +find_library(IPP_S ${IPP_PRE}ipps${IPP_POST} PATHS ${IPP_ROOT_DIR} $ENV{CONDA_PREFIX} PATH_SUFFIXES lib Library/lib) +find_library(IPP_VM ${IPP_PRE}ippvm${IPP_POST} PATHS ${IPP_ROOT_DIR} $ENV{CONDA_PREFIX} PATH_SUFFIXES lib Library/lib) +find_library(IPP_I ${IPP_PRE}ippi${IPP_POST} PATHS ${IPP_ROOT_DIR} $ENV{CONDA_PREFIX} PATH_SUFFIXES lib Library/lib) if(IPP_INCLUDE_DIR AND IPP_CORE AND IPP_S AND IPP_VM AND IPP_I) message(STATUS "IPP found in: ${IPP_ROOT_DIR}") set(IPP_FOUND TRUE) set(IPP_INCLUDE_DIRS "${IPP_INCLUDE_DIR}") set(IPP_LIBRARIES "${IPP_CORE};${IPP_S};${IPP_VM};${IPP_I}") + message(STATUS "IPP libraries: ${IPP_LIBRARIES}") + message(STATUS "IPP include dirs: ${IPP_INCLUDE_DIRS}") else() message(STATUS "IPP not found") endif()