Merge pull request #456 from hjmjohnson/fix/ci-itk-pin-and-deps

COMP: Build against ITK main; repair ExternalData, baselines, and docs CI

Author: hjmjohnson

.github/workflows/build-test-publish.yml

@@ -264,7 +264,7 @@ jobs:
         run: |
             python -m pip install --upgrade pip
             python -m pip install ninja
-            python -m pip install itk>=5.4rc3
+            python -m pip install itk>=5.4.6
             python -m pip install matplotlib
             python -m pip install itkwidgets
             # Issues with 5.7.0

.pre-commit-config.yaml

@@ -0,0 +1,53 @@
+# Mirrors ITK's .pre-commit-config.yaml style gates for C++ and Python.
+# The clang-format rev must track ITK main's pin (the lint CI job
+# resolves its version from ITK main at run time).
+fail_fast: true
+default_stages: [pre-commit]
+repos:
+- repo: https://github.com/pre-commit/pre-commit-hooks
+  rev: v5.0.0
+  hooks:
+   - id: check-added-large-files
+     args: ['--maxkb=200']
+   - id: check-ast
+   - id: check-case-conflict
+   - id: check-illegal-windows-names
+   - id: check-json
+     exclude: "\\.ipynb$"
+   - id: check-merge-conflict
+   - id: check-toml
+   - id: check-vcs-permalinks
+   - id: check-xml
+   - id: check-yaml
+   - id: check-shebang-scripts-are-executable
+   - id: debug-statements
+     exclude: "^Utilities\\/SphinxExtensions\\/"
+   - id: destroyed-symlinks
+   - id: detect-private-key
+   - id: end-of-file-fixer
+     exclude: "\\.(md5|sha512|cid|svg|vtk|vtp|ipynb)$|^Superbuild\\/"
+   - id: forbid-new-submodules
+   - id: mixed-line-ending
+     exclude: "\\.(sha512|cid|svg|vtk|vtp|ipynb)$"
+   - id: no-commit-to-branch
+     args: ['--branch','main','--branch','dashboard','--branch','gh-pages']
+   - id: trailing-whitespace
+     exclude: "\\.(sha512|cid|svg|vtk|vtp|ipynb)$"
+- repo: https://github.com/pre-commit/mirrors-clang-format
+  rev: v19.1.7
+  hooks:
+  - id: clang-format
+    args: ['--style=file']
+    files: '\.(c|cc|h|cxx|hxx)$'
+- repo: https://github.com/psf/black
+  rev: 25.12.0
+  hooks:
+   - id: black
+     args: ['--target-version', 'py310']
+     exclude: "^Utilities\\/SphinxExtensions\\/"
+- repo: https://github.com/asottile/pyupgrade
+  rev: v3.21.2
+  hooks:
+  - id: pyupgrade
+    args: [--py310-plus]
+    exclude: "^Utilities\\/SphinxExtensions\\/"

CMake/CTestCustom.cmake.in

@@ -37,6 +37,18 @@ set(CTEST_CUSTOM_COVERAGE_EXCLUDE
 set(CTEST_CUSTOM_WARNING_EXCEPTION
   ${CTEST_CUSTOM_WARNING_EXCEPTION}
   "has no symbols"
+  "UseITK\\.cmake"
+  "This warning is for project developers"
+  "DirectWarningToFile"
+  "WITH_MAINTAINER_WARNINGS"
+  "which will suppress this warning"
+  "WARNING: Cache entry deserialization failed, entry ignored"
+  "LabelGeometryImageFilter.*deprecated"
+  "note: declared here"
+  "RemovedInSphinx[0-9]+Warning"
+  # Nightly InsightDoxygen XML is currently missing these classes upstream.
+  "doxygenclass: Cannot find class"
+  "Could not find match for .itk::"
   "ipo: warning #11053"
   "ipo: warning #11053"
   "vxl.core.vnl.algo"
@@ -91,3 +103,8 @@ set(CTEST_CUSTOM_WARNING_EXCEPTION
   "ld.*warning.*duplicate dylib.*"
   )
 endif()
+
+set(CTEST_CUSTOM_ERROR_EXCEPTION
+  ${CTEST_CUSTOM_ERROR_EXCEPTION}
+  "RemovedInSphinx[0-9]+Warning"
+  )

CMake/DownloadDoxygenTAG.cmake

@@ -1,4 +1,3 @@
 file(DOWNLOAD https://github.com/InsightSoftwareConsortium/ITK/releases/download/v5.3.0/InsightDoxygenDocTag-5.3.0.gz
   ${ITKDoxygenTAG_TEMP_DIR}/InsightDoxygen.tag.gz SHOW_PROGRESS
   )
-

CMake/ExternalData.cmake

@@ -999,7 +999,6 @@ function(_ExternalData_get_from_object_store hash algo var_obj var_success)
   foreach(dir ${ExternalData_OBJECT_STORES})
     set(obj "${dir}/${algo}/${hash}")
     if(EXISTS "${obj}")
-      message(STATUS "Found object: \"${obj}\"")
       set("${var_obj}" "${obj}" PARENT_SCOPE)
       set("${var_success}" 1 PARENT_SCOPE)
       return()
@@ -1013,7 +1012,6 @@ function(_ExternalData_download_object name hash algo var_obj var_success var_er
   foreach(dir ${ExternalData_OBJECT_STORES})
     set(obj "${dir}/${algo}/${hash}")
     if(EXISTS "${obj}")
-      message(STATUS "Found object: \"${obj}\"")
       set("${var_obj}" "${obj}" PARENT_SCOPE)
       set("${var_success}" "${success}" PARENT_SCOPE)
       return()

CMake/ITKDoxygen.cmake

@@ -12,4 +12,3 @@ add_custom_command(OUTPUT ${ITKDoxygen_DIR}/index.html
 
 add_custom_target(ITKDoxygen ${CMAKE_COMMAND} -E echo "Finished obtaining Doxygen"
   DEPENDS ${ITKDoxygen_DIR}/index.html)
-

CMake/ITKDoxygenTAG.cmake

@@ -15,4 +15,3 @@ add_custom_command(OUTPUT ${ITKDoxygenTAG_DIR}/InsightDoxygen.tag
   )
 add_custom_target(ITKDoxygenTAG ${CMAKE_COMMAND} -E echo "Finished obtaining Doxygen TAG"
   DEPENDS ${ITKDoxygenTAG_DIR}/InsightDoxygen.tag)
-

CMake/ITKSphinxExamplesExternalData.cmake

@@ -1,3 +1,4 @@
+set(ExternalData_SOURCE_ROOT "${PROJECT_SOURCE_DIR}")
 get_filename_component(_ITKExternalData_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
 include(${_ITKExternalData_DIR}/ExternalData.cmake)
 
@@ -18,52 +19,39 @@ if(NOT ExternalData_OBJECT_STORES)
 endif()
 list(APPEND ExternalData_OBJECT_STORES
   # Local data store populated by the ITK pre-commit hook
-  "${CMAKE_SOURCE_DIR}/.ExternalData"
+  "${PROJECT_SOURCE_DIR}/.ExternalData"
   )
 
-set(ITK_SPHINX_EXAMPLES_DATA_RELEASED_ROOT_CID bafybeib6yijwyl52gkalx6griio2wsibowlrt3v5wmkkuqlnuvku64c3ra)
+# Tests reference staged data at ${CMAKE_CURRENT_BINARY_DIR}/<Example>/<file>,
+# which requires the default ExternalData_BINARY_ROOT (${CMAKE_BINARY_DIR}).
 
+# Expands %(algo:lower)
 set(ExternalData_URL_ALGO_CID_lower cid)
+set(ExternalData_URL_ALGO_MD5_lower md5)
 set(ExternalData_URL_TEMPLATES "" CACHE STRING
   "Additional URL templates for the ExternalData CMake script to look for testing data. E.g.
 file:///var/bigharddrive/%(algo)/%(hash)")
 mark_as_advanced(ExternalData_URL_TEMPLATES)
 if(NOT ITK_FORBID_DOWNLOADS)
   list(APPEND ExternalData_URL_TEMPLATES
+    # Data published on GitHub Pages
+    "https://insightsoftwareconsortium.github.io/ITKTestingData/%(algo)/%(hash)"
     # Local IPFS gateway
     "http://127.0.0.1:8080/ipfs/%(hash)"
-
-    # Data published on GitHub Pages (note: not complete due to file size
-    # limit)
-    "https://insightsoftwareconsortium.github.io/ITKTestingData/%(algo)/%(hash)"
-
-    # Released data rsync'd to Kitware's Apache web server
-    "https://itk.org/files/ExternalData/%(algo)/%(hash)"
-
-    # Restricted gateway with released data
-    "https://itk.mypinata.cloud/ipfs/${ITK_SPHINX_EXAMPLES_DATA_RELEASED_ROOT_CID}/Objects/CID/%(hash)"
-
-    # New data on web3.storage
-    "https://w3s.link/ipfs/%(hash)"
-
-    # Released data on web3.storage
-    "https://${ITK_SPHINX_EXAMPLES_DATA_RELEASED_ROOT_CID}.ipfs.w3s.link/Objects/CID/%(hash)"
-
-    # Released data on estuary.tech
-    "https://api.estuary.tech/gw/ipfs/${ITK_SPHINX_EXAMPLES_DATA_RELEASED_ROOT_CID}/Objects/CID/%(hash)"
-
-    # Protocol Labs gateway
-    "https://${ITK_SPHINX_EXAMPLES_DATA_RELEASED_ROOT_CID}.ipfs.dweb.link/Objects/CID/%(hash)"
-
-    # Gateway for arbitrary new files, uploaded to web3.storage
-    "https://dweb.link/ipfs/%(hash)"
-
     # Protocol Labs gateway
     "https://ipfs.io/ipfs/%(hash)"
-
+    # Filebase gateway
+    "https://ipfs.filebase.io/ipfs/%(hash)"
+    # Gateway for data pinned on pinata
+    "https://gateway.pinata.cloud/ipfs/%(hash)"
     # Cloudflare gateway
     "https://cloudflare-ipfs.com/ipfs/%(hash)"
-
+    # Additional gateway (modern but has redirect)
+    "https://dweb.link/ipfs/%(hash)"
+    # Data published on Girder
+    "https://data.kitware.com:443/api/v1/file/hashsum/%(algo)/%(hash)/download"
+    # Data published by developers using git-gerrit-push.
+    "https://itk.org/files/ExternalData/%(algo)/%(hash)"
     # DataLad repository on gin.g-node.org
     "https://gin.g-node.org/InsightSoftwareConsortium/ITKSphinxExamplesData/raw/main/Objects/CID/%(hash)"
     )
@@ -73,7 +61,7 @@ endif()
 set(ExternalData_LINK_CONTENT CID)
 
 # Emscripten currently has difficulty reading symlinks.
-if(EMSCRIPTEN)
+if(EMSCRIPTEN OR WASI)
   set(ExternalData_NO_SYMLINKS 1)
 endif()
 

CMakeLists.txt

@@ -99,7 +99,7 @@ set(CMAKE_RUNTIME_OUTPUT_DIRECTORY ${ITKSphinxExamples_BINARY_DIR}/bin)
 include(CTest)
 # Tests are added with add_test() can be run with the 'ctest' command line program.
 enable_testing()
-configure_file(CMake/CTestCustom.cmake.in CTestCustom.cmake)
+configure_file(CMake/CTestCustom.cmake.in CTestCustom.cmake COPYONLY)
 
 # ExternalData setup.  We use the CMake ExternalData features to download binary
 # files from a configurable location(s) -- including the local filesystem -- to

Documentation/Build/index.rst

@@ -247,4 +247,3 @@ VV
 .. _Qt:                    https://www.qt.io/developers/
 .. _VTK:                   https://vtk.org/
 .. _VV:                    https://www.creatis.insa-lyon.fr/rio/vv
-