Skip to content

Commit 72f6005

Browse files
sylveonsylveon
authored
Update windows-deps to 2.7.0, fix Windows issues (#2018)
Windows: - Fix OpenCV Python extension build by reverting to Python 3.12.9 - Build gdal in vcpkg instead of relying on prebuilt wheels - Build Fiona & rasterio wheels from source by linking to gdal from vcpkg
1 parent d597947 commit 72f6005

13 files changed

Lines changed: 112 additions & 83 deletions

‎.github/workflows/publish-windows.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ jobs:
2121
- name: Setup Python
2222
uses: actions/setup-python@v2
2323
with:
24-
python-version: '3.12.10'
24+
python-version: '3.12.9'
2525
architecture: 'x64'
2626
- uses: Jimver/cuda-toolkit@v0.2.24
2727
id: cuda-toolkit

‎.github/workflows/test-build-prs.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,7 @@ jobs:
5656
- name: Setup Python
5757
uses: actions/setup-python@v2
5858
with:
59-
python-version: '3.12.10'
59+
python-version: '3.12.9'
6060
architecture: 'x64'
6161
- uses: Jimver/cuda-toolkit@v0.2.24
6262
id: cuda-toolkit

‎SuperBuild/CMakeLists.txt‎

Lines changed: 38 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -85,41 +85,43 @@ mark_as_advanced(SB_BINARY_DIR)
8585
message(STATUS "SuperBuild binary files will be located to: ${SB_BINARY_DIR}")
8686

8787
if (WIN32)
88-
if (NOT DEFINED CMAKE_TOOLCHAIN_FILE)
89-
message(FATAL_ERROR "CMAKE_TOOLCHAIN_FILE not set. You need to set it to the path of vcpkg.cmake")
90-
endif()
91-
get_filename_component(CMAKE_TOOLCHAIN_DIR ${CMAKE_TOOLCHAIN_FILE} DIRECTORY)
92-
get_filename_component(VCPKG_ROOT "${CMAKE_TOOLCHAIN_DIR}/../../" ABSOLUTE)
93-
set(WIN32_CMAKE_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}")
94-
set(PYTHON_HOME "${SB_ROOT_DIR}/../venv")
95-
set(PYTHON_EXE_PATH "${PYTHON_HOME}/Scripts/python")
96-
97-
# Use the GDAL version that comes with pip
98-
set(GDAL_ROOT "${PYTHON_HOME}/Lib/site-packages/osgeo")
99-
set(GDAL_LIBRARY "${GDAL_ROOT}/gdal.lib")
100-
set(GDAL_INCLUDE_DIR "${GDAL_ROOT}/include/gdal")
101-
102-
message("Copying VCPKG DLLs...")
103-
file(GLOB COPY_DLLS "${VCPKG_ROOT}/installed/x64-windows/bin/*.dll")
104-
file(COPY ${COPY_DLLS} DESTINATION "${SB_INSTALL_DIR}/bin")
105-
106-
message("Copying CUDA DLLs...")
107-
file(GLOB CUDA_DLLS "$ENV{CUDA_PATH}/bin/cudart64*.dll")
108-
file(COPY ${CUDA_DLLS} DESTINATION "${SB_INSTALL_DIR}/bin")
109-
110-
set(WIN32_GDAL_ARGS -DGDAL_FOUND=TRUE -DGDAL_LIBRARY=${GDAL_LIBRARY} -DGDAL_INCLUDE_DIR=${GDAL_INCLUDE_DIR})
88+
if (NOT DEFINED CMAKE_TOOLCHAIN_FILE)
89+
message(FATAL_ERROR "CMAKE_TOOLCHAIN_FILE not set. You need to set it to the path of vcpkg.cmake")
90+
endif()
91+
get_filename_component(CMAKE_TOOLCHAIN_DIR ${CMAKE_TOOLCHAIN_FILE} DIRECTORY)
92+
get_filename_component(VCPKG_ROOT "${CMAKE_TOOLCHAIN_DIR}/../../" ABSOLUTE)
93+
set(WIN32_CMAKE_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}")
94+
set(PYTHON_HOME "${SB_ROOT_DIR}/../venv")
95+
set(PYTHON_EXE_PATH "${PYTHON_HOME}/Scripts/python")
96+
97+
message("Copying VCPKG DLLs...")
98+
file(GLOB COPY_DLLS "${VCPKG_ROOT}/installed/x64-windows/bin/*.dll")
99+
file(COPY ${COPY_DLLS} DESTINATION "${SB_INSTALL_DIR}/bin")
100+
101+
message("Copying CUDA DLLs...")
102+
file(GLOB CUDA_DLLS "$ENV{CUDA_PATH}/bin/cudart64*.dll")
103+
file(COPY ${CUDA_DLLS} DESTINATION "${SB_INSTALL_DIR}/bin")
104+
105+
message("Copying GDAL tools...")
106+
file(GLOB GDAL_TOOLS "${VCPKG_ROOT}/installed/x64-windows/tools/gdal/*")
107+
file(COPY ${GDAL_TOOLS} DESTINATION "${SB_INSTALL_DIR}/bin")
108+
109+
message("Copying GDAL data...")
110+
file(GLOB GDAL_DATA "${VCPKG_ROOT}/installed/x64-windows/share/gdal/*")
111+
file(COPY ${GDAL_DATA} DESTINATION "${SB_INSTALL_DIR}/bin/data/gdal")
112+
113+
message("Copying PROJ data...")
114+
file(GLOB PROJ_DATA "${VCPKG_ROOT}/installed/x64-windows/share/proj/*")
115+
file(COPY ${PROJ_DATA} DESTINATION "${SB_INSTALL_DIR}/bin/data/proj")
116+
117+
# GDAL comes from vcpkg, create a dummy target to satisfy dependencies
118+
add_custom_target(gdal)
111119
elseif(APPLE)
112120
set(PYTHON_HOME "${SB_ROOT_DIR}/../venv")
113121
set(PYTHON_EXE_PATH "${PYTHON_HOME}/bin/python")
114-
set(GDAL_ROOT "${SB_INSTALL_DIR}")
115-
set(GDAL_LIBRARY "${GDAL_ROOT}/lib/libgdal.dylib")
116-
set(GDAL_INCLUDE_DIR "${GDAL_ROOT}/include")
117122
else()
118123
set(PYTHON_HOME "${SB_ROOT_DIR}/../venv")
119124
set(PYTHON_EXE_PATH "${PYTHON_HOME}/bin/python3")
120-
set(GDAL_ROOT "${SB_INSTALL_DIR}")
121-
set(GDAL_LIBRARY "${GDAL_ROOT}/lib/libgdal.so")
122-
set(GDAL_INCLUDE_DIR "${GDAL_ROOT}/include")
123125
endif()
124126

125127
# Path to additional CMake modules
@@ -136,14 +138,9 @@ include(ExternalProject-Setup)
136138
# Geospatial Data Abstraction Library (GDAL)
137139
#
138140
set(ODM_GDAL_Version 3.11.1)
139-
if(WIN32)
140-
message(STATUS "GDAL: Using pip-installed version from ${GDAL_ROOT}")
141-
# Create a dummy target to satisfy dependencies
142-
add_custom_target(gdal)
143-
else()
144-
option(ODM_BUILD_GDAL "Force to build GDAL library" ON)
145-
SETUP_EXTERNAL_PROJECT(GDAL ${ODM_GDAL_Version} ${ODM_BUILD_GDAL})
146-
endif()
141+
option(ODM_BUILD_GDAL "Force to build GDAL library" OFF)
142+
143+
SETUP_EXTERNAL_PROJECT(GDAL ${ODM_GDAL_Version} ${ODM_BUILD_GDAL})
147144

148145
# ---------------------------------------------------------------------------------------------
149146
# Open Source Computer Vision (OpenCV)
@@ -243,7 +240,7 @@ externalproject_add(dem2mesh
243240
PREFIX ${SB_BINARY_DIR}/dem2mesh
244241
SOURCE_DIR ${SB_SOURCE_DIR}/dem2mesh
245242
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
246-
${WIN32_GDAL_ARGS}
243+
${WIN32_CMAKE_ARGS}
247244
${APPLE_CMAKE_ARGS}
248245
)
249246

@@ -254,7 +251,7 @@ externalproject_add(dem2points
254251
PREFIX ${SB_BINARY_DIR}/dem2points
255252
SOURCE_DIR ${SB_SOURCE_DIR}/dem2points
256253
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
257-
${WIN32_GDAL_ARGS}
254+
${WIN32_CMAKE_ARGS}
258255
${APPLE_CMAKE_ARGS}
259256
)
260257

@@ -265,7 +262,7 @@ externalproject_add(odm_orthophoto
265262
PREFIX ${SB_BINARY_DIR}/odm_orthophoto
266263
SOURCE_DIR ${SB_SOURCE_DIR}/odm_orthophoto
267264
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
268-
${WIN32_CMAKE_ARGS} ${WIN32_GDAL_ARGS}
265+
${WIN32_CMAKE_ARGS}
269266
)
270267

271268
externalproject_add(fastrasterfilter
@@ -275,7 +272,7 @@ externalproject_add(fastrasterfilter
275272
PREFIX ${SB_BINARY_DIR}/fastrasterfilter
276273
SOURCE_DIR ${SB_SOURCE_DIR}/fastrasterfilter
277274
CMAKE_ARGS -DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
278-
${WIN32_CMAKE_ARGS} ${WIN32_GDAL_ARGS}
275+
${WIN32_CMAKE_ARGS}
279276
)
280277

281278
externalproject_add(lastools

‎SuperBuild/cmake/External-GDAL.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ ExternalProject_Add(${_proj_name}
2323
BINARY_DIR ${_SB_BINARY_DIR}
2424
#--Install step---------------
2525
INSTALL_DIR ${SB_INSTALL_DIR}
26-
INSTALL_COMMAND "${CMAKE_COMMAND}" --build . --target install
26+
INSTALL_COMMAND "${CMAKE_COMMAND}" --build . --config $<CONFIG> --target install
2727
#--Output logging-------------
2828
LOG_DOWNLOAD OFF
2929
LOG_CONFIGURE OFF

‎SuperBuild/cmake/External-Hexer.cmake‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,14 @@ ExternalProject_Add(${_proj_name}
88
STAMP_DIR ${_SB_BINARY_DIR}/stamp
99
#--Download step--------------
1010
DOWNLOAD_DIR ${SB_DOWNLOAD_DIR}
11-
URL https://github.com/hobuinc/hexer/archive/5876a5ab1d5b504cf1ea985d66ae359287eef31e.tar.gz
11+
URL https://github.com/hobuinc/hexer/archive/bc226544ca941b6754c343d0e3b42808a3278dec.tar.gz
1212
#--Update/Patch step----------
1313
UPDATE_COMMAND ""
1414
#--Configure step-------------
1515
SOURCE_DIR ${SB_SOURCE_DIR}/${_proj_name}
1616
CMAKE_ARGS
1717
-DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
18-
${WIN32_GDAL_ARGS}
18+
${WIN32_CMAKE_ARGS}
1919
#--Build step-----------------
2020
BINARY_DIR ${_SB_BINARY_DIR}
2121
#--Install step---------------

‎SuperBuild/cmake/External-PDAL.cmake‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,6 @@ ExternalProject_Add(${_proj_name}
5252
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
5353
-DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
5454
${WIN32_CMAKE_ARGS}
55-
${WIN32_GDAL_ARGS}
5655
#--Build step-----------------
5756
BINARY_DIR ${_SB_BINARY_DIR}
5857
#--Install step---------------

‎SuperBuild/cmake/External-PDALPython.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ ExternalProject_Add(${_proj_name}
3333
BINARY_DIR ${_SB_BINARY_DIR}
3434
#--Install step---------------
3535
INSTALL_DIR ${SB_INSTALL_DIR}
36-
INSTALL_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --target install
36+
INSTALL_COMMAND ${CMAKE_COMMAND} --build <BINARY_DIR> --config $<CONFIG> --target install
3737
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SB_SOURCE_DIR}/${_proj_name}/src/pdal/__init__.py ${SB_INSTALL_DIR}/lib/python3.12/dist-packages/pdal
3838
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SB_SOURCE_DIR}/${_proj_name}/src/pdal/pipeline.py ${SB_INSTALL_DIR}/lib/python3.12/dist-packages/pdal
3939
COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SB_SOURCE_DIR}/${_proj_name}/src/pdal/drivers.py ${SB_INSTALL_DIR}/lib/python3.12/dist-packages/pdal

‎SuperBuild/cmake/External-Untwine.cmake‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ ExternalProject_Add(${_proj_name}
1616
SOURCE_DIR ${SB_SOURCE_DIR}/${_proj_name}
1717
CMAKE_ARGS
1818
-DPDAL_DIR=${SB_INSTALL_DIR}/lib/cmake/PDAL
19-
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
19+
-DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE}
2020
-DCMAKE_INSTALL_PREFIX:PATH=${SB_INSTALL_DIR}
2121
#--Build step-----------------
2222
BINARY_DIR ${_SB_BINARY_DIR}

‎configure.py‎

Lines changed: 43 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,10 @@
1111
import os
1212
import stat
1313
import urllib.request
14+
from urllib.parse import urlparse
1415
import shutil
1516
import zipfile
17+
import tarfile
1618

1719
from venv import EnvBuilder
1820

@@ -26,7 +28,7 @@
2628
help='Build VCPKG environment from scratch instead of downloading prebuilt one.')
2729
parser.add_argument('--vcpkg-archive-url',
2830
type=str,
29-
default='https://github.com/OpenDroneMap/windows-deps/releases/download/2.6.0/vcpkg-export.zip',
31+
default='https://github.com/OpenDroneMap/windows-deps/releases/download/2.7.0/vcpkg-export.zip',
3032
required=False,
3133
help='Path to VCPKG export archive')
3234
parser.add_argument('--signtool-path',
@@ -47,8 +49,7 @@
4749

4850
args = parser.parse_args()
4951

50-
def run(cmd, cwd=os.getcwd()):
51-
env = os.environ.copy()
52+
def run(cmd, cwd=os.getcwd(), env=os.environ.copy()):
5253
print(cmd)
5354
p = subprocess.Popen(cmd, shell=True, env=env, cwd=cwd)
5455
retcode = p.wait()
@@ -71,15 +72,42 @@ def vcpkg_requirements():
7172
pckgs = list(filter(lambda l: len(l) > 0, map(str.strip, f.read().split("\n"))))
7273
return pckgs
7374

75+
def install_python_package_from_source(url, vcpkg, extractor, get_top_dir):
76+
filename = os.path.basename(urlparse(url).path)
77+
print("Downloading %s --> %s" % (url, filename))
78+
with urllib.request.urlopen(url) as response, open(os.path.join("SuperBuild", "download", filename), 'wb') as out_file:
79+
shutil.copyfileobj(response, out_file)
80+
81+
print(f"Extracting {filename}", end="")
82+
top_dir = None
83+
with extractor(os.path.join("SuperBuild", "download", filename)) as z:
84+
top_dir = get_top_dir(z)
85+
z.extractall(os.path.join("SuperBuild", "src"))
86+
87+
print(f" --> {top_dir}")
88+
src_dir = os.path.join("SuperBuild", "src", top_dir)
89+
90+
with open(os.path.join(src_dir, "setup.cfg"), "w") as file:
91+
file.write("[build_ext]\n")
92+
file.write("include-dirs = %s\n" % os.path.abspath(os.path.join(vcpkg, "include")))
93+
file.write("libraries = gdal\n")
94+
file.write("library-dirs = %s\n" % os.path.abspath(os.path.join(vcpkg, "lib")))
95+
96+
pip_abs = os.path.abspath(os.path.join("venv", "Scripts", "pip.exe"))
97+
98+
run(f"\"{pip_abs}\" install .", cwd=src_dir, env={**os.environ, "GDAL_VERSION": "3.11.1"})
99+
74100
def build():
75101
# Create python virtual env
76102
if not os.path.isdir("venv"):
77103
print("Creating virtual env --> venv/")
78104
ebuilder = EnvBuilder(with_pip=True)
79105
ebuilder.create("venv")
80106

81-
run("pip install setuptools")
82-
run("venv\\Scripts\\pip install --ignore-installed -r requirements.txt")
107+
run("venv\\Scripts\\pip install setuptools")
108+
109+
# Install numpy for OpenCV build. TODO: read the exact version from requirements.txt?
110+
run("venv\\Scripts\\pip install numpy==2.3.2")
83111

84112
# Download / build VCPKG environment
85113
if not os.path.isdir("vcpkg"):
@@ -114,9 +142,17 @@ def build():
114142
os.mkdir(build_dir)
115143

116144
toolchain_file = os.path.join(os.getcwd(), "vcpkg", "scripts", "buildsystems", "vcpkg.cmake")
117-
run("cmake .. -DCMAKE_TOOLCHAIN_FILE=\"%s\"" % toolchain_file, cwd=build_dir)
145+
run("cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_TOOLCHAIN_FILE=\"%s\"" % toolchain_file, cwd=build_dir)
118146
run("cmake --build . --config Release -j2", cwd=build_dir)
119147

148+
# Build GDAL bindings, fiona and rasterio from source using vcpkg GDAL. TODO: read the exact versions from requirements.txt?
149+
vcpkg_installed = os.path.join(os.getcwd(), "vcpkg", "installed", "x64-windows")
150+
install_python_package_from_source("https://files.pythonhosted.org/packages/source/g/gdal/gdal-3.11.1.tar.gz", vcpkg_installed, lambda path: tarfile.open(path, "r:gz"), lambda t: t.getnames()[0])
151+
install_python_package_from_source("https://github.com/Toblerity/Fiona/archive/refs/tags/1.10.1.zip", vcpkg_installed, lambda path: zipfile.ZipFile(path), lambda z: z.namelist()[0])
152+
install_python_package_from_source("https://github.com/rasterio/rasterio/archive/refs/tags/1.4.3.zip", vcpkg_installed, lambda path: zipfile.ZipFile(path), lambda z: z.namelist()[0])
153+
154+
run("venv\\Scripts\\pip install -r requirements.txt")
155+
120156
def vcpkg_export():
121157
if not os.path.exists("vcpkg"):
122158
print("vcpkg directory does not exist. Did you build the environment?")
@@ -168,7 +204,7 @@ def dist():
168204
# Download portable python
169205
if not os.path.isdir("python312"):
170206
pythonzip_path = os.path.join("SuperBuild", "download", "python312.zip")
171-
python_url = "https://github.com/OpenDroneMap/windows-deps/releases/download/2.6.0/python-3.12.10-embed-amd64-less-pth.zip"
207+
python_url = "https://github.com/OpenDroneMap/windows-deps/releases/download/2.7.0/python-3.12.9-embed-amd64-less-pth-sqlite.zip"
172208
if not os.path.exists(pythonzip_path):
173209
print("Downloading %s" % python_url)
174210
with urllib.request.urlopen(python_url) as response, open( pythonzip_path, 'wb') as out_file:

‎opendm/cutline.py‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,6 @@
1616
from shapely.geometry import LineString, mapping, shape
1717
from shapely.ops import polygonize, unary_union
1818

19-
if sys.platform == 'win32':
20-
# Temporary fix for: ValueError: GEOSGeom_createLinearRing_r returned a NULL pointer
21-
# https://github.com/Toblerity/Shapely/issues/1005
22-
shapely.speedups.disable()
23-
2419
def write_raster(data, file):
2520
profile = {
2621
'driver': 'GTiff',

0 commit comments

Comments
 (0)