Skip to content

Commit d1845d5

Browse files
ammarcsjammarcsj
authored
Merge pull request #143 from MannLabs/fix-installer-packaging-downloads
Fix release installer packaging
2 parents c51c146 + fd26801 commit d1845d5

6 files changed

Lines changed: 146 additions & 20 deletions

File tree

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ wheels/
4343
*.manifest
4444
#*.spec
4545
build_pyinstaller
46+
build_pyinstaller_data
4647
dist_pyinstaller
4748

4849
# Installer logs

‎release/common/download_installer_data.sh‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,98 @@
1+
#!/bin/bash
2+
set -e -u
3+
4+
INSTALLER_DATA_DIR=${INSTALLER_DATA_DIR:-build_pyinstaller_data}
5+
ALPHAMAP_DATA_DIR="${INSTALLER_DATA_DIR}/alphamap/data"
6+
ALPHAQUANT_RESOURCES_DIR="${INSTALLER_DATA_DIR}/alphaquant/resources"
7+
8+
curl_with_retries() {
9+
local args=(
10+
-L
11+
-f
12+
--retry 3
13+
--retry-delay 2
14+
-H "User-Agent: alphaquant-release-build"
15+
)
16+
17+
curl "${args[@]}" "$@"
18+
}
19+
20+
list_alphamap_data_downloads() {
21+
python -c 'from urllib.parse import quote
22+
from alphamap.organisms_data import all_organisms
23+
base_url = "https://raw.githubusercontent.com/MannLabs/alphamap/main/alphamap/data/"
24+
seen = set()
25+
for organism in all_organisms.values():
26+
for key in ("fasta_name", "uniprot_name"):
27+
name = organism[key]
28+
if name not in seen:
29+
seen.add(name)
30+
print(f"{name}\t{base_url}{quote(name)}")'
31+
}
32+
33+
download_datashare_zip() {
34+
local url=$1
35+
local label=$2
36+
local temp_zip
37+
38+
temp_zip=$(mktemp)
39+
echo "Downloading ${label} from datashare..."
40+
if ! curl_with_retries "${url}" -o "${temp_zip}"; then
41+
echo "Error: Failed to download ${label} from datashare"
42+
rm -f "${temp_zip}"
43+
exit 1
44+
fi
45+
46+
echo "Extracting ${label}..."
47+
unzip -o "${temp_zip}" -d "${ALPHAQUANT_RESOURCES_DIR}"
48+
rm -f "${temp_zip}"
49+
}
50+
51+
rm -rf "${INSTALLER_DATA_DIR}"
52+
mkdir -p "${ALPHAMAP_DATA_DIR}" "${ALPHAQUANT_RESOURCES_DIR}"
53+
54+
echo "Downloading AlphaMap FASTA and CSV files..."
55+
list_alphamap_data_downloads | \
56+
while IFS=$'\t' read -r filename url; do
57+
if [ -z "$filename" ] || [ -z "$url" ]; then
58+
echo "Warning: Empty URL detected, skipping..."
59+
continue
60+
fi
61+
62+
echo "Downloading ${filename}..."
63+
if ! curl_with_retries "$url" -o "${ALPHAMAP_DATA_DIR}/${filename}"; then
64+
echo "Error: Failed to download ${filename}"
65+
exit 1
66+
fi
67+
done
68+
69+
alphamap_file_count=$(find "${ALPHAMAP_DATA_DIR}" -type f \( -name "*.fasta" -o -name "*.csv" \) | wc -l)
70+
echo "Downloaded ${alphamap_file_count} AlphaMap files"
71+
if [ "${alphamap_file_count}" -eq 0 ]; then
72+
echo "Error: No AlphaMap files were downloaded"
73+
exit 1
74+
fi
75+
76+
download_datashare_zip "https://datashare.biochem.mpg.de/s/ezPzeqStEgDD8gg/download" "reference databases"
77+
download_datashare_zip "https://datashare.biochem.mpg.de/s/stH9pmNe6O9CRHG/download" "phosphopred databases"
78+
79+
REFERENCE_DB_DIR="${ALPHAQUANT_RESOURCES_DIR}/reference_databases"
80+
PHOSPHOPRED_DB_DIR="${ALPHAQUANT_RESOURCES_DIR}/phosphopred_databases"
81+
HUMAN_PHOSPHO_FILE="${PHOSPHOPRED_DB_DIR}/human_uniprot_reviewed_phos_prob.tsv"
82+
83+
if [ ! -d "${REFERENCE_DB_DIR}" ]; then
84+
echo "Error: reference_databases directory not found at ${REFERENCE_DB_DIR}"
85+
exit 1
86+
fi
87+
88+
if [ ! -d "${PHOSPHOPRED_DB_DIR}" ]; then
89+
echo "Error: phosphopred_databases directory not found at ${PHOSPHOPRED_DB_DIR}"
90+
exit 1
91+
fi
92+
93+
if [ ! -f "${HUMAN_PHOSPHO_FILE}" ]; then
94+
echo "Error: Expected phosphopred database file not found at ${HUMAN_PHOSPHO_FILE}"
95+
exit 1
96+
fi
97+
98+
echo "Installer data staged in ${INSTALLER_DATA_DIR}"

‎release/linux/build_installer_linux.sh‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,12 +3,13 @@ set -e -u
33

44
# Build the installer for Linux.
55
# This script must be run from the root of the repository.
6-
rm -rf dist_pyinstaller build_pyinstaller
6+
rm -rf dist_pyinstaller build_pyinstaller build_pyinstaller_data
77

88
# Find the wheel file in dist directory
99
WHL_NAME=$(cd dist && ls ./*.whl && cd ..)
1010
pip install "dist/${WHL_NAME}[stable,gui-stable,dask-stable]"
1111

12+
INSTALLER_DATA_DIR=build_pyinstaller_data release/common/download_installer_data.sh
13+
1214
# Creating the stand-alone pyinstaller folder
1315
pyinstaller release/pyinstaller/alphaquant.spec --distpath dist_pyinstaller --workpath build_pyinstaller -y
14-

‎release/linux/build_package_linux.sh‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,6 @@ rm -rf ${BUILD_NAME}.deb
1313
mkdir -p dist_pyinstaller/${BUILD_NAME}/usr/local/bin
1414
mv dist_pyinstaller/${PACKAGE_NAME} dist_pyinstaller/${BUILD_NAME}/usr/local/bin/${PACKAGE_NAME}
1515

16-
#make directory for AlphaMap. This is where AlphaMap stores downloaded data, such as fasta files
17-
mkdir -p dist_pyinstaller/${BUILD_NAME}/usr/local/bin/${PACKAGE_NAME}/alphamap/data/
18-
1916
mkdir dist_pyinstaller/${BUILD_NAME}/DEBIAN
2017
cp release/linux/control dist_pyinstaller/${BUILD_NAME}/DEBIAN
2118
dpkg-deb --build --root-owner-group dist_pyinstaller/${BUILD_NAME}

‎release/macos/build_package_macos.sh‎

Lines changed: 31 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,31 @@ PACKAGE_NAME=alphaquant
88
# BUILD_NAME is taken from environment variables, e.g. alphaquant-1.2.3-macos-darwin-arm64 or alphaquant-1.2.3-macos-darwin-x64
99
rm -rf ${BUILD_NAME}.pkg
1010

11+
curl_with_retries() {
12+
local args=(
13+
-L
14+
-f
15+
--retry 3
16+
--retry-delay 2
17+
-H "User-Agent: alphaquant-release-build"
18+
)
19+
20+
curl "${args[@]}" "$@"
21+
}
22+
23+
list_alphamap_data_downloads() {
24+
python -c 'from urllib.parse import quote
25+
from alphamap.organisms_data import all_organisms
26+
base_url = "https://raw.githubusercontent.com/MannLabs/alphamap/main/alphamap/data/"
27+
seen = set()
28+
for organism in all_organisms.values():
29+
for key in ("fasta_name", "uniprot_name"):
30+
name = organism[key]
31+
if name not in seen:
32+
seen.add(name)
33+
print(f"{name}\t{base_url}{quote(name)}")'
34+
}
35+
1136
# If needed, include additional source such as e.g.:
1237
# cp ../../alphaquant/data/*.fasta dist/alphaquant/data
1338

@@ -36,23 +61,14 @@ mkdir -p ${CONTENTS_FOLDER}/Frameworks/alphamap/data/
3661
####
3762
####Download all AlphaMap FASTA and CSV files from GitHub, which are needed for the further analyses. There is a lot of error checking to ensure that the files get actually added during the build
3863
echo "Starting downloads of FASTA and CSV files..."
39-
DOWNLOAD_LIST=$(curl -L -f https://api.github.com/repos/MannLabs/alphamap/contents/alphamap/data?ref=main)
40-
if [ $? -ne 0 ]; then
41-
echo "Error: Failed to fetch file list from GitHub API"
42-
exit 1
43-
fi
44-
45-
echo "$DOWNLOAD_LIST" | \
46-
grep "\"download_url\".*\.\(fasta\|csv\)\"" | \
47-
cut -d '"' -f 4 | \
48-
while read url; do
49-
if [ -z "$url" ]; then
64+
list_alphamap_data_downloads | \
65+
while IFS=$'\t' read -r filename url; do
66+
if [ -z "$filename" ] || [ -z "$url" ]; then
5067
echo "Warning: Empty URL detected, skipping..."
5168
continue
5269
fi
53-
filename=$(basename $url)
5470
echo "Downloading $filename..."
55-
if ! curl -L -f "$url" -o "${CONTENTS_FOLDER}/Frameworks/alphamap/data/$filename"; then
71+
if ! curl_with_retries "$url" -o "${CONTENTS_FOLDER}/Frameworks/alphamap/data/$filename"; then
5672
echo "Error: Failed to download $filename"
5773
exit 1
5874
fi
@@ -76,7 +92,7 @@ mkdir -p ${CONTENTS_FOLDER}/MacOS/_internal/alphaquant/resources/
7692
# Download and extract the first zip file
7793
echo "Downloading and extracting first resource from datashare..."
7894
TEMP_ZIP1=$(mktemp)
79-
if ! curl -L -f "https://datashare.biochem.mpg.de/s/ezPzeqStEgDD8gg/download" -o "$TEMP_ZIP1"; then
95+
if ! curl_with_retries "https://datashare.biochem.mpg.de/s/ezPzeqStEgDD8gg/download" -o "$TEMP_ZIP1"; then
8096
echo "Error: Failed to download first resource from datashare"
8197
exit 1
8298
fi
@@ -87,7 +103,7 @@ rm "$TEMP_ZIP1"
87103
# Download and extract the second zip file
88104
echo "Downloading and extracting second resource from datashare..."
89105
TEMP_ZIP2=$(mktemp)
90-
if ! curl -L -f "https://datashare.biochem.mpg.de/s/stH9pmNe6O9CRHG/download" -o "$TEMP_ZIP2"; then
106+
if ! curl_with_retries "https://datashare.biochem.mpg.de/s/stH9pmNe6O9CRHG/download" -o "$TEMP_ZIP2"; then
91107
echo "Error: Failed to download second resource from datashare"
92108
exit 1
93109
fi

‎release/pyinstaller/alphaquant.spec‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
import os
44
import sys
5+
from pathlib import Path
56
from PyInstaller.building.build_main import Analysis, PYZ, EXE, COLLECT, BUNDLE, TOC
67
import PyInstaller.utils.hooks
78

@@ -43,6 +44,18 @@ hidden_imports = [h for h in hidden_imports if "__pycache__" not in h]
4344
# )
4445
datas = [d for d in datas if ("__pycache__" not in d[0]) and (d[1] not in [".", "Resources", "scripts"])]
4546

47+
installer_data_dir = Path(os.environ.get("INSTALLER_DATA_DIR", "build_pyinstaller_data"))
48+
if not installer_data_dir.is_absolute():
49+
installer_data_dir = Path(location) / installer_data_dir
50+
for source_root, destination_root in (
51+
(installer_data_dir / "alphamap" / "data", Path("alphamap") / "data"),
52+
(installer_data_dir / "alphaquant" / "resources", Path("alphaquant") / "resources"),
53+
):
54+
if source_root.exists():
55+
for source_file in source_root.rglob("*"):
56+
if source_file.is_file():
57+
datas.append((str(source_file), str(destination_root / source_file.relative_to(source_root).parent)))
58+
4659
# add certifi to datas, otherwise ssh connections fail when they are triggered from the installer, because the certificates are not available
4760
# In the case of the AlphaQuant repo, AlphaMap needs to download data from GitHub and this fails without certifi
4861
datas.extend(PyInstaller.utils.hooks.collect_data_files('certifi'))

0 commit comments

Comments
 (0)