helpers/devops/docker_build/install_python_packages.sh at cb622b78170af3f9204914ffa8efb5ff2c1bd13e · causify-ai/helpers · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
#!/usr/bin/env bash
#
# Install Python packages.
#
# Usage:
#   ./install_python_packages.sh [dependency_groups]
#   dependency_groups: Comma-separated list of dependency groups to install.
#

echo "#############################################################################"
echo "##> $0"
echo "#############################################################################"

set -ex

source utils.sh

echo "# Disk space before $0"
report_disk_usage

echo "PYTHON VERSION="$(python3 --version)
echo "PIP VERSION="$(pip3 --version)
echo "POETRY VERSION="$(poetry --version)

echo "# Installing ${ENV_NAME}"

if [[ 1 == 1 ]]; then
  # Poetry flow.
  echo "# Building environment with poetry"
  # Print config.
  poetry config --list --local
  echo "POETRY_MODE=$POETRY_MODE"
  if [[ $POETRY_MODE == "update" ]]; then
    # Compute dependencies.
    poetry lock -v
    cp poetry.lock /install/poetry.lock.out
  elif [[ $POETRY_MODE == "no_update" ]]; then
    # Reuse the lock file.
    cp /install/poetry.lock.in poetry.lock
  else
    echo "ERROR: Unknown POETRY_MODE=$POETRY_MODE"
    exit 1
  fi;
  # Install with poetry inside a venv.
  echo "# Install with venv + poetry"
  python3 -m ${ENV_NAME} /${ENV_NAME}
  source /${ENV_NAME}/bin/activate
  #pip3 install wheel
  # Install only specified dependency groups.
  # e.g.: `poetry install --no-root --with dev,docs`.
  if [[ -n "$1" ]]; then
    poetry install --no-root --with "$1"
  else
    poetry install --no-root
  fi
  poetry env list
  # Clean up.
  if [[ $CLEAN_UP_INSTALLATION ]]; then
    poetry cache clear --all -q pypi
  else
    echo "WARNING: Skipping clean up installation"
  fi;
  pip freeze 2>&1 >/install/pip_list.txt
  #
  if [[ $CLEAN_UP_INSTALLATION ]]; then
    pip cache purge
  else
    echo "WARNING: Skipping clean up installation"
  fi;
else
  # Conda flow.
  echo "# Building environment with conda"
  update_env () {
    echo "Installing ${ENV_FILE} in ${ENV_NAME}"
    ENV_FILE=${1}
    conda env update -n ${ENV_NAME} --file ${ENV_FILE}
  }

  AMP_CONDA_FILE="devops/docker_build/conda.yml"
  update_env ${AMP_CONDA_FILE}

  if [[ $CLEAN_UP_INSTALLATION ]]; then
    conda clean --all --yes
  else
    echo "WARNING: Skipping clean up installation"
  fi;
fi;

# Custom package installation.
echo "# Checking for custom package installation..."
if [[ -f "./install_custom_packages.sh" ]]; then
    echo "# Found custom installation script, executing..."
    chmod +x "./install_custom_packages.sh"
    source "./install_custom_packages.sh"
else
    echo "# No custom installation script found, skipping"
fi

# Clean up.
if [[ $CLEAN_UP_INSTALLATION ]]; then
  echo "Cleaning up installation..."
  DIRS="/usr/lib/gcc /app/tmp.pypoetry /tmp/*"
  echo "Cleaning up installation... done"
  du -hs $DIRS | sort -h
  rm -rf $DIRS
else
  echo "WARNING: Skipping clean up installation"
fi;

echo "# Disk space before $0"
report_disk_usage