Skip to content

Commit 245fb18

Browse files
mdellwegmdellweg
committed
WIP
1 parent c13c1ea commit 245fb18

2 files changed

Lines changed: 136 additions & 192 deletions

File tree

‎templates/github/.github/workflows/scripts/before_install.sh.j2‎

Lines changed: 123 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -2,42 +2,34 @@
22

33
{% include 'header.j2' %}
44

5-
set -mveuo pipefail
5+
set -euo pipefail
66

77
# make sure this script runs at the repo root
88
cd "$(dirname "$(realpath -e "$0")")"/../../..
99

10-
export PRE_BEFORE_INSTALL=$PWD/.github/workflows/scripts/pre_before_install.sh
11-
export POST_BEFORE_INSTALL=$PWD/.github/workflows/scripts/post_before_install.sh
12-
13-
{% if setup_py -%}
14-
COMPONENT_VERSION=$(sed -ne "s/\s*version.*=.*['\"]\(.*\)['\"][\s,]*/\1/p" setup.py)
15-
{%- else -%}
16-
COMPONENT_VERSION=$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])")
17-
{%- endif %}
10+
if [ -f .github/workflows/scripts/pre_before_install.sh ]; then
11+
source .github/workflows/scripts/pre_before_install.sh
12+
fi
1813

19-
# Compose ansible container definition.
20-
mkdir .ci/ansible/vars || true
21-
echo "---" > .ci/ansible/vars/main.yaml
22-
echo "legacy_component_name: {{ plugin_name | snake }}" >> .ci/ansible/vars/main.yaml
23-
echo "component_name: {{ plugin_app_label }}" >> .ci/ansible/vars/main.yaml
24-
echo "component_version: '${COMPONENT_VERSION}'" >> .ci/ansible/vars/main.yaml
14+
COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
15+
COMPONENT_SOURCE="./{{ plugin_name }}/dist/{{ plugin_name | snake }}-${COMPONENT_VERSION}-py3-none-any.whl"
2516

26-
if [ -f $PRE_BEFORE_INSTALL ]; then
27-
source $PRE_BEFORE_INSTALL
17+
{%- set PULPCORE_PREFIX = "" if plugin_name == "pulpcore" else " pulpcore" %}
18+
{%- if test_s3 %}
19+
if [ "$TEST" = "s3" ]; then
20+
COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[s3]"
2821
fi
29-
30-
if [[ "$TEST" = "s3" ]]; then
31-
for i in {1..3}
32-
do
33-
ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
34-
done
35-
if [[ $s -gt 0 ]]
36-
then
37-
echo "Failed to install amazon.aws"
38-
exit $s
22+
{%- endif %}
23+
{%- if test_azure %}
24+
if [ "$TEST" = "azure" ]; then
25+
COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[azure]"
3926
fi
27+
{%- endif %}
28+
{%- if test_gcp %}
29+
if [ "$TEST" = "gcp" ]; then
30+
COMPONENT_SOURCE="${COMPONENT_SOURCE}{{ PULPCORE_PREFIX }}[google]"
4031
fi
32+
{%- endif %}
4133

4234
if [[ "$TEST" = "pulp" ]]; then
4335
python3 .ci/scripts/calc_constraints.py -u {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > upperbounds_constraints.txt
@@ -46,6 +38,108 @@ if [[ "$TEST" = "lowerbounds" ]]; then
4638
python3 .ci/scripts/calc_constraints.py {% if setup_py -%} requirements.txt {% else -%} pyproject.toml {% endif -%} > lowerbounds_constraints.txt
4739
fi
4840

49-
if [ -f $POST_BEFORE_INSTALL ]; then
50-
source $POST_BEFORE_INSTALL
41+
{%- if test_reroute %}
42+
export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "{{ api_root }}")
43+
{%- else %}
44+
export PULP_API_ROOT="{{ api_root }}"
45+
{%- endif %}
46+
47+
echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
48+
49+
# Compose the scenario definition.
50+
mkdir -p .ci/ansible/vars
51+
52+
cat > .ci/ansible/vars/main.yaml << VARSYAML
53+
---
54+
scenario: "${TEST}"
55+
legacy_component_name: "{{ plugin_name | snake }}"
56+
component_name: "{{ plugin_app_label }}"
57+
component_version: "${COMPONENT_VERSION}"
58+
pulp_env: {{ pulp_env | tojson }}
59+
pulp_settings: {{ pulp_settings | tojson }}
60+
pulp_scheme: "{{ pulp_scheme }}"
61+
{%- if ci_base_image.count(":") %}
62+
pulp_default_container: "{{ ci_base_image }}"
63+
{% else %}
64+
pulp_default_container: "{{ ci_base_image + ":latest" }}"
65+
{% endif -%}
66+
api_root: "${PULP_API_ROOT}"
67+
image:
68+
name: "pulp"
69+
tag: "ci_build"
70+
plugins:
71+
- name: "{{ plugin_name }}"
72+
source: "${COMPONENT_SOURCE}"
73+
ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
74+
upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
75+
lowerounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)
76+
{%- if extra_files %}
77+
extra_files:
78+
{%- for item in extra_files %}
79+
- origin: "{{ item.origin }}"
80+
destination: "{{ item.destination }}"
81+
{%- endfor %}
82+
{%- endif %}
83+
services:
84+
- name: "pulp"
85+
image: "pulp:ci_build"
86+
volumes:
87+
- "./settings:/etc/pulp"
88+
- "./ssh:/keys/"
89+
- "~/.config:/var/lib/pulp/.config"
90+
- "../../../pulp-openapi-generator:/root/pulp-openapi-generator"
91+
env:
92+
PULP_WORKERS: "4"
93+
PULP_HTTPS: "{{ 'true' if pulp_scheme == 'https' else 'false' }}"
94+
{%- if docker_fixtures %}
95+
- name: "pulp-fixtures"
96+
image: "docker.io/pulp/pulp-fixtures:latest"
97+
env:
98+
BASE_URL: "http://pulp-fixtures:8080"
99+
{%- endif %}
100+
VARSYAML
101+
102+
if [ "$TEST" = "s3" ]; then
103+
MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
104+
MINIO_SECRET_KEY=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
105+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
106+
- name: "minio"
107+
image: "minio/minio"
108+
env:
109+
MINIO_ACCESS_KEY: "${MINIO_ACCESS_KEY}"
110+
MINIO_SECRET_KEY: "${MINIO_SECRET_KEY}"
111+
command: "server /data"
112+
minio_access_key: "${MINIO_ACCESS_KEY}"
113+
minio_secret_key: "${MINIO_SECRET_KEY}"
114+
pulp_scenario_settings: {{ pulp_settings_s3 | tojson }}
115+
pulp_scenario_env: {{ pulp_env_s3 | tojson }}
116+
VARSYAML
117+
fi
118+
119+
if [ "$TEST" = "azure" ]; then
120+
cat >> .ci/ansible/vars/main.yaml << VARSYAML
121+
- name: "ci-azurite"
122+
image: "mcr.microsoft.com/azure-storage/azurite"
123+
volumes:
124+
- "./azurite:/etc/pulp\"
125+
command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"
126+
pulp_scenario_settings: {{ pulp_settings_azure | tojson }}
127+
pulp_scenario_env: {{ pulp_env_azure | tojson }}
128+
VARSYAML
129+
fi
130+
131+
if [ "$TEST" = "gcp" ]; then
132+
- name: "ci-gcp"
133+
image: "fsouza/fake-gcs-server"
134+
volumes:
135+
- "storage_data:/etc/pulp"
136+
command: " -scheme http"
137+
pulp_scenario_settings: {{ pulp_settings_gcp | tojson }}
138+
pulp_scenario_env: {{ pulp_env_gcp | tojson }}
139+
fi
140+
141+
cat .ci/ansible/vars/main.yaml
142+
143+
if [ -f .github/workflows/scripts/post_before_install.sh ]; then
144+
source .github/workflows/scripts/post_before_install.sh
51145
fi

‎templates/github/.github/workflows/scripts/install.sh.j2‎

Lines changed: 13 additions & 163 deletions
Original file line numberDiff line numberDiff line change
@@ -10,16 +10,23 @@ REPO_ROOT="$PWD"
1010

1111
source .github/workflows/scripts/utils.sh
1212

13-
PLUGIN_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
14-
PLUGIN_SOURCE="./{{ plugin_name }}/dist/{{ plugin_name | snake }}-${PLUGIN_VERSION}-py3-none-any.whl"
15-
16-
export PULP_API_ROOT="{{ api_root }}"
17-
1813
PIP_REQUIREMENTS=("{{ cli_package }}")
1914

2015
# This must be the **only** call to "pip install" on the test runner.
2116
pip install ${PIP_REQUIREMENTS[*]}
2217

18+
if [[ "$TEST" = "s3" ]]; then
19+
for i in {1..3}
20+
do
21+
ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3
22+
done
23+
if [[ $s -gt 0 ]]
24+
then
25+
echo "Failed to install amazon.aws"
26+
exit $s
27+
fi
28+
fi
29+
2330
{% if test_cli -%}
2431
# Check out the {{ cli_package }} branch matching the installed version.
2532
PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/{{ cli_package }}==//p')"
@@ -28,164 +35,7 @@ git clone --depth 1 --branch "$PULP_CLI_VERSION" {{ cli_repo }} ../{{ cli_packag
2835

2936
cd .ci/ansible/
3037

31-
{%- set PULPCORE_PREFIX = "" if plugin_name == "pulpcore" else " pulpcore" %}
32-
{%- if test_s3 %}
33-
if [ "$TEST" = "s3" ]; then
34-
PLUGIN_SOURCE="${PLUGIN_SOURCE}{{ PULPCORE_PREFIX }}[s3]"
35-
fi
36-
{%- endif %}
37-
{%- if test_azure %}
38-
if [ "$TEST" = "azure" ]; then
39-
PLUGIN_SOURCE="${PLUGIN_SOURCE}{{ PULPCORE_PREFIX }}[azure]"
40-
fi
41-
{%- endif %}
42-
{%- if test_gcp %}
43-
if [ "$TEST" = "gcp" ]; then
44-
PLUGIN_SOURCE="${PLUGIN_SOURCE}{{ PULPCORE_PREFIX }}[google]"
45-
fi
46-
{%- endif %}
47-
48-
cat >> vars/main.yaml << VARSYAML
49-
image:
50-
name: pulp
51-
tag: "ci_build"
52-
plugins:
53-
- name: {{ plugin_name }}
54-
source: "${PLUGIN_SOURCE}"
55-
VARSYAML
56-
if [[ -f ../../ci_requirements.txt ]]; then
57-
cat >> vars/main.yaml << VARSYAML
58-
ci_requirements: true
59-
VARSYAML
60-
fi
61-
if [ "$TEST" = "pulp" ]; then
62-
cat >> vars/main.yaml << VARSYAML
63-
upperbounds: true
64-
VARSYAML
65-
fi
66-
{%- if test_lowerbounds %}
67-
if [ "$TEST" = "lowerbounds" ]; then
68-
cat >> vars/main.yaml << VARSYAML
69-
lowerbounds: true
70-
VARSYAML
71-
fi
72-
{%- endif %}
73-
74-
cat >> vars/main.yaml << VARSYAML
75-
{%- if extra_files %}
76-
extra_files:
77-
{%- for item in extra_files %}
78-
- origin: "{{ item.origin }}"
79-
destination: "{{ item.destination }}"
80-
{%- endfor %}
81-
{%- endif %}
82-
services:
83-
- name: pulp
84-
image: "pulp:ci_build"
85-
volumes:
86-
- ./settings:/etc/pulp
87-
- ./ssh:/keys/
88-
- ~/.config:/var/lib/pulp/.config
89-
- ../../../pulp-openapi-generator:/root/pulp-openapi-generator
90-
env:
91-
PULP_WORKERS: "4"
92-
PULP_HTTPS: "{{ 'true' if pulp_scheme == 'https' else 'false' }}"
93-
VARSYAML
94-
95-
cat >> vars/main.yaml << VARSYAML
96-
pulp_env: {{ pulp_env | tojson }}
97-
pulp_settings: {{ pulp_settings | tojson }}
98-
pulp_scheme: {{ pulp_scheme }}
99-
{%- if ci_base_image.count(":") %}
100-
pulp_default_container: {{ ci_base_image }}
101-
{% else %}
102-
pulp_default_container: {{ ci_base_image + ":latest" }}
103-
{% endif -%}
104-
VARSYAML
105-
106-
{%- if docker_fixtures %}
107-
108-
SCENARIOS=("pulp" "performance" "azure" "gcp" "s3" "generate-bindings" "lowerbounds")
109-
if [[ " ${SCENARIOS[*]} " =~ " ${TEST} " ]]; then
110-
sed -i -e '/^services:/a \
111-
- name: pulp-fixtures\
112-
image: docker.io/pulp/pulp-fixtures:latest\
113-
env: {BASE_URL: "http://pulp-fixtures:8080"}' vars/main.yaml
114-
115-
export REMOTE_FIXTURES_ORIGIN="http://pulp-fixtures:8080"
116-
fi
117-
{%- endif %}
118-
119-
{%- if test_s3 %}
120-
121-
if [ "$TEST" = "s3" ]; then
122-
export MINIO_ACCESS_KEY=AKIAIT2Z5TDYPX3ARJBA
123-
export MINIO_SECRET_KEY=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
124-
sed -i -e '/^services:/a \
125-
- name: minio\
126-
image: minio/minio\
127-
env:\
128-
MINIO_ACCESS_KEY: "'$MINIO_ACCESS_KEY'"\
129-
MINIO_SECRET_KEY: "'$MINIO_SECRET_KEY'"\
130-
command: "server /data"' vars/main.yaml
131-
sed -i -e '$a s3_test: true\
132-
minio_access_key: "'$MINIO_ACCESS_KEY'"\
133-
minio_secret_key: "'$MINIO_SECRET_KEY'"\
134-
pulp_scenario_settings: {{ pulp_settings_s3 | tojson }}\
135-
pulp_scenario_env: {{ pulp_env_s3 | tojson }}\
136-
' vars/main.yaml
137-
138-
{%- if test_reroute %}
139-
export PULP_API_ROOT="/rerouted/djnd/"
140-
{%- endif %}
141-
fi
142-
{%- endif %}
143-
144-
{%- if test_azure %}
145-
146-
if [ "$TEST" = "azure" ]; then
147-
sed -i -e '/^services:/a \
148-
- name: ci-azurite\
149-
image: mcr.microsoft.com/azure-storage/azurite\
150-
volumes:\
151-
- ./azurite:/etc/pulp\
152-
command: "azurite-blob --skipApiVersionCheck --blobHost 0.0.0.0"' vars/main.yaml
153-
sed -i -e '$a azure_test: true\
154-
pulp_scenario_settings: {{ pulp_settings_azure | tojson }}\
155-
pulp_scenario_env: {{ pulp_env_azure | tojson }}\
156-
' vars/main.yaml
157-
fi
158-
{%- endif %}
159-
160-
{%- if test_gcp %}
161-
162-
if [ "$TEST" = "gcp" ]; then
163-
mkdir -p cigcp
164-
cd cigcp
165-
openssl req -newkey rsa:2048 -x509 -nodes -keyout gcpkey.pem -new -out gcpcert.pem -sha256 -days 365 -addext "subjectAltName=DNS:ci-gcp" -subj "/C=CO/ST=ST/L=LO/O=OR/OU=OU/CN=CN"
166-
sudo cp gcpcert.pem /usr/local/share/ca-certificates/gcpcert.crt
167-
sudo dpkg-reconfigure ca-certificates
168-
cd ..
169-
sed -i -e '/^services:/a \
170-
- name: ci-gcp\
171-
image: fsouza/fake-gcs-server\
172-
volumes:\
173-
- storage_data:/etc/pulp\
174-
command: "{% if pulp_scheme == "https" %} -cert-location /etc/pulp/gcpcert.pem {% else %} -scheme http{% endif %}"' vars/main.yaml
175-
sed -i -e '$a gcp_test: true\
176-
pulp_scenario_settings: {{ pulp_settings_gcp | tojson }}\
177-
pulp_scenario_env: {{ pulp_env_gcp | tojson }}\
178-
' vars/main.yaml
179-
fi
180-
{%- endif %}
181-
182-
echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
183-
184-
if [ "${PULP_API_ROOT:-}" ]; then
185-
sed -i -e '$a api_root: "'"$PULP_API_ROOT"'"' vars/main.yaml
186-
fi
187-
188-
pulp config create --base-url {{ pulp_scheme }}://pulp{% if pulp_scheme != 'https' %} --no-verify-ssl{% endif %} --api-root "$PULP_API_ROOT" --username "admin" --password "password"
38+
pulp config create --base-url {{ pulp_scheme }}://pulp{% if pulp_scheme != 'https' %} --no-verify-ssl{% endif %} --api-root "${PULP_API_ROOT}" --username "admin" --password "password"
18939
{% if test_cli -%}
19040
cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../{{ cli_package }}/tests/cli.toml"
19141
{%- endif %}

0 commit comments

Comments
 (0)