Skip to content

Commit 4f43bc5

Browse files
mdellwegmdellweg
committed
Update CI
1 parent 2b43dfd commit 4f43bc5

11 files changed

Lines changed: 162 additions & 105 deletions

File tree

‎.ci/ansible/Containerfile.j2‎

Lines changed: 20 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,28 @@
1-
FROM {{ ci_base | default(pulp_default_container) }}
1+
FROM {{ image.ci_base }}
2+
{%- if image.webserver_snippet %}
23

3-
# Add source directories to container
4-
{% for item in plugins %}
5-
ADD ./{{ item.name }} ./{{ item.name }}
6-
{% endfor %}
4+
ADD ./{{ plugin_name }}/{{ plugin_name | replace("-", "_") }}/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ plugin_name }}.conf
5+
{%- endif %}
6+
7+
{%- for item in extra_files | default([]) %}
78

8-
{% for item in extra_files | default([]) %}
99
ADD ./{{ item.origin }} {{ item.destination }}
10-
{% endfor %}
10+
{%- endfor %}
1111

1212
# This MUST be the ONLY call to pip install in inside the container.
1313
RUN pip3 install --upgrade pip setuptools wheel && \
1414
rm -rf /root/.cache/pip && \
15-
pip3 install
16-
{%- if s3_test | default(false) -%}
17-
{{ " " }}git+https://github.com/gerrod3/botocore.git@fix-100-continue
15+
pip3 install {{ image.source }}
16+
{%- if image.upperbounds | default(false) -%}
17+
{{ " " }}-c ./{{ plugin_name }}/upperbounds_constraints.txt
1818
{%- endif -%}
19-
{%- for item in plugins -%}
20-
{{ " " }}{{ item.source }}
21-
{%- if item.upperbounds | default(false) -%}
22-
{{ " " }}-c ./{{ item.name }}/upperbounds_constraints.txt
19+
{%- if image.lowerbounds | default(false) -%}
20+
{{ " " }}-c ./{{ plugin_name }}/lowerbounds_constraints.txt
2321
{%- endif -%}
24-
{%- if item.lowerbounds | default(false) -%}
25-
{{ " " }}-c ./{{ item.name }}/lowerbounds_constraints.txt
22+
{%- if image.ci_requirements | default(false) -%}
23+
{{ " " }}-r ./{{ plugin_name }}/ci_requirements.txt
2624
{%- endif -%}
27-
{%- if item.ci_requirements | default(false) -%}
28-
{{ " " }}-r ./{{ item.name }}/ci_requirements.txt
29-
{%- endif -%}
30-
{%- endfor %}
31-
{{ " " }}-c ./{{ plugins[0].name }}/.ci/assets/ci_constraints.txt && \
25+
{{ " " }}-c ./{{ plugin_name }}/.ci/assets/ci_constraints.txt && \
3226
rm -rf /root/.cache/pip
3327

3428
{% if pulp_env is defined and pulp_env %}
@@ -46,11 +40,12 @@ ENV {{ key | upper }}={{ value }}
4640
USER pulp:pulp
4741
RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \
4842
/usr/local/bin/pulpcore-manager collectstatic --clear --noinput --link
43+
44+
RUN mkdir /var/lib/pulp/.config
4945
USER root:root
5046

51-
{% for item in plugins %}
52-
RUN export plugin_path="$(pip3 show {{ item.name }} | sed -n -e 's/Location: //p')/{{ item.name }}" && \
53-
ln $plugin_path/app/webserver_snippets/nginx.conf /etc/nginx/pulp/{{ item.name }}.conf || true
54-
{% endfor %}
47+
# Lots of plugins try to use this path, and throw warnings if they cannot access it.
48+
RUN mkdir /.pytest_cache
49+
RUN chown pulp:pulp /.pytest_cache
5550

5651
ENTRYPOINT ["/init"]

‎.ci/ansible/build_container.yaml‎

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,14 @@
11
# Ansible playbook to create the pulp service containers image
22
---
3-
- hosts: localhost
3+
- hosts: "localhost"
44
gather_facts: false
55
vars_files:
6-
- vars/main.yaml
6+
- "vars/main.yaml"
77
tasks:
88
- name: "Generate Containerfile from template"
9-
template:
10-
src: Containerfile.j2
11-
dest: Containerfile
12-
9+
ansible.builtin.template:
10+
src: "Containerfile.j2"
11+
dest: "Containerfile"
1312
- name: "Build pulp image"
1413
# We build from the ../.. (parent dir of pulpcore git repo) Docker build
1514
# "context" so that repos like pulp-smash are accessible to Docker

‎.ci/ansible/settings.py.j2‎

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,6 @@ REDIS_HOST = "localhost"
1010
REDIS_PORT = 6379
1111
ANALYTICS = False
1212

13-
{% if api_root is defined %}
14-
API_ROOT = {{ api_root | repr }}
15-
{% endif %}
16-
1713
{% if pulp_settings %}
1814
{% for key, value in pulp_settings.items() %}
1915
{{ key | upper }} = {{ value | repr }}

‎.ci/ansible/settings/settings.py‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
CONTENT_ORIGIN = "https://pulp:443"
2+
ANSIBLE_API_HOSTNAME = "https://pulp:443"
3+
ANSIBLE_CONTENT_HOSTNAME = "https://pulp:443/pulp/content"
4+
PRIVATE_KEY_PATH = "/etc/pulp/certs/token_private_key.pem"
5+
PUBLIC_KEY_PATH = "/etc/pulp/certs/token_public_key.pem"
6+
TOKEN_SERVER = "https://pulp:443/token/"
7+
TOKEN_SIGNATURE_ALGORITHM = "ES256"
8+
CACHE_ENABLED = True
9+
REDIS_HOST = "localhost"
10+
REDIS_PORT = 6379
11+
ANALYTICS = False
12+
13+
API_ROOT = "/rerouted/djnd/"
14+
15+
16+
MEDIA_ROOT = ""
17+
STORAGES = {
18+
"default": {
19+
"BACKEND": "storages.backends.s3boto3.S3Boto3Storage",
20+
"OPTIONS": {
21+
"access_key": "AKIAIT2Z5TDYPX3ARJBA",
22+
"addressing_style": "path",
23+
"bucket_name": "pulp3",
24+
"default_acl": "@none",
25+
"endpoint_url": "http://minio:9000",
26+
"region_name": "eu-central-1",
27+
"secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS",
28+
"signature_version": "s3v4",
29+
},
30+
},
31+
"staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"},
32+
}
33+
ALLOWED_CONTENT_CHECKSUMS = ["md5", "sha224", "sha256", "sha384", "sha512"]
34+
DOMAIN_ENABLED = True

‎.ci/ansible/start_container.yaml‎

Lines changed: 36 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -1,33 +1,26 @@
11
# Ansible playbook to start the pulp service container and its supporting services
22
---
3-
- hosts: localhost
3+
- hosts: "localhost"
44
gather_facts: false
55
vars_files:
6-
- vars/main.yaml
6+
- "vars/main.yaml"
77
tasks:
88
- name: "Create Settings Directories"
9-
file:
9+
ansible.builtin.file:
1010
path: "{{ item }}"
11-
state: directory
11+
state: "directory"
1212
mode: "0755"
1313
loop:
14-
- settings
15-
- ssh
16-
- ~/.config/pulp_smash
14+
- "settings"
1715

1816
- name: "Generate Pulp Settings"
1917
template:
20-
src: settings.py.j2
21-
dest: settings/settings.py
22-
23-
- name: "Configure pulp-smash"
24-
copy:
25-
src: smash-config.json
26-
dest: ~/.config/pulp_smash/settings.json
18+
src: "settings.py.j2"
19+
dest: "settings/settings.py"
2720

2821
- name: "Setup docker networking"
2922
docker_network:
30-
name: pulp_ci_bridge
23+
name: "pulp_ci_bridge"
3124

3225
- name: "Start Service Containers"
3326
docker_container:
@@ -37,24 +30,24 @@
3730
recreate: true
3831
privileged: true
3932
networks:
40-
- name: pulp_ci_bridge
33+
- name: "pulp_ci_bridge"
4134
aliases: "{{ item.name }}"
4235
volumes: "{{ item.volumes | default(omit) }}"
4336
env: "{{ item.env | default(omit) }}"
4437
command: "{{ item.command | default(omit) }}"
45-
state: started
38+
state: "started"
4639
loop: "{{ services | default([]) }}"
4740

4841
- name: "Retrieve Docker Network Info"
4942
docker_network_info:
50-
name: pulp_ci_bridge
51-
register: pulp_ci_bridge_info
43+
name: "pulp_ci_bridge"
44+
register: "pulp_ci_bridge_info"
5245

5346
- name: "Update /etc/hosts"
5447
lineinfile:
55-
path: /etc/hosts
48+
path: "/etc/hosts"
5649
regexp: "\\s{{ item.value.Name }}\\s*$"
57-
line: "{{ item.value.IPv4Address | ipaddr('address') }}\t{{ item.value.Name }}"
50+
line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}"
5851
loop: "{{ pulp_ci_bridge_info.network.Containers | dict2items }}"
5952
become: true
6053

@@ -63,19 +56,19 @@
6356
aws_access_key: "{{ minio_access_key }}"
6457
aws_secret_key: "{{ minio_secret_key }}"
6558
s3_url: "http://minio:9000"
66-
region: eu-central-1
67-
name: pulp3
68-
state: present
69-
when: s3_test | default(false)
59+
region: "eu-central-1"
60+
name: "pulp3"
61+
state: "present"
62+
when: "s3_test | default(false)"
7063

7164
- block:
7265
- name: "Wait for Pulp"
7366
uri:
74-
url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/"
75-
follow_redirects: all
76-
validate_certs: no
77-
register: result
78-
until: result.status == 200
67+
url: "http://pulp{{ pulp_scenario_settings.api_root | default(pulp_settings.api_root | default('\/pulp\/', True), True) }}api/v3/status/"
68+
follow_redirects: "all"
69+
validate_certs: "no"
70+
register: "result"
71+
until: "result.status == 200"
7972
retries: 12
8073
delay: 5
8174
rescue:
@@ -86,7 +79,7 @@
8679
- name: "Check version of component being tested"
8780
assert:
8881
that:
89-
- (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] | canonical_semver == (component_version | canonical_semver)
82+
- "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)"
9083
fail_msg: |
9184
Component {{ item.app_label }} was expected to be installed in version {{ component_version }}.
9285
Instead it is reported as version {{ (result.json.versions | items2dict(key_name="component", value_name="version"))[item.app_label] }}.
@@ -100,9 +93,20 @@
10093
login admin
10194
password password
10295
103-
- hosts: pulp
96+
- hosts: "pulp"
10497
gather_facts: false
10598
tasks:
99+
- name: "Create directory for pulp-smash config"
100+
ansible.builtin.file:
101+
path: "/var/lib/pulp/.config/pulp_smash/"
102+
state: "directory"
103+
mode: "0755"
104+
105+
- name: "Configure pulp-smash"
106+
ansible.builtin.copy:
107+
src: "smash-config.json"
108+
dest: "/var/lib/pulp/.config/pulp_smash/settings.json"
109+
106110
- name: "Set pulp admin password"
107111
command:
108112
cmd: "pulpcore-manager reset-admin-password --password password"

‎.ci/ansible/vars/main.yaml‎

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
---
2+
scenario: "s3"
3+
plugin_name: "pulp_gem"
4+
legacy_component_name: "pulp_gem"
5+
component_name: "gem"
6+
component_version: "0.8.0.dev0"
7+
pulp_env: {}
8+
pulp_settings: null
9+
pulp_scheme: "https"
10+
api_root: "/rerouted/djnd/"
11+
image:
12+
name: "pulp"
13+
tag: "ci_build"
14+
ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest"
15+
source: "./pulp_gem/dist/pulp_gem-0.8.0.dev0-py3-none-any.whl pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue"
16+
ci_requirements: false
17+
upperbounds: false
18+
lowerbounds: false
19+
webserver_snippet: false
20+
extra_files:
21+
- origin: "pulp_gem"
22+
destination: "pulp_gem"
23+
services:
24+
- name: "pulp"
25+
image: "pulp:ci_build"
26+
volumes:
27+
- "./settings:/etc/pulp"
28+
- "../../../pulp-openapi-generator:/root/pulp-openapi-generator"
29+
env:
30+
PULP_WORKERS: "4"
31+
PULP_HTTPS: "true"
32+
- name: "minio"
33+
image: "minio/minio"
34+
env:
35+
MINIO_ACCESS_KEY: "AKIAIT2Z5TDYPX3ARJBA"
36+
MINIO_SECRET_KEY: "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"
37+
command: "server /data"
38+
s3_test: true
39+
minio_access_key: "AKIAIT2Z5TDYPX3ARJBA"
40+
minio_secret_key: "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"
41+
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "allowed_content_checksums": ["md5", "sha224", "sha256", "sha384", "sha512"], "domain_enabled": true}
42+
pulp_scenario_env: {}
43+
...

‎.github/workflows/scripts/before_install.sh‎

Lines changed: 13 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ fi
2626
COMPONENT_VERSION="$(bump-my-version show current_version | tail -n -1 | python -c 'from packaging.version import Version; print(Version(input()))')"
2727
COMPONENT_SOURCE="./pulp_gem/dist/pulp_gem-${COMPONENT_VERSION}-py3-none-any.whl"
2828
if [ "$TEST" = "s3" ]; then
29-
COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3]"
29+
COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue"
3030
fi
3131
if [ "$TEST" = "azure" ]; then
3232
COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]"
@@ -38,40 +38,37 @@ fi
3838
if [[ "$TEST" = "lowerbounds" ]]; then
3939
python3 .ci/scripts/calc_constraints.py pyproject.toml > lowerbounds_constraints.txt
4040
fi
41-
export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/")
42-
43-
echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV"
4441

4542
# Compose the scenario definition.
4643
mkdir -p .ci/ansible/vars
4744

4845
cat > .ci/ansible/vars/main.yaml << VARSYAML
4946
---
5047
scenario: "${TEST}"
48+
plugin_name: "pulp_gem"
5149
legacy_component_name: "pulp_gem"
5250
component_name: "gem"
5351
component_version: "${COMPONENT_VERSION}"
5452
pulp_env: {}
55-
pulp_settings: null
53+
pulp_settings: {"api_root": "/pulp/"}
5654
pulp_scheme: "https"
57-
pulp_default_container: "ghcr.io/pulp/pulp-ci-centos9:latest"
58-
api_root: "${PULP_API_ROOT}"
5955
image:
6056
name: "pulp"
6157
tag: "ci_build"
62-
plugins:
63-
- name: "pulp_gem"
64-
source: "${COMPONENT_SOURCE}"
65-
ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
66-
upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
67-
lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)
58+
ci_base: "ghcr.io/pulp/pulp-ci-centos9:latest"
59+
source: "${COMPONENT_SOURCE}"
60+
ci_requirements: $(test -f ci_requirements.txt && echo -n true || echo -n false)
61+
upperbounds: $(test "${TEST}" = "pulp" && echo -n true || echo -n false)
62+
lowerbounds: $(test "${TEST}" = "lowerbounds" && echo -n true || echo -n false)
63+
webserver_snippet: $(test -f pulp_gem/app/webserver_snippets/nginx.conf && echo -n true || echo -n false )
64+
extra_files:
65+
- origin: "pulp_gem"
66+
destination: "pulp_gem"
6867
services:
6968
- name: "pulp"
7069
image: "pulp:ci_build"
7170
volumes:
7271
- "./settings:/etc/pulp"
73-
- "./ssh:/keys/"
74-
- "~/.config:/var/lib/pulp/.config"
7572
- "../../../pulp-openapi-generator:/root/pulp-openapi-generator"
7673
env:
7774
PULP_WORKERS: "4"
@@ -91,7 +88,7 @@ if [ "$TEST" = "s3" ]; then
9188
s3_test: true
9289
minio_access_key: "${MINIO_ACCESS_KEY}"
9390
minio_secret_key: "${MINIO_SECRET_KEY}"
94-
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "allowed_content_checksums": ["md5", "sha224", "sha256", "sha384", "sha512"], "domain_enabled": true}
91+
pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "allowed_content_checksums": ["md5", "sha224", "sha256", "sha384", "sha512"], "api_root": "/rerouted/djnd/", "domain_enabled": true}
9592
pulp_scenario_env: {}
9693
VARSYAML
9794
fi

‎.github/workflows/scripts/before_script.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ fi
2323
# Developers often want to know the final pulp config
2424
echo
2525
echo "# Pulp config:"
26-
tail -v -n +1 .ci/ansible/settings/settings.* ~/.config/pulp_smash/settings.json
26+
tail -v -n +1 .ci/ansible/settings/settings.*
2727

2828
echo
2929
echo "# Containerfile:"

0 commit comments

Comments
 (0)