From 54d7648f3323e79152b68f863f88cb8fdb1cd49b Mon Sep 17 00:00:00 2001 From: Daniel Pawlik Date: Thu, 8 May 2025 13:26:37 +0200 Subject: [PATCH 1/2] DNM Checking CI job result after enabling disk IOPS/RW limitation The CI still have from time to time issue related to "noisy neighbor". We still can ask infra team to apply limitation inside the flavor, but until we don't know what quota can be set, let's do it via systemd. For many services and for CRC (kubelet has set cgroup systemd in /etc/kubernetes/kubelet.conf), so should respect that. More info what value were set are in commit message and PR [1]. [1] https://review.rdoproject.org/r/c/config/+/57582/ Depends-On: https://github.com/openstack-k8s-operators/openstack-operator/pull/1434 --- zuul.d/base.yaml | 2 ++ zuul.d/edpm.yaml | 2 ++ zuul.d/edpm_multinode.yaml | 4 ++++ zuul.d/kuttl_multinode.yaml | 2 ++ zuul.d/tempest_multinode.yaml | 2 ++ 5 files changed, 12 insertions(+) diff --git a/zuul.d/base.yaml b/zuul.d/base.yaml index 25e92d2fc..46be0a3e7 100644 --- a/zuul.d/base.yaml +++ b/zuul.d/base.yaml @@ -106,6 +106,8 @@ run: - ci/playbooks/content_provider/run.yml vars: + # More info: https://review.rdoproject.org/r/c/config/+/57582/ + crc_disk_limitation: true cifmw_artifacts_basedir: "{{ cifmw_basedir | default(ansible_user_dir ~ '/ci-framework-data') }}" diff --git a/zuul.d/edpm.yaml b/zuul.d/edpm.yaml index f325f9c40..161e83899 100644 --- a/zuul.d/edpm.yaml +++ b/zuul.d/edpm.yaml @@ -15,6 +15,8 @@ parent: cifmw-base-crc-openstack run: ci/playbooks/edpm_baremetal_deployment/run.yml vars: + # More info: https://review.rdoproject.org/r/c/config/+/57582 + crc_disk_limitation: true crc_parameters: "--memory 32000 --disk-size 240 --cpus 12" cifmw_manage_secrets_pullsecret_content: '{}' cifmw_rhol_crc_binary_folder: "/usr/local/bin" diff --git a/zuul.d/edpm_multinode.yaml b/zuul.d/edpm_multinode.yaml index 055bde916..a8f7649fb 100644 --- a/zuul.d/edpm_multinode.yaml +++ b/zuul.d/edpm_multinode.yaml @@ -7,6 +7,8 @@ A multinode EDPM Zuul job which has one controller, one extracted crc and two compute nodes. It is used in whitebox neutron tempest plugin testing. vars: + # More info: https://review.rdoproject.org/r/c/config/+/57582 + crc_disk_limitation: true crc_ci_bootstrap_cloud_name: "{{ nodepool.cloud | replace('-nodepool-tripleo','') }}" crc_ci_bootstrap_networking: networks: @@ -146,6 +148,8 @@ parent: podified-multinode-edpm-deployment-crc nodeset: centos-9-medium-3x-centos-9-crc-cloud-ocp-4-18-1-xxl vars: + # more info: https://review.rdoproject.org/r/c/config/+/57582/ + crc_disk_limitation: true cifmw_edpm_deploy_hci: true crc_ci_bootstrap_cloud_name: "{{ nodepool.cloud | replace('-nodepool-tripleo','') }}" crc_ci_bootstrap_networking: diff --git a/zuul.d/kuttl_multinode.yaml b/zuul.d/kuttl_multinode.yaml index e45498d19..08362fcce 100644 --- a/zuul.d/kuttl_multinode.yaml +++ b/zuul.d/kuttl_multinode.yaml @@ -7,6 +7,8 @@ nodeset: centos-9-medium-crc-cloud-ocp-4-18-1-3xl vars: zuul_log_collection: true + # More info: https://review.rdoproject.org/r/c/config/+/57582/ + crc_disk_limitation: true extra-vars: crc_ci_bootstrap_cloud_name: "{{ nodepool.cloud | replace('-nodepool-tripleo','') }}" crc_ci_bootstrap_networking: diff --git a/zuul.d/tempest_multinode.yaml b/zuul.d/tempest_multinode.yaml index 0d76851d4..d7bb619fe 100644 --- a/zuul.d/tempest_multinode.yaml +++ b/zuul.d/tempest_multinode.yaml @@ -68,6 +68,8 @@ name: podified-multinode-edpm-deployment-crc-tempest parent: podified-multinode-edpm-deployment-crc vars: + # More info: https://review.rdoproject.org/r/c/config/+/57582 + crc_disk_limitation: true cifmw_extras: - '@scenarios/centos-9/ci.yml' - '@scenarios/centos-9/multinode-ci.yml' From fc44237a578ac957fd1669d7b62c9b94b4034554 Mon Sep 17 00:00:00 2001 From: Daniel Pawlik Date: Fri, 9 May 2025 10:28:40 +0200 Subject: [PATCH 2/2] DNM Checking kubelet logs The kubelet service logs provides many interesting information, that might be helpful to see what is the root cause of failing job. Signed-off-by: Daniel Pawlik --- ci/playbooks/e2e-collect-logs.yml | 4 ++-- zuul.d/kuttl_multinode.yaml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/ci/playbooks/e2e-collect-logs.yml b/ci/playbooks/e2e-collect-logs.yml index b7733626a..3f0b3bde4 100644 --- a/ci/playbooks/e2e-collect-logs.yml +++ b/ci/playbooks/e2e-collect-logs.yml @@ -1,6 +1,6 @@ --- - name: "Run ci/playbooks/e2e-collect-logs.yml" - hosts: "{{ cifmw_zuul_target_host | default('all') }}" + hosts: "{{ cifmw_zuul_target_host | default('all') }}" gather_facts: true tasks: - name: Filter out host if needed @@ -27,7 +27,7 @@ ansible-playbook playbooks/99-logs.yml -e @scenarios/centos-9/base.yml -- name: "Run ci/playbooks/collect-logs.yml on CRC host" +- name: "Run ci/playbooks/e2e-collect-logs.yml on CRC host" hosts: crc gather_facts: false tasks: diff --git a/zuul.d/kuttl_multinode.yaml b/zuul.d/kuttl_multinode.yaml index 08362fcce..71954b331 100644 --- a/zuul.d/kuttl_multinode.yaml +++ b/zuul.d/kuttl_multinode.yaml @@ -9,6 +9,8 @@ zuul_log_collection: true # More info: https://review.rdoproject.org/r/c/config/+/57582/ crc_disk_limitation: true + # 250MB + rw_total_limit: 262144000 extra-vars: crc_ci_bootstrap_cloud_name: "{{ nodepool.cloud | replace('-nodepool-tripleo','') }}" crc_ci_bootstrap_networking: