diff --git a/ansible/group_vars/all/keystone.yml b/ansible/group_vars/all/keystone.yml index a4da55199f..f0897dbe4a 100644 --- a/ansible/group_vars/all/keystone.yml +++ b/ansible/group_vars/all/keystone.yml @@ -16,7 +16,7 @@ keystone_admin_project: "admin" # Whether or not to apply changes to service user passwords when services are # reconfigured -update_keystone_service_user_passwords: true +update_keystone_service_user_passwords: false default_project_domain_name: "Default" default_project_domain_id: "default" @@ -68,13 +68,21 @@ keystone_default_user_role: "member" # The configuration of attribute mappings is a list of objects, where each # object must have a 'name' (that mapps to the 'attribute_mapping' to the IdP # object in the IdPs set), and the 'file' with a full qualified path to a mapping file. +# +# The "schema_version" attribute is optional. Here we just demonstrate a few examples of +# configuration; before using them, one must check the supported version by Keystone. If +# one wants to use the default of Keystone, she/he can omit the "schema_version" option. +# # keystone_identity_mappings: # - name: "mappingId1" # file: "/full/qualified/path/to/mapping/json/file/to/mappingId1" +# schema_version: "1.0" # - name: "mappingId2" # file: "/full/qualified/path/to/mapping/json/file/to/mappingId2" +# schema_version: "2.0" # - name: "mappingId3" # file: "/full/qualified/path/to/mapping/json/file/to/mappingId3" +# schema_version: "3.0" keystone_identity_providers: [] keystone_identity_mappings: [] keystone_public_port: "{{ haproxy_single_external_frontend_public_port if haproxy_single_external_frontend | bool else keystone_public_listen_port }}" diff --git a/ansible/module_utils/kolla_podman_worker.py b/ansible/module_utils/kolla_podman_worker.py index 4fb8df08b2..c39bac3e00 100644 --- a/ansible/module_utils/kolla_podman_worker.py +++ b/ansible/module_utils/kolla_podman_worker.py @@ -539,8 +539,12 @@ def compare_config(self): if rc == 0: return False elif rc == 1: - self._config_diff = (raw_output.decode('utf-8') if - isinstance(raw_output, bytes) else raw_output) + try: + self._config_diff = (raw_output.decode('utf-8') if + isinstance(raw_output, bytes) else + raw_output) + except UnicodeDecodeError: + self._config_diff = 'container changed during config check' return True else: raise Exception('Failed to compare container configuration: ' diff --git a/ansible/roles/barbican/defaults/main.yml b/ansible/roles/barbican/defaults/main.yml index 5e1ad02871..9d25bf6ea8 100644 --- a/ansible/roles/barbican/defaults/main.yml +++ b/ansible/roles/barbican/defaults/main.yml @@ -109,7 +109,7 @@ barbican_api_healthcheck_test: - >- healthcheck_curl {{ 'https' if barbican_enable_tls_backend | bool else 'http' }}://{{ - api_interface_address | put_address_in_context('url') }}:{{ barbican_api_listen_port }}/healthcheck" + api_interface_address | put_address_in_context('url') }}:{{ barbican_api_listen_port }}/healthcheck barbican_api_healthcheck_timeout: "{{ default_container_healthcheck_timeout }}" barbican_api_healthcheck: interval: "{{ barbican_api_healthcheck_interval }}" diff --git a/ansible/roles/ironic/defaults/main.yml b/ansible/roles/ironic/defaults/main.yml index 47d17649ff..7dbe56e069 100644 --- a/ansible/roles/ironic/defaults/main.yml +++ b/ansible/roles/ironic/defaults/main.yml @@ -34,7 +34,6 @@ ironic_services: group: ironic-conductor enabled: true image: "{{ ironic_conductor_image_full }}" - privileged: true volumes: "{{ ironic_conductor_default_volumes + ironic_conductor_extra_volumes + lookup('vars', 'run_default_volumes_' + kolla_container_engine) }}" dimensions: "{{ ironic_conductor_dimensions }}" healthcheck: "{{ ironic_conductor_healthcheck }}" diff --git a/ansible/roles/keystone/tasks/register_identity_providers.yml b/ansible/roles/keystone/tasks/register_identity_providers.yml index ab00002fb2..2c88c3d443 100644 --- a/ansible/roles/keystone/tasks/register_identity_providers.yml +++ b/ansible/roles/keystone/tasks/register_identity_providers.yml @@ -61,6 +61,7 @@ ca_cert: "{{ openstack_cacert if openstack_cacert != '' else omit }}" name: "{{ item.name }}" rules: "{{ '{{ keystone_identity_mapping_rules | from_json }}' }}" + schema_version: "{{ item.schema_version | default(omit) }}" state: present module_extra_vars: keystone_identity_mapping_rules: "{{ lookup('ansible.builtin.file', item.file) }}" diff --git a/ansible/roles/nova-cell/defaults/main.yml b/ansible/roles/nova-cell/defaults/main.yml index 27414d5e01..0872bb3324 100644 --- a/ansible/roles/nova-cell/defaults/main.yml +++ b/ansible/roles/nova-cell/defaults/main.yml @@ -639,6 +639,17 @@ libvirt_sasl_authname: "nova" libvirt_sasl_mech_list: - "{{ 'SCRAM-SHA-256' if libvirt_tls | bool else 'DIGEST-MD5' }}" +# Connection processing limits for libvirtd. nova-compute is effectively the +# only privileged client of libvirtd under Kolla, so libvirt's stock per-client +# request cap (max_client_requests=5) is the binding constraint rather than a +# safety net: concurrent instance builds plus the resource-tracker periodic +# routinely exceed five in-flight RPCs, which starves keepalive responses until +# libvirtd drops the connection. Raise the per-client cap, and the worker pool +# along with it so that lifting the per-client limit does not starve other +# connections (live migration, health checks, periodics). +libvirt_max_client_requests: 20 +libvirt_max_workers: 50 + #################### # Kolla #################### diff --git a/ansible/roles/nova-cell/templates/libvirtd.conf.j2 b/ansible/roles/nova-cell/templates/libvirtd.conf.j2 index aa01f743d8..d156fb4be7 100644 --- a/ansible/roles/nova-cell/templates/libvirtd.conf.j2 +++ b/ansible/roles/nova-cell/templates/libvirtd.conf.j2 @@ -25,3 +25,13 @@ listen_addr = "{{ migration_interface_address }}" # Enable read-only access to libvirt socket auth_unix_ro = "none" {% endif %} + +# Connection processing limits. libvirt's defaults (max_workers=20, +# max_client_requests=5) assume a multi-tenant host with many mutually +# distrustful clients. Under Kolla, nova-compute is effectively the sole +# privileged client, so the per-client cap of 5 becomes the binding +# constraint: concurrent builds plus the resource-tracker periodic exceed +# five in-flight RPCs and starve keepalive responses, causing libvirtd to +# drop the connection after keepalive_interval (5s) x keepalive_count (5). +max_workers = {{ libvirt_max_workers }} +max_client_requests = {{ libvirt_max_client_requests }} diff --git a/doc/source/admin/password-rotation.rst b/doc/source/admin/password-rotation.rst index 40f4ac9a6a..de2e996d09 100644 --- a/doc/source/admin/password-rotation.rst +++ b/doc/source/admin/password-rotation.rst @@ -38,6 +38,13 @@ The majority of the secrets can be applied by simply reconfiguring services with ``kolla-ansible reconfigure``. Below is a list of secrets that can be applied this way. +.. note:: + + Rotating ``*_keystone_password`` secrets requires + ``update_keystone_service_user_passwords`` to be set to ``true`` in + ``globals.yml`` before running ``kolla-ansible reconfigure``, as the default + is ``false``. This setting may be reverted to ``false`` after the reconfigure + completes. * ``*_keystone_password`` * ``*_database_password`` (excluding ``nova_database_password``) diff --git a/doc/source/reference/compute/libvirt-guide.rst b/doc/source/reference/compute/libvirt-guide.rst index 24a55e1fbe..d6212b9086 100644 --- a/doc/source/reference/compute/libvirt-guide.rst +++ b/doc/source/reference/compute/libvirt-guide.rst @@ -198,3 +198,26 @@ Since the Yoga release, the ``kolla-ansible certificates`` command generates certificates for libvirt TLS. A single key and certificate is used for all hosts, with a Subject Alternative Name (SAN) entry for each compute host hostname. + +Libvirt Connection Limits +========================= + +By default ``libvirt`` is configured by upstream packages with reasonable +defaults for an environment where many users are starting and stopping virtual +machines. This isn't true for Kolla Hypervisor nodes, where a single +``nova-compute`` container manages all of the virtual machine instances on the +machine. + +Kolla-Ansible therefore tunes the following values in ``libvirtd.conf``: + +* libvirt_max_client_requests: raised to 20 compared to the usual default of + 5. +* libvirt_max_workers: raised to 50 compared to the usual default of 20. + +Both of these values may be overridden in ``globals.yml``. + +This means that ``nova-compute`` can therefore perform 20 operations via +libvirt simultaneously instead of the usual default of 5. These values should +be safe, but if you see libvirt error messages saying ``Client hit max requests +limit``, you may want to consider an additional increase -- especially if you +have particularly large hypervisors. diff --git a/doc/source/reference/shared-services/keystone-guide.rst b/doc/source/reference/shared-services/keystone-guide.rst index 173766022f..d21a099bb7 100644 --- a/doc/source/reference/shared-services/keystone-guide.rst +++ b/doc/source/reference/shared-services/keystone-guide.rst @@ -87,6 +87,7 @@ below: keystone_identity_mappings: - name: "mappingId1" file: "/full/qualified/path/to/mapping/json/file/to/mappingId1" + schema_version: "2.0" In some cases it's necessary to add JWKS (JSON Web Key Set) uri. It is required for auth-openidc endpoint - which is diff --git a/doc/source/reference/storage/external-ceph-guide.rst b/doc/source/reference/storage/external-ceph-guide.rst index 90d488b19f..a831015ba7 100644 --- a/doc/source/reference/storage/external-ceph-guide.rst +++ b/doc/source/reference/storage/external-ceph-guide.rst @@ -537,15 +537,6 @@ This prevents cross-project and public object access. This can be resolved by setting ``ceph_rgw_swift_account_in_url`` to ``true``. This should match the ``rgw_swift_account_in_url`` configuration option in Ceph RadosGW. -.. warning:: - - When using an external Ceph RadosGW for object storage endpoints - (when ``enable_ceph_rgw: true``), operators are advised to set - ``update_keystone_service_user_passwords: false`` in``globals.yml``. - Leaving this option enabled causes the service user password to be reset - on every reconfigure, which invalidates existing tokens and can cause - unforeseen outages for object storage consumers. - Load balancing ============== diff --git a/etc/kolla/globals.yml b/etc/kolla/globals.yml index 09b47702c2..7754daba9c 100644 --- a/etc/kolla/globals.yml +++ b/etc/kolla/globals.yml @@ -507,7 +507,7 @@ workaround_ansible_issue_8743: true # Whether or not to apply changes to service user passwords when services are # reconfigured -#update_keystone_service_user_passwords: "true" +#update_keystone_service_user_passwords: "false" ######################## # Glance - Image Options diff --git a/releasenotes/notes/add_keystone_attribute_mapping_schema_version-7dff68169a306695.yaml b/releasenotes/notes/add_keystone_attribute_mapping_schema_version-7dff68169a306695.yaml new file mode 100644 index 0000000000..0fefeff359 --- /dev/null +++ b/releasenotes/notes/add_keystone_attribute_mapping_schema_version-7dff68169a306695.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + Add support to ``schema_version`` in the federated attribute mapping API + of Keystone. diff --git a/releasenotes/notes/bug-2157639-93e313f168b18133.yaml b/releasenotes/notes/bug-2157639-93e313f168b18133.yaml new file mode 100644 index 0000000000..f93516f79e --- /dev/null +++ b/releasenotes/notes/bug-2157639-93e313f168b18133.yaml @@ -0,0 +1,6 @@ +--- +fixes: + - | + Fixed a stray trailing quote in the ``barbican_api`` healthcheck + command that caused containers to be falsely reported as unhealthy. + `LP#2157639 `__ diff --git a/releasenotes/notes/libvirtd-connection-tuning-7c4f1e9a2b8d6f30.yaml b/releasenotes/notes/libvirtd-connection-tuning-7c4f1e9a2b8d6f30.yaml new file mode 100644 index 0000000000..fcc8a65d41 --- /dev/null +++ b/releasenotes/notes/libvirtd-connection-tuning-7c4f1e9a2b8d6f30.yaml @@ -0,0 +1,18 @@ +--- +features: + - | + The ``nova_libvirt`` container's ``libvirtd.conf`` now raises libvirt's + connection processing limits above their stock defaults, which are tuned + for a multi-tenant host rather than for Kolla's single-privileged-client + pattern. ``max_client_requests`` is raised from 5 to 20 and ``max_workers`` + from 20 to 50. Both are exposed as the ``libvirt_max_client_requests`` and + ``libvirt_max_workers`` variables so they can be tuned further. +fixes: + - | + Fixed intermittent instance build failures (``Cannot recv data: + Input/output error`` during ``defineXML``, leading to instances in + ``ERROR`` state) that occurred when several instances were built + concurrently. nova-compute is effectively the sole client of ``libvirtd``, + and libvirt's default per-client request cap of 5 was being exceeded by + concurrent builds plus periodic tasks, starving keepalive responses until + ``libvirtd`` dropped the connection. diff --git a/releasenotes/notes/update-ks-password-default-59af8b275e883d5e.yaml b/releasenotes/notes/update-ks-password-default-59af8b275e883d5e.yaml new file mode 100644 index 0000000000..ca9ccbf7ea --- /dev/null +++ b/releasenotes/notes/update-ks-password-default-59af8b275e883d5e.yaml @@ -0,0 +1,10 @@ +--- +upgrade: + - | + The default value of ``update_keystone_service_user_passwords`` has changed + from ``true`` to ``false``. Previously, service user passwords were updated + on every reconfigure run, which invalidated existing Keystone tokens and + could cause service disruptions. With the new default, passwords are only + set at user creation time. Operators who wish to force password rotation on + reconfigure must explicitly set ``update_keystone_service_user_passwords`` + to ``true`` in ``globals.yml``. diff --git a/tests/kolla_container_tests/test_podman_worker.py b/tests/kolla_container_tests/test_podman_worker.py index 13a968ca35..c9e7864c81 100644 --- a/tests/kolla_container_tests/test_podman_worker.py +++ b/tests/kolla_container_tests/test_podman_worker.py @@ -767,6 +767,24 @@ def test_compare_config_changed(self): user='root') self.assertTrue(return_data) + def test_compare_config_chenged_unicode_error(self): + self.fake_data['params']['name'] = 'my_container' + self.pw = get_PodmanWorker(self.fake_data['params']) + my_container = construct_container(self.fake_data['containers'][0]) + invalid_bytes = b'\x8e\xfe\xfd' + my_container.exec_run = mock.Mock(return_value=(1, invalid_bytes)) + self.pw.pc.containers.get.return_value = my_container + + return_data = self.pw.compare_config() + self.pw.pc.containers.get.assert_called_once_with( + self.fake_data['params']['name']) + my_container.exec_run.assert_called_once_with( + pwm.COMPARE_CONFIG_CMD, + user='root') + self.assertTrue(return_data) + self.assertEqual(self.pw._config_diff, + 'container changed during config check') + def test_compare_config_changed_container_exited(self): self.fake_data['params']['name'] = 'my_container' self.pw = get_PodmanWorker(self.fake_data['params']) diff --git a/tests/run.yml b/tests/run.yml index bb06ebd6ba..294bc90f32 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -301,12 +301,11 @@ - name: Install kolla-ansible and dependencies pip: - extra_args: "-c {{ upper_constraints_file }}" name: - "{{ kolla_ansible_src_dir }}" - "ansible-core{{ ansible_core_version_constraint }}" - "ara[server]<1.8" - - "openstacksdk" + - "openstacksdk<4.14" virtualenv: "{{ kolla_ansible_venv_path }}" - name: Install Ansible collections diff --git a/tests/setup_gate.sh b/tests/setup_gate.sh index 0be0c7b6da..9eb21e1222 100755 --- a/tests/setup_gate.sh +++ b/tests/setup_gate.sh @@ -12,10 +12,12 @@ function prepare_images { return fi - if [[ "${CONTAINER_ENGINE}" != "docker" && "${BASE_DISTRO}" =~ ^(debian|ubuntu)$ ]]; then - sudo apt install -y docker-buildx-plugin - else - sudo dnf install -y docker-buildx-plugin + if [[ "${CONTAINER_ENGINE}" == "docker" ]]; then + if [[ "${BASE_DISTRO}" =~ ^(debian|ubuntu)$ ]]; then + sudo apt install -y docker-buildx-plugin + else + sudo dnf install -y docker-buildx-plugin + fi fi sudo mkdir -p /tmp/logs/build