Change memcache to use memcache_pool driver

In HA tests it was identified that pymemcache backend does not
recover/fail over to the next memcache instance when one goes
away unexpected. Using memcache_pool with bmemcache does failover
properly.

The memcache_sasl_enable=true is a workaround to force oslo.cache
to switch to the bmemcache driver, which is the only driver in
memcache_pool supporting tls+fips.

Jira: OSPRH-16651

Co-outhored-by: Luca Miccini <lmiccini@redhat.com>

Signed-off-by: Martin Schuppert <mschuppert@redhat.com>

Author: stuggi

templates/nova.conf

@@ -172,16 +172,17 @@ enabled = True
 {{if (index . "MemcachedServers")}}
 # on contoler we prefer to use memcache when its deployed
 {{if .MemcachedTLS}}
-backend = dogpile.cache.pymemcache
+backend = oslo_cache.memcache_pool
 memcache_servers={{ .MemcachedServers }}
-enable_retry_client = true
-retry_attempts = 2
-retry_delay = 0
+memcache_socket_timeout = 0.5
+memcache_pool_connection_get_timeout = 1
+# workaround to force bmemcache driver
+memcache_sasl_enabled = true
 {{else}}
 backend = dogpile.cache.memcached
 memcache_servers={{ .MemcachedServersWithInet }}
-memcache_dead_retry = 10
 {{end}}
+memcache_dead_retry = 30
 tls_enabled={{ .MemcachedTLS }}
 {{else}}
 # on compute nodes or where memcache is not deployed we should use an in memory

test/functional/nova_metadata_controller_test.go

@@ -974,7 +974,7 @@ var _ = Describe("NovaMetadata controller", func() {
 
 			configData = string(configDataMap.Data["01-nova.conf"])
 			Expect(configData).Should(
-				ContainSubstring("backend = dogpile.cache.pymemcache"))
+				ContainSubstring("backend = oslo_cache.memcache_pool"))
 			Expect(configData).Should(
 				ContainSubstring(fmt.Sprintf("memcache_servers=memcached-0.memcached.%s.svc:11211,memcached-1.memcached.%s.svc:11211,memcached-2.memcached.%s.svc:11211",
 					novaNames.Namespace, novaNames.Namespace, novaNames.Namespace)))

test/functional/nova_novncproxy_test.go

@@ -869,7 +869,7 @@ var _ = Describe("NovaNoVNCProxy controller", func() {
 
 			configData = string(configDataMap.Data["01-nova.conf"])
 			Expect(configData).Should(
-				ContainSubstring("backend = dogpile.cache.pymemcache"))
+				ContainSubstring("backend = oslo_cache.memcache_pool"))
 			Expect(configData).Should(
 				ContainSubstring(fmt.Sprintf("memcache_servers=memcached-0.memcached.%s.svc:11211,memcached-1.memcached.%s.svc:11211,memcached-2.memcached.%s.svc:11211",
 					novaNames.Namespace, novaNames.Namespace, novaNames.Namespace)))
@@ -1031,7 +1031,7 @@ var _ = Describe("NovaNoVNCProxy controller", func() {
 
 			configData = string(configDataMap.Data["01-nova.conf"])
 			Expect(configData).Should(
-				ContainSubstring("backend = dogpile.cache.pymemcache"))
+				ContainSubstring("backend = oslo_cache.memcache_pool"))
 			Expect(configData).Should(
 				ContainSubstring(fmt.Sprintf("memcache_servers=memcached-0.memcached.%s.svc:11211,memcached-1.memcached.%s.svc:11211,memcached-2.memcached.%s.svc:11211",
 					novaNames.Namespace, novaNames.Namespace, novaNames.Namespace)))
@@ -1221,7 +1221,7 @@ var _ = Describe("NovaNoVNCProxy controller", func() {
 
 			configData = string(configDataMap.Data["01-nova.conf"])
 			Expect(configData).Should(
-				ContainSubstring("backend = dogpile.cache.pymemcache"))
+				ContainSubstring("backend = oslo_cache.memcache_pool"))
 			Expect(configData).Should(
 				ContainSubstring(fmt.Sprintf("memcache_servers=memcached-0.memcached.%s.svc:11211,memcached-1.memcached.%s.svc:11211,memcached-2.memcached.%s.svc:11211",
 					novaNames.Namespace, novaNames.Namespace, novaNames.Namespace)))

test/functional/nova_scheduler_test.go

@@ -741,7 +741,7 @@ var _ = Describe("NovaScheduler controller", func() {
 
 			configData := string(configDataMap.Data["01-nova.conf"])
 			Expect(configData).Should(
-				ContainSubstring("backend = dogpile.cache.pymemcache"))
+				ContainSubstring("backend = oslo_cache.memcache_pool"))
 			Expect(configData).Should(
 				ContainSubstring(fmt.Sprintf("memcache_servers=memcached-0.memcached.%s.svc:11211,memcached-1.memcached.%s.svc:11211,memcached-2.memcached.%s.svc:11211",
 					novaNames.Namespace, novaNames.Namespace, novaNames.Namespace)))

test/functional/novaapi_controller_test.go

@@ -1037,7 +1037,7 @@ var _ = Describe("NovaAPI controller", func() {
 
 			configData = string(configDataMap.Data["01-nova.conf"])
 			Expect(configData).Should(
-				ContainSubstring("backend = dogpile.cache.pymemcache"))
+				ContainSubstring("backend = oslo_cache.memcache_pool"))
 			Expect(configData).Should(
 				ContainSubstring(fmt.Sprintf("memcache_servers=memcached-0.memcached.%s.svc:11211,memcached-1.memcached.%s.svc:11211,memcached-2.memcached.%s.svc:11211",
 					novaNames.Namespace, novaNames.Namespace, novaNames.Namespace)))

test/functional/novaconductor_controller_test.go

@@ -1090,7 +1090,7 @@ var _ = Describe("NovaConductor controller", func() {
 
 			configData := string(configDataMap.Data["01-nova.conf"])
 			Expect(configData).Should(
-				ContainSubstring("backend = dogpile.cache.pymemcache"))
+				ContainSubstring("backend = oslo_cache.memcache_pool"))
 			Expect(configData).Should(
 				ContainSubstring(fmt.Sprintf("memcache_servers=memcached-0.memcached.%s.svc:11211,memcached-1.memcached.%s.svc:11211,memcached-2.memcached.%s.svc:11211",
 					novaNames.Namespace, novaNames.Namespace, novaNames.Namespace)))