Add AuTest for connect_attempts rr_retries and max_retries (#12932)

* Add AuTest for connect_attempts

* Make error.log gold file platform neutral

* Address issue on Docker env

* Make Copilot happy

(cherry picked from commit b5557e6)

Author: masaori335

src/proxy/http/HttpTransact.cc

@@ -3995,9 +3995,9 @@ HttpTransact::error_log_connection_failure(State *s, ServerState_t conn_state)
       host_name = s->unmapped_url.host_get();
     }
     swoc::bwprint(error_bw_buffer,
-                  "CONNECT: attempt fail [{}] to {} for host='{}' "
+                  "CONNECT: attempt fail [{}] to {} for host='{}' sm_id={} "
                   "connection_result={::s} error={::s} retry_attempts={} url='{}'",
-                  HttpDebugNames::get_server_state_name(conn_state), s->current.server->dst_addr, host_name,
+                  HttpDebugNames::get_server_state_name(conn_state), s->current.server->dst_addr, host_name, s->state_machine_id(),
                   swoc::bwf::Errno(s->current.server->connect_result), swoc::bwf::Errno(s->cause_of_death_errno),
                   s->current.retry_attempts.get(), swoc::bwf::FirstOf(url_str, "<none>"));
     Log::error("%s", error_bw_buffer.c_str());

tests/gold_tests/autest-site/ats_replay.test.ext

@@ -119,6 +119,21 @@ def configure_ats(obj: 'TestRun', server: 'Process', ats_config: dict, dns: Opti
             gold_file = diags_log['gold_file']
             ts.Disk.diags_log.Content += gold_file
 
+        # error_log validation.
+        error_log = log_validation.get('error_log', {})
+        for contains_entry in error_log.get('contains', []):
+            expression = contains_entry['expression']
+            description = contains_entry.get('description', f'Verify error_log contains: {expression}')
+            ts.Disk.error_log.Content += Testers.ContainsExpression(expression, description)
+        for excludes_entry in error_log.get('excludes', []):
+            expression = excludes_entry['expression']
+            description = excludes_entry.get('description', f'Verify error_log excludes: {expression}')
+            ts.Disk.error_log.Content += Testers.ExcludesExpression(expression, description)
+        # Gold file validation for error_log.
+        if 'gold_file' in error_log:
+            gold_file = error_log['gold_file']
+            ts.Disk.error_log.Content += gold_file
+
         # access_log validation.
         access_log = log_validation.get('access_log', {})
         if 'gold_file' in access_log:
@@ -182,6 +197,8 @@ def ATSReplayTest(obj, replay_file: str):
             dns = tr.MakeDNServer(name, **process_config)
         else:
             dns = tr.MakeDNServer(name, default='127.0.0.1')
+        if 'records' in dns_config:
+            dns.addRecords(dns_config['records'])
 
     # Proxy Verifier Server configuration.
     if not 'server' in autest_config:
@@ -215,6 +232,7 @@ def ATSReplayTest(obj, replay_file: str):
     ats_config = autest_config['ats']
     enable_tls = ats_config.get('enable_tls', False)
     metric_checks = ats_config.get('metric_checks', [])
+    log_validation = ats_config.get('log_validation', None)
 
     ats_owner = obj if _requires_persistent_ats(ats_config) else tr
     ts = configure_ats(ats_owner, server=server, ats_config=ats_config, dns=dns)
@@ -272,6 +290,13 @@ def ATSReplayTest(obj, replay_file: str):
                 f'^{re.escape(metric_name)}\\s+{expected_value}$', f'{metric_name} should be {expected_value}')
             check_tr.StillRunningAfter = ts
 
+    # wait for error log
+    if log_validation and log_validation.get('error_log'):
+        wait_for_log = obj.AddTestRun('Wait for logs of ' + ats_config.get('name', 'ts'))
+        wait_for_log.Processes.Default.Command = (
+            os.path.join(obj.Variables.AtsTestToolsDir, 'condwait') + ' 60 1 -f ' + os.path.join(ts.Variables.LOGDIR, 'error.log'))
+        wait_for_log.Processes.Default.ReturnCode = 0
+
     return tr
 
 

tests/gold_tests/dns/connect_attempts.test.py

@@ -0,0 +1,31 @@
+'''
+Verify Origin Server Connect Attempts Behavior
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+Test.Summary = '''
+Verify Origin Server Connect Attempts Behavior
+'''
+
+# No retry
+Test.ATSReplayTest(replay_file="replay/connect_attempts_rr_no_retry.replay.yaml")
+
+# max_retries
+Test.ATSReplayTest(replay_file="replay/connect_attempts_rr_max_retries.replay.yaml")
+
+# rr_retries
+Test.ATSReplayTest(replay_file="replay/connect_attempts_rr_retries.replay.yaml")

tests/gold_tests/dns/gold/connect_attempts_rr_max_retries_error_log.gold

@@ -0,0 +1,8 @@
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=0 `` retry_attempts=0 url='http://backend.example.com:``/path/'
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=0 `` retry_attempts=1 url='http://backend.example.com:``/path/'
+`` CONNECT : `` connecting to 0.0.0.1:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='1' marking down
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=1 `` retry_attempts=0 url='http://backend.example.com:``/path/'
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=1 `` retry_attempts=1 url='http://backend.example.com:``/path/'
+`` CONNECT : `` connecting to 0.0.0.2:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='1' marking down
+`` DNS Error: no valid server http://backend.example.com:``/path/
+``

tests/gold_tests/dns/gold/connect_attempts_rr_no_error_log.gold

@@ -0,0 +1,6 @@
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=0 `` retry_attempts=0 url='http://backend.example.com:``/path/'
+`` CONNECT : `` connecting to 0.0.0.1:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='1' marking down
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=1 `` retry_attempts=0 url='http://backend.example.com:``/path/'
+`` CONNECT : `` connecting to 0.0.0.2:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='1' marking down
+`` DNS Error: no valid server http://backend.example.com:``/path/
+``

tests/gold_tests/dns/gold/connect_attempts_rr_retries_error_log.gold

@@ -0,0 +1,8 @@
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=0 connection_result=`` error=`` retry_attempts=0 url='http://backend.example.com:``/path/'
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.1:`` for host='example.com' sm_id=1 connection_result=`` error=`` retry_attempts=0 url='http://backend.example.com:``/path/'
+`` CONNECT : `` connecting to 0.0.0.1:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='2' marking down
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=2 connection_result=`` error=`` retry_attempts=0 url='http://backend.example.com:``/path/'
+`` CONNECT: attempt fail [CONNECTION_ERROR] to 0.0.0.2:`` for host='example.com' sm_id=3 connection_result=`` error=`` retry_attempts=0 url='http://backend.example.com:``/path/'
+`` CONNECT : `` connecting to 0.0.0.2:`` for host='example.com' url='http://backend.example.com:``/path/' fail_count='2' marking down
+`` DNS Error: no valid server http://backend.example.com:``/path/
+``

tests/gold_tests/dns/replay/connect_attempts_rr_max_retries.replay.yaml

@@ -0,0 +1,129 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+meta:
+  version: "1.0"
+
+# Configuration section for autest integration
+autest:
+  description: 'Verify connect attempts behavior - round robin'
+
+  dns:
+    name: 'dns-rr'
+    records: {"backend.example.com": ["0.0.0.1", "0.0.0.2"]}
+
+  server:
+    name: 'server-rr'
+
+  client:
+    name: 'client-rr'
+
+  ats:
+    name: 'ts-rr'
+    process_config:
+      enable_cache: false
+
+    records_config:
+      proxy.config.diags.debug.enabled: 1
+      proxy.config.diags.debug.tags: 'http|hostdb|dns'
+      proxy.config.http.connect_attempts_rr_retries: 0
+      proxy.config.http.connect_attempts_max_retries: 1
+      proxy.config.http.connect_attempts_max_retries_down_server: 0
+      proxy.config.http.connect_attempts_timeout: 1
+      proxy.config.http.down_server.cache_time: 10
+
+    remap_config:
+      - from: "http://example.com/"
+        to: "http://backend.example.com:{SERVER_HTTP_PORT}/"
+
+    log_validation:
+      error_log:
+        gold_file: "gold/connect_attempts_rr_max_retries_error_log.gold"
+
+sessions:
+- transactions:
+  # try 0.0.0.1
+  - client-request:
+      method: GET
+      url: /path/
+      version: '1.1'
+      headers:
+        fields:
+        - [Host, example.com]
+        - [uuid, 1]
+
+    # should not hit
+    server-response:
+      status: 200
+      reason: OK
+
+    proxy-response:
+      status: 502
+
+  # try 0.0.0.2
+  - client-request:
+      method: GET
+      url: /path/
+      version: '1.1'
+      headers:
+        fields:
+        - [Host, example.com]
+        - [uuid, 2]
+
+    # should not hit
+    server-response:
+      status: 200
+      reason: OK
+
+    proxy-response:
+      status: 502
+
+  # The request is expected to hit the down_server cache and immediately receive a 500 response.
+  - client-request:
+      method: GET
+      url: /path/
+      version: '1.1'
+      headers:
+        fields:
+        - [Host, example.com]
+        - [uuid, 10]
+
+    # should not hit
+    server-response:
+      status: 200
+      reason: OK
+
+    proxy-response:
+      status: 500
+
+  # when down_server.cache_time is expired, try connect attempts
+  - client-request:
+      method: GET
+      url: /path/
+      version: '1.1'
+      headers:
+        fields:
+        - [Host, example.com]
+        - [uuid, 20]
+      delay: 10s
+
+    # should not hit
+    server-response:
+      status: 200
+      reason: OK
+
+    proxy-response:
+      status: 502

tests/gold_tests/dns/replay/connect_attempts_rr_no_retry.replay.yaml

@@ -0,0 +1,129 @@
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+meta:
+  version: "1.0"
+
+# Configuration section for autest integration
+autest:
+  description: 'Verify connect attempts behavior - no retry'
+
+  dns:
+    name: 'dns-no'
+    records: {"backend.example.com": ["0.0.0.1", "0.0.0.2"]}
+
+  server:
+    name: 'server-no'
+
+  client:
+    name: 'client-no'
+
+  ats:
+    name: 'ts-no'
+    process_config:
+      enable_cache: false
+
+    records_config:
+      proxy.config.diags.debug.enabled: 1
+      proxy.config.diags.debug.tags: 'http|hostdb|dns'
+      proxy.config.http.connect_attempts_rr_retries: 0
+      proxy.config.http.connect_attempts_max_retries: 0
+      proxy.config.http.connect_attempts_max_retries_down_server: 0
+      proxy.config.http.connect_attempts_timeout: 1
+      proxy.config.http.down_server.cache_time: 5
+
+    remap_config:
+      - from: "http://example.com/"
+        to: "http://backend.example.com:{SERVER_HTTP_PORT}/"
+
+    log_validation:
+      error_log:
+        gold_file: "gold/connect_attempts_rr_no_error_log.gold"
+
+sessions:
+- transactions:
+  # try 0.0.0.1
+  - client-request:
+      method: GET
+      url: /path/
+      version: '1.1'
+      headers:
+        fields:
+        - [Host, example.com]
+        - [uuid, 1]
+
+    # should not hit
+    server-response:
+      status: 200
+      reason: OK
+
+    proxy-response:
+      status: 502
+
+  # try 0.0.0.2
+  - client-request:
+      method: GET
+      url: /path/
+      version: '1.1'
+      headers:
+        fields:
+        - [Host, example.com]
+        - [uuid, 2]
+
+    # should not hit
+    server-response:
+      status: 200
+      reason: OK
+
+    proxy-response:
+      status: 502
+
+  # This request is expected to hit the down_server cache and immediately receive a 500 response.
+  - client-request:
+      method: GET
+      url: /path/
+      version: '1.1'
+      headers:
+        fields:
+        - [Host, example.com]
+        - [uuid, 10]
+
+    # should not hit
+    server-response:
+      status: 200
+      reason: OK
+
+    proxy-response:
+      status: 500
+
+  # when down_server.cache_time is expired, try connect attempts
+  - client-request:
+      method: GET
+      url: /path/
+      version: '1.1'
+      headers:
+        fields:
+        - [Host, example.com]
+        - [uuid, 20]
+      delay: 10s
+
+    # should not hit
+    server-response:
+      status: 200
+      reason: OK
+
+    proxy-response:
+      status: 502