From bddb56aa7c1590819272ef57ce021335b8972e1d Mon Sep 17 00:00:00 2001 From: Tatsuhiro Tsujikawa Date: Sat, 23 Jul 2022 17:27:30 +0900 Subject: [PATCH] Fix flaky compliance checks Previously, the compliance check seeks for the strings "exited with code 127" or "exit status 127" in stdout + stderr. It turns out that these strings might not be present in rare cases. In order to workaround this, this commit directly checks the exit code of the relevant container with docker-compose --exit-code-from flag. The flag implies --abort-on-container-exit. When running the actual test case, the interop runner checks whether the test case is supported by an implementation. The same method cannot be applied there because we only get an exit code from a single service. However, the downside of not detecting unsupported test case is not severe, it just results in failed test. In contrast, the failed compliance check skips all test cases for the particular client and server combination. --- interop.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/interop.py b/interop.py index 4dea51d3..9b0d9b0e 100644 --- a/interop.py +++ b/interop.py @@ -130,12 +130,12 @@ def _check_impl_is_compliant(self, name: str) -> bool: "DOWNLOADS=" + downloads_dir.name + " " 'SCENARIO="simple-p2p --delay=15ms --bandwidth=10Mbps --queue=25" ' "CLIENT=" + self._implementations[name]["image"] + " " - "docker-compose up --timeout 0 --abort-on-container-exit -V sim client" + "docker-compose up --timeout 0 --exit-code-from client -V sim client" ) output = subprocess.run( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - if not self._is_unsupported(output.stdout.splitlines()): + if output.returncode != 127: logging.error("%s client not compliant.", name) logging.debug("%s", output.stdout.decode("utf-8")) self.compliant[name] = False @@ -153,12 +153,12 @@ def _check_impl_is_compliant(self, name: str) -> bool: "WWW=" + www_dir.name + " " "DOWNLOADS=" + downloads_dir.name + " " "SERVER=" + self._implementations[name]["image"] + " " - "docker-compose up -V server" + "docker-compose up --exit-code-from server -V server" ) output = subprocess.run( cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT ) - if not self._is_unsupported(output.stdout.splitlines()): + if output.returncode != 127: logging.error("%s server not compliant.", name) logging.debug("%s", output.stdout.decode("utf-8")) self.compliant[name] = False