pants-plugins/uses_services: respect ST2_DATABASE__* vars (#6278)

Author: cognifloyd

CHANGELOG.rst

@@ -69,7 +69,7 @@ Added
   working on StackStorm, improve our security posture, and improve CI reliability thanks in part
   to pants' use of PEX lockfiles. This is not a user-facing addition.
   #6118 #6141 #6133 #6120 #6181 #6183 #6200 #6237 #6229 #6240 #6241 #6244 #6251 #6253
-  #6254 #6258 #6259 #6260 #6269 #6275 #6279
+  #6254 #6258 #6259 #6260 #6269 #6275 #6279 #6278
   Contributed by @cognifloyd
 * Build of ST2 EL9 packages #6153
   Contributed by @amanda11

pants-plugins/uses_services/exceptions.py

@@ -34,6 +34,8 @@ class ServiceSpecificMessages:
 
     service_start_cmd_generic: str
 
+    env_vars_hint: str = ""
+
 
 class ServiceMissingError(Exception):
     """Error raised when a test uses a service but that service is missing."""
@@ -162,6 +164,9 @@ def generate(
                 """
             )
 
+        if messages.env_vars_hint:
+            instructions += f"\n\n{messages.env_vars_hint}"
+
         return cls(
             service=service,
             platform=platform,

pants-plugins/uses_services/mongo_rules.py

@@ -11,6 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+from __future__ import annotations
 
 from dataclasses import dataclass
 from textwrap import dedent
@@ -27,6 +28,8 @@
     VenvPexProcess,
     rules as pex_rules,
 )
+from pants.core.goals.test import TestExtraEnv
+from pants.engine.env_vars import EnvironmentVars
 from pants.engine.fs import CreateDigest, Digest, FileContent
 from pants.engine.rules import collect_rules, Get, MultiGet, rule
 from pants.engine.process import FallibleProcessResult, ProcessCacheScope
@@ -55,21 +58,50 @@ class UsesMongoRequest:
     #   for unit tests: st2tests/st2tests/config.py
     #   for integration tests: conf/st2.tests*.conf st2tests/st2tests/fixtures/conf/st2.tests*.conf
     #       (changed by setting ST2_CONFIG_PATH env var inside the tests)
-    # TODO: for unit tests: modify code to pull db connect settings from env vars
-    # TODO: for int tests: modify st2.tests*.conf on the fly to set the per-pantsd-slot db_name
-    #                      and either add env vars for db connect settings or modify conf files as well
-
-    #   with our version of oslo.config (newer are slower) we can't directly override opts w/ environment variables.
+    # These can also be updated via the ST2_DATABASE__* env vars (which oslo_config reads).
+    # Integration tests should pass these changes onto subprocesses via the same env vars.
 
     db_host: str = "127.0.0.1"  # localhost in test_db.DbConnectionTestCase
     db_port: int = 27017
     # db_name is "st2" in test_db.DbConnectionTestCase
     db_name: str = "st2-test{}"  # {} will be replaced by test slot (a format string)
 
+    # username and password are not required to validate connectivity, so this doesn't have them.
+
     db_connection_timeout: int = 3000
 
     execution_slot_var: str = "ST2TESTS_PARALLEL_SLOT"
 
+    @classmethod
+    def from_env(
+        cls, execution_slot_var: str, env: EnvironmentVars
+    ) -> UsesMongoRequest:
+        default = cls()
+        host = env.get("ST2_DATABASE__HOST", default.db_host)
+        port_raw = env.get("ST2_DATABASE__PORT", str(default.db_port))
+        db_name = default.db_name  # not overridable via ST2_DATABASE__DB_NAME
+        db_connection_timeout_raw = env.get(
+            "ST2_DATABASE__CONNECTION_TIMEOUT", str(default.db_connection_timeout)
+        )
+
+        try:
+            port = int(port_raw)
+        except (TypeError, ValueError):
+            port = default.db_port
+
+        try:
+            db_connection_timeout = int(db_connection_timeout_raw)
+        except (TypeError, ValueError):
+            db_connection_timeout = default.db_connection_timeout
+
+        return cls(
+            db_host=host,
+            db_port=port,
+            db_name=db_name,
+            db_connection_timeout=db_connection_timeout,
+            execution_slot_var=execution_slot_var,
+        )
+
 
 @dataclass(frozen=True)
 class MongoIsRunning:
@@ -90,7 +122,9 @@ def is_applicable(cls, target: Target) -> bool:
     level=LogLevel.DEBUG,
 )
 async def mongo_is_running_for_pytest(
-    request: PytestUsesMongoRequest, pytest: PyTest
+    request: PytestUsesMongoRequest,
+    pytest: PyTest,
+    test_extra_env: TestExtraEnv,
 ) -> PytestPluginSetup:
     # TODO: delete these comments once the Makefile becomes irrelevant.
     #       the comments explore how the Makefile prepares to run and runs tests
@@ -109,7 +143,9 @@ async def mongo_is_running_for_pytest(
     # this will raise an error if mongo is not running
     _ = await Get(
         MongoIsRunning,
-        UsesMongoRequest(execution_slot_var=pytest.execution_slot_var or ""),
+        UsesMongoRequest.from_env(
+            execution_slot_var=pytest.execution_slot_var or "", env=test_extra_env.env
+        ),
     )
 
     return PytestPluginSetup()
@@ -151,8 +187,8 @@ async def mongo_is_running(
                 str(request.db_port),
                 request.db_name,
                 str(request.db_connection_timeout),
-                request.execution_slot_var,
             ),
+            extra_env={"PANTS_PYTEST_EXECUTION_SLOT_VAR": request.execution_slot_var},
             input_digest=script_digest,
             execution_slot_variable=request.execution_slot_var,
             description="Checking to see if Mongo is up and accessible.",
@@ -200,6 +236,18 @@ async def mongo_is_running(
                 """
             ),
             service_start_cmd_generic="systemctl start mongod",
+            env_vars_hint=dedent(
+                """\
+                You can also export the ST2_DATABASE__HOST and ST2_DATABASE__PORT
+                env vars to automatically use any MongoDB host, local or remote,
+                while running unit and integration tests. Note that you cannot
+                override the db-name, which is st2-test suffixed by an integer
+                to allow tests to safely run in parallel. If needed you can also
+                override the default username, password, and connection timeout
+                by exporting one or more of: ST2_DATABASE__USERNAME,
+                ST2_DATABASE__PASSWORD, and ST2_DATABASE__CONNECTION_TIMEOUT.
+                """
+            ),
         ),
     )
 

pants-plugins/uses_services/mongo_rules_test.py

@@ -51,7 +51,17 @@ def run_mongo_is_running(
             "--backend-packages=uses_services",
             *(extra_args or ()),
         ],
-        env_inherit={"PATH", "PYENV_ROOT", "HOME"},
+        env_inherit={
+            "PATH",
+            "PYENV_ROOT",
+            "HOME",
+            "ST2_DATABASE__HOST",
+            "ST2_DATABASE__PORT",
+            "ST2_DATABASE__USERNAME",
+            "ST2_DATABASE__PASSWORD",
+            "ST2_DATABASE__CONNECTION_TIMEOUT",
+            "ST2TESTS_PARALLEL_SLOT",
+        },
     )
     result = rule_runner.request(
         MongoIsRunning,
@@ -62,7 +72,9 @@ def run_mongo_is_running(
 
 # Warning this requires that mongo be running
 def test_mongo_is_running(rule_runner: RuleRunner) -> None:
-    request = UsesMongoRequest()
+    request = UsesMongoRequest.from_env(
+        execution_slot_var="ST2TESTS_PARALLEL_SLOT", env=rule_runner.environment
+    )
     mock_platform = platform(os="TestMock")
 
     # we are asserting that this does not raise an exception

pants-plugins/uses_services/redis_rules.py

@@ -143,13 +143,6 @@ async def redis_is_running(
     if is_running:
         return RedisIsRunning()
 
-    env_vars_hint = dedent(
-        """
-        You can also export the ST2TESTS_REDIS_HOST and ST2TESTS_REDIS_PORT
-        env vars to automatically use any redis host, local or remote,
-        while running unit and integration tests.
-        """
-    )
     # redis is not running, so raise an error with instructions.
     raise ServiceMissingError.generate(
         platform=platform,
@@ -162,21 +155,24 @@ async def redis_is_running(
                 """\
                 sudo yum -y install redis
                 # Don't forget to start redis.
-
                 """
-            )
-            + env_vars_hint,
+            ),
             service_start_cmd_deb="systemctl start redis",
             not_installed_clause_deb="this is one way to install it:",
             install_instructions_deb=dedent(
                 """\
                 sudo apt-get install -y mongodb redis
                 # Don't forget to start redis.
-
                 """
-            )
-            + env_vars_hint,
+            ),
             service_start_cmd_generic="systemctl start redis",
+            env_vars_hint=dedent(
+                """\
+                You can also export the ST2TESTS_REDIS_HOST and ST2TESTS_REDIS_PORT
+                env vars to automatically use any redis host, local or remote,
+                while running unit and integration tests.
+                """
+            ),
         ),
     )
 

pants-plugins/uses_services/scripts/is_mongo_running.py

@@ -54,7 +54,9 @@ def _is_mongo_running(
     db_name = args.get(3, "st2-test{}")
     connection_timeout_ms = args.get(4, 3000)
 
-    slot_var = args.get(5, "ST2TESTS_PARALLEL_SLOT")
+    slot_var = os.environ.get(
+        "PANTS_PYTEST_EXECUTION_SLOT_VAR", "ST2TESTS_PARALLEL_SLOT"
+    )
     db_name = db_name.format(os.environ.get(slot_var) or "")
 
     is_running = _is_mongo_running(

pants.toml

@@ -240,6 +240,13 @@ extra_env_vars = [
   # Use this so that the test system does not require the stanley user.
   # For example: export ST2TESTS_SYSTEM_USER=${USER}
   "ST2TESTS_SYSTEM_USER",
+  # Use these to override MongoDB connection details
+  # "ST2_DATABASE__HOST", # Tests override this with "127.0.0.1"
+  "ST2_DATABASE__PORT",
+  # "ST2_DATABASE__DB_NAME", # Tests override this with: "st2-test{ST2TESTS_PARALLEL_SLOT}"
+  "ST2_DATABASE__CONNECTION_TIMEOUT",
+  "ST2_DATABASE__USERNAME",
+  "ST2_DATABASE__PASSWORD",
   # Use these to override the redis host and port
   "ST2TESTS_REDIS_HOST",
   "ST2TESTS_REDIS_PORT",