feat(api): New MaskString

src/pyinfra/api/__init__.py

@@ -2,11 +2,12 @@
     FileDownloadCommand,  # noqa: F401 # pragma: no cover
     FileUploadCommand,
     FunctionCommand,
-    MaskString,
     QuoteString,
     RsyncCommand,
     StringCommand,
+    MaskString,
 )
+from .hiddenvalue import HiddenValue
 from .config import Config  # noqa: F401 # pragma: no cover
 from .deploy import deploy  # noqa: F401 # pragma: no cover
 from .exceptions import (  # noqa: F401

src/pyinfra/api/command.py

@@ -5,11 +5,12 @@
 from string import Formatter
 from typing import IO, TYPE_CHECKING
 from collections.abc import Callable
-
 import gevent
 from typing_extensions import Unpack, override
 
 from pyinfra.context import LocalContextObject, ctx_config, ctx_host
+from pyinfra import logger
+from .hiddenvalue import HiddenValue
 
 from .arguments import ConnectorArguments
 
@@ -55,13 +56,15 @@ def make_formatted_string_command(string: str, *args, **kwargs) -> StringCommand
 
 
 class MaskString(str):
-    pass
+    def __new__(cls, s):
+        logger.warning("MaskString is deprecated please switch to HiddenValue")
+        return super().__new__(cls, s)
 
 
 class QuoteString:
-    obj: str | StringCommand
+    obj: str | StringCommand | HiddenValue
 
-    def __init__(self, obj: str | StringCommand):
+    def __init__(self, obj: str | StringCommand | HiddenValue):
         self.obj = obj
 
     @override
@@ -104,7 +107,7 @@ def __str__(self) -> str:
     def __repr__(self) -> str:
         return f"StringCommand({self.get_masked_value()})"
 
-    def _get_all_bits(self, bit_accessor):
+    def _get_all_bits(self, bit_accessor, *, unmask=False):
         all_bits = []
 
         for bit in self.bits:
@@ -116,6 +119,13 @@ def _get_all_bits(self, bit_accessor):
             if isinstance(bit, StringCommand):
                 bit = bit_accessor(bit)
 
+            if unmask:
+                if isinstance(bit, HiddenValue):
+                    bit = bit.unmask()
+            else:
+                if isinstance(bit, MaskString):
+                    bit = "*MASKED*"
+
             if not isinstance(bit, str):
                 bit = f"{bit}"
 
@@ -128,18 +138,11 @@ def _get_all_bits(self, bit_accessor):
 
     def get_raw_value(self) -> str:
         return self.separator.join(
-            self._get_all_bits(
-                lambda bit: bit.get_raw_value(),
-            ),
+            self._get_all_bits(lambda bit: bit.get_raw_value(), unmask=True),
         )
 
     def get_masked_value(self) -> str:
-        return self.separator.join(
-            [
-                "***" if isinstance(bit, MaskString) else bit
-                for bit in self._get_all_bits(lambda bit: bit.get_masked_value())
-            ],
-        )
+        return self.separator.join(self._get_all_bits(lambda bit: bit.get_masked_value()))
 
     @override
     def execute(self, state: State, host: Host, connector_arguments: ConnectorArguments):

src/pyinfra/api/hiddenvalue.py

@@ -0,0 +1,23 @@
+from typing_extensions import override
+
+
+class HiddenValue:
+    """
+    A class to contain a hidden value
+    To retrieve the real value use .unmask()
+    """
+
+    def unmask(self) -> str:
+        return self.raw_value
+
+    def __init__(self, content="", masked_value="*MASKED*"):
+        self.masked_value = masked_value
+        self.raw_value = content
+
+    @override
+    def __str__(self) -> str:
+        return str(self.masked_value)
+
+    @override
+    def __repr__(self) -> str:
+        return repr(self.masked_value)

src/pyinfra/connectors/util.py

@@ -11,7 +11,7 @@
 
 from pyinfra import logger
 from pyinfra.api.output import echo, format_text
-from pyinfra.api import MaskString, QuoteString, StringCommand
+from pyinfra.api import HiddenValue, QuoteString, StringCommand
 from pyinfra.api.exceptions import PyinfraError
 from pyinfra.api.util import memoize
 
@@ -446,8 +446,10 @@ def make_unix_command(
             [
                 "env",
                 StringCommand("SUDO_ASKPASS=", QuoteString(_sudo_askpass_path), _separator=""),
-                MaskString(
-                    f"{SUDO_ASKPASS_ENV_VAR}={StringCommand(QuoteString(_sudo_password)).get_raw_value()}"
+                StringCommand(
+                    SUDO_ASKPASS_ENV_VAR,
+                    QuoteString(HiddenValue(_sudo_password)),
+                    _separator="=",
                 ),
             ],
         )
@@ -474,8 +476,10 @@ def make_unix_command(
             command_bits.extend(
                 [
                     "env",
-                    MaskString(
-                        f"{SU_ASKPASS_ENV_VAR}={StringCommand(QuoteString(_su_password)).get_raw_value()}"
+                    StringCommand(
+                        SU_ASKPASS_ENV_VAR,
+                        QuoteString(HiddenValue(_su_password)),
+                        _separator="=",
                     ),
                     QuoteString(_su_askpass_path),
                     "|",

src/pyinfra/facts/mysql.py

@@ -5,7 +5,7 @@
 
 from typing_extensions import override
 
-from pyinfra.api import FactBase, MaskString, QuoteString, StringCommand
+from pyinfra.api import FactBase, HiddenValue, QuoteString, StringCommand
 from pyinfra.api.util import try_int
 
 from .util.databases import parse_columns_and_rows
@@ -30,7 +30,7 @@ def make_mysql_command(
 
     if password:
         # Quote the password as it may contain special characters
-        target_bits.append(MaskString(f'-p"{password}"'))
+        target_bits.append(StringCommand("-p", QuoteString(HiddenValue(password)), _separator=""))
 
     if host:
         target_bits.append(f"-h{host}")

src/pyinfra/facts/postgres.py

@@ -2,7 +2,7 @@
 
 from typing_extensions import override
 
-from pyinfra.api import FactBase, MaskString, QuoteString, StringCommand
+from pyinfra.api import FactBase, HiddenValue, QuoteString, StringCommand
 from pyinfra.api.util import try_int
 
 from .util.databases import parse_columns_and_rows
@@ -16,10 +16,16 @@ def make_psql_command(
     port: str | int | None = None,
     executable="psql",
 ) -> StringCommand:
-    target_bits: list[str] = []
+    target_bits: list[str | StringCommand] = []
 
     if password:
-        target_bits.append(MaskString(f'PGPASSWORD="{password}"'))
+        target_bits.append(
+            StringCommand(
+                "PGPASSWORD",
+                QuoteString(HiddenValue(password)),
+                _separator="=",
+            )
+        )
 
     target_bits.append(executable)
 

src/pyinfra/operations/docker.py

@@ -6,10 +6,8 @@
 
 from __future__ import annotations
 
-from shlex import quote as shlex_quote
-
 from pyinfra import host
-from pyinfra.api import MaskString, OperationError, QuoteString, StringCommand, operation
+from pyinfra.api import HiddenValue, OperationError, QuoteString, StringCommand, operation
 from pyinfra.facts.docker import (
     DockerAuths,
     DockerContainer,
@@ -752,7 +750,7 @@ def login(
 
     command_bits: list = [
         "printf '%s'",
-        MaskString(shlex_quote(password)),
+        QuoteString(HiddenValue(password)),
         "| docker login --username",
         QuoteString(username),
         "--password-stdin",

src/pyinfra/operations/mysql.py

@@ -16,7 +16,7 @@
 from __future__ import annotations
 
 from pyinfra import host
-from pyinfra.api import MaskString, OperationError, QuoteString, StringCommand, operation
+from pyinfra.api import HiddenValue, OperationError, QuoteString, StringCommand, operation
 from pyinfra.facts.mysql import (
     MysqlDatabases,
     MysqlUserGrants,
@@ -171,9 +171,13 @@ def user(
         return
 
     if present and not is_present:
-        sql_bits = [f'CREATE USER "{user}"@"{user_hostname}"']
+        sql_bits: list[str | StringCommand] = [f'CREATE USER "{user}"@"{user_hostname}"']
         if password:
-            sql_bits.append(MaskString(f'IDENTIFIED BY "{password}"'))
+            sql_bits.append(
+                StringCommand(
+                    "IDENTIFIED BY", StringCommand('"', HiddenValue(password), '"', _separator="")
+                )
+            )
 
         if require == "SSL":
             sql_bits.append("REQUIRE SSL")

src/pyinfra/operations/postgres.py

@@ -17,7 +17,7 @@
 from __future__ import annotations
 
 from pyinfra import host
-from pyinfra.api import MaskString, QuoteString, StringCommand, operation
+from pyinfra.api import HiddenValue, QuoteString, StringCommand, operation
 from pyinfra.facts.postgres import (
     PostgresDatabases,
     PostgresRoles,
@@ -141,7 +141,7 @@ def role(
 
     # If we want the user and they don't exist
     if not is_present:
-        sql_bits = [f'CREATE ROLE "{role}"']
+        sql_bits: list[str | StringCommand] = [f'CREATE ROLE "{role}"']
 
         for key, value in (
             ("LOGIN", login),
@@ -158,7 +158,11 @@ def role(
             sql_bits.append(f"CONNECTION LIMIT {connection_limit}")
 
         if password:
-            sql_bits.append(MaskString(f"PASSWORD '{password}'"))
+            sql_bits.append(
+                StringCommand(
+                    "PASSWORD", StringCommand("'", HiddenValue(password), "'", _separator="")
+                )
+            )
 
         yield make_execute_psql_command(
             StringCommand(*sql_bits),
@@ -196,7 +200,11 @@ def role(
             sql_bits.append(f"CONNECTION LIMIT {connection_limit}")
             should_execute = True
         if password:
-            sql_bits.append(MaskString(f"PASSWORD '{password}'"))
+            sql_bits.append(
+                StringCommand(
+                    "PASSWORD", StringCommand("'", HiddenValue(password), "'", _separator="")
+                )
+            )
             should_execute = True
 
         if should_execute:

tests/facts/mysql.MysqlDatabases/multiple_custom_connection.json

@@ -1,8 +1,8 @@
 {
     "arg": ["myuser", "mypassword", "myhost", "myport"],
     "command": {
-        "raw": "mysql -u\"myuser\" -p\"mypassword\" -hmyhost -Pmyport -Be 'SELECT * FROM information_schema.SCHEMATA'",
-        "masked": "mysql -u\"myuser\" *** -hmyhost -Pmyport -Be 'SELECT * FROM information_schema.SCHEMATA'"
+        "raw": "mysql -u\"myuser\" -pmypassword -hmyhost -Pmyport -Be 'SELECT * FROM information_schema.SCHEMATA'",
+        "masked": "mysql -u\"myuser\" -p'*MASKED*' -hmyhost -Pmyport -Be 'SELECT * FROM information_schema.SCHEMATA'"
     },
     "requires_command": "mysql",
     "output": [

tests/facts/postgresql.PostgresqlRoles/multiple_custom_connection.json

@@ -1,8 +1,8 @@
 {
     "arg": ["myuser", "mypassword", "myhost", "myport"],
     "command": {
-        "raw": "PGPASSWORD=\"mypassword\" psql -U myuser -h myhost -p myport -Ac 'SELECT * FROM pg_catalog.pg_roles'",
-        "masked": "*** psql -U myuser -h myhost -p myport -Ac 'SELECT * FROM pg_catalog.pg_roles'"
+        "raw": "PGPASSWORD=mypassword psql -U myuser -h myhost -p myport -Ac 'SELECT * FROM pg_catalog.pg_roles'",
+        "masked": "PGPASSWORD='*MASKED*' psql -U myuser -h myhost -p myport -Ac 'SELECT * FROM pg_catalog.pg_roles'"
     },
     "requires_command": "psql",
     "output": [

tests/operations/docker.login/login_force_reauth.json

@@ -8,7 +8,7 @@
     "commands": [
         {
             "raw": "printf '%s' newpw | docker login --username ci --password-stdin myregistry.io:5000",
-            "masked": "printf '%s' *** | docker login --username ci --password-stdin myregistry.io:5000"
+            "masked": "printf '%s' '*MASKED*' | docker login --username ci --password-stdin myregistry.io:5000"
         }
     ]
 }

tests/operations/docker.login/login_to_docker_hub.json

@@ -9,7 +9,7 @@
     "commands": [
         {
             "raw": "printf '%s' s3cret | docker login --username ci --password-stdin",
-            "masked": "printf '%s' *** | docker login --username ci --password-stdin"
+            "masked": "printf '%s' '*MASKED*' | docker login --username ci --password-stdin"
         }
     ]
 }

tests/operations/docker.login/login_to_private_registry.json

@@ -10,7 +10,7 @@
     "commands": [
         {
             "raw": "printf '%s' s3cret | docker login --username ci --password-stdin myregistry.io:5000",
-            "masked": "printf '%s' *** | docker login --username ci --password-stdin myregistry.io:5000"
+            "masked": "printf '%s' '*MASKED*' | docker login --username ci --password-stdin myregistry.io:5000"
         }
     ]
 }

tests/operations/docker.login/login_with_special_chars_password.json

@@ -10,7 +10,7 @@
     "commands": [
         {
             "raw": "printf '%s' 's!%^&$' | docker login --username ci --password-stdin myregistry.io:5000",
-            "masked": "printf '%s' *** | docker login --username ci --password-stdin myregistry.io:5000"
+            "masked": "printf '%s' '*MASKED*' | docker login --username ci --password-stdin myregistry.io:5000"
         }
     ]
 }

tests/operations/mysql.dump/dump.json

@@ -7,8 +7,8 @@
     },
     "commands": [
         {
-            "raw": "mysqldump somedb -u\"root\" -p\"somepass\" > somefile",
-            "masked": "mysqldump somedb -u\"root\" *** > somefile"
+            "raw": "mysqldump somedb -u\"root\" -psomepass > somefile",
+            "masked": "mysqldump somedb -u\"root\" -p'*MASKED*' > somefile"
         }
     ],
     "idempotent": false

tests/operations/mysql.sql/execute_mysql_kwargs.json

@@ -8,8 +8,8 @@
     },
     "commands": [
         {
-            "raw": "mysql -u\"someuser\" -p\"somepass\" -h127.0.0.1 -P100 -Be 'SELECT 1'",
-            "masked": "mysql -u\"someuser\" *** -h127.0.0.1 -P100 -Be 'SELECT 1'"
+            "raw": "mysql -u\"someuser\" -psomepass -h127.0.0.1 -P100 -Be 'SELECT 1'",
+            "masked": "mysql -u\"someuser\" -p'*MASKED*' -h127.0.0.1 -P100 -Be 'SELECT 1'"
         }
     ],
     "idempotent": false

tests/operations/mysql.user/add_password.json

@@ -12,7 +12,7 @@
     "commands": [
         {
             "raw": "mysql -Be 'CREATE USER \"someuser\"@\"localwhat\" IDENTIFIED BY \"mypass\"'",
-            "masked": "mysql -Be 'CREATE USER \"someuser\"@\"localwhat\" ***'"
+            "masked": "mysql -Be 'CREATE USER \"someuser\"@\"localwhat\" IDENTIFIED BY \"*MASKED*\"'"
         }
     ]
 }

tests/operations/postgresql.dump/dump.json

@@ -9,8 +9,8 @@
     },
     "commands": [
         {
-            "raw": "PGPASSWORD=\"somepass\" pg_dump -d somedb -U root -h localhost -p 3306 > somefile",
-            "masked": "*** pg_dump -d somedb -U root -h localhost -p 3306 > somefile"
+            "raw": "PGPASSWORD=somepass pg_dump -d somedb -U root -h localhost -p 3306 > somefile",
+            "masked": "PGPASSWORD='*MASKED*' pg_dump -d somedb -U root -h localhost -p 3306 > somefile"
         }
     ],
     "idempotent": false

tests/operations/postgresql.load/load.json

@@ -7,8 +7,8 @@
     },
     "commands": [
         {
-            "raw": "PGPASSWORD=\"somepass\" psql -d somedb -U root < somefile",
-            "masked": "*** psql -d somedb -U root < somefile"
+            "raw": "PGPASSWORD=somepass psql -d somedb -U root < somefile",
+            "masked": "PGPASSWORD='*MASKED*' psql -d somedb -U root < somefile"
         }
     ],
     "idempotent": false