AMLII-2173 - Add DD_DOGSTATSD_URL support to datadogpy

Adds support for the unix:// and udp:// variants of DD_DOGSTATSD_URL.
Will only be applied if the host and port are their default values
and no socket_path has been provided.

Author: ddrthall

README.md

@@ -94,6 +94,11 @@ options = {
 initialize(**options)
 ```
 
+Alternatively, the environment variable `DD_DOGSTATSD_URL` can be used to define a udp connection:
+`DD_DOGSTATSD_URL=udp://localhost:8125`
+
+Manually supplying a host/port will take precedence over using this environment variable.
+
 See the full list of available [DogStatsD client instantiation parameters](https://docs.datadoghq.com/developers/dogstatsd/?code-lang=python#client-instantiation-parameters).
 
 #### Instantiate the DogStatsd client with UDS
@@ -110,6 +115,12 @@ options = {
 initialize(**options)
 ```
 
+Alternatively, the environment variable `DD_DOGSTATSD_URL` can be used to define a UDS connection:
+`DD_DOGSTATSD_URL=unix:///var/run/datadog/dsd.socket`
+
+As with the udp variant, manually supplying a statsd_socket_path will take precedence over the environment variable.
+
+
 #### Origin detection over UDP and UDS
 
 Origin detection is a method to detect which pod `DogStatsD` packets are coming from in order to add the pod's tags to the tag list.

datadog/dogstatsd/base.py

@@ -25,6 +25,11 @@
     # pypy has the same module, but capitalized.
     import Queue as queue  # type: ignore[no-redef]
 
+try:
+    from urllib.parse import urlparse  # type: ignore
+except ImportError:
+    # Python 2 has the same functionality stored under a different module.
+    from urlparse import urlparse  # type: ignore
 
 # pylint: disable=unused-import
 from typing import Optional, List, Text, Union
@@ -54,6 +59,7 @@
 UNIX_ADDRESS_SCHEME = "unix://"
 UNIX_ADDRESS_DATAGRAM_SCHEME = "unixgram://"
 UNIX_ADDRESS_STREAM_SCHEME = "unixstream://"
+WINDOWS_NAMEDPIPE_SCHEME = "\\\\.\\pipe\\"
 
 # Buffering-related values (in seconds)
 DEFAULT_BUFFERING_FLUSH_INTERVAL = 0.3
@@ -188,12 +194,19 @@ def __init__(
 
         >>> statsd = DogStatsd()
 
+        :envvar DD_DOGSTATSD_URL: the connection information for the dogstatsd server.
+        If set, it overrides the default values.
+        Example for UDP url: `DD_DOGSTATSD_URL=udp://localhost:8125`
+        Example for UDS: `DD_DOGSTATSD_URL=unix:///var/run/datadog/dsd.socket`
+        Windows named pipes are currently unsupported.
+        :type DD_DOGSTATSD_URL: string
+
         :envvar DD_AGENT_HOST: the host of the DogStatsd server.
-        If set, it overrides default value.
+        If set, it overrides default value. DD_DOGSTATSD_URL takes precedence over this value.
         :type DD_AGENT_HOST: string
 
         :envvar DD_DOGSTATSD_PORT: the port of the DogStatsd server.
-        If set, it overrides default value.
+        If set, it overrides default value. DD_DOGSTATSD_URL takes precedence over this value.
         :type DD_DOGSTATSD_PORT: integer
 
         :envvar DATADOG_TAGS: Tags to attach to every metric reported by dogstatsd client.
@@ -374,22 +387,8 @@ def __init__(
         # Check for deprecated option
         if max_buffer_size is not None:
             log.warning("The parameter max_buffer_size is now deprecated and is not used anymore")
-        # Check host and port env vars
-        agent_host = os.environ.get("DD_AGENT_HOST")
-        if agent_host and host == DEFAULT_HOST:
-            host = agent_host
 
-        dogstatsd_port = os.environ.get("DD_DOGSTATSD_PORT")
-        if dogstatsd_port and port == DEFAULT_PORT:
-            try:
-                port = int(dogstatsd_port)
-            except ValueError:
-                log.warning(
-                    "Port number provided in DD_DOGSTATSD_PORT env var is not an integer: \
-                %s, using %s as port number",
-                    dogstatsd_port,
-                    port,
-                )
+        host, port, socket_path = self._parse_env_connection_overrides(host, port, socket_path)
 
         # Assuming environment variables always override
         telemetry_host = os.environ.get("DD_TELEMETRY_HOST", telemetry_host)
@@ -601,6 +600,74 @@ def disable_telemetry(self):
     def enable_telemetry(self):
         self._telemetry = True
 
+    def _parse_env_connection_overrides(self, host, port, socket_path):
+        dogstatsd_url = os.environ.get("DD_DOGSTATSD_URL")
+
+        if (
+            host == DEFAULT_HOST
+            and port == DEFAULT_PORT
+            and socket_path is None
+            and dogstatsd_url is not None
+        ):
+            parsed = urlparse(dogstatsd_url)
+            # If all values are defaults, prefer DD_DOGSTATSD_URL if present.
+            if parsed.scheme == "unix":
+                log.debug(
+                    "Found a DD_DOGSTATSD_URL matching the uds syntax, "
+                    "setting socket path %s.", dogstatsd_url
+                )
+                return host, port, dogstatsd_url
+
+            elif dogstatsd_url.startswith(WINDOWS_NAMEDPIPE_SCHEME):
+                log.debug(
+                    "DD_DOGSTATSD_URL is configured to utilize a windows named pipe, "
+                    "which is not currently supported by datadogpy. Falling back to "
+                    "alternate connection identifiers."
+                )
+
+            elif parsed.scheme == "udp":
+                try:
+                    p_port = parsed.port
+                    # Python 2 doesn't automatically perform bounds checking on the port
+                    if p_port is None or p_port < 0 or p_port > 65535:
+                        log.debug("Invalid port number provided, reverting to default port")
+                        p_port = DEFAULT_PORT
+                except ValueError:
+                    log.debug("Invalid port number provided, reverting to default port")
+                    p_port = DEFAULT_PORT
+
+                log.debug(
+                    "Found a DD_DOGSTATSD_URL matching the udp sytnax, "
+                    "setting host and port %s:%d.", parsed.hostname, p_port
+                )
+
+                return parsed.hostname, p_port, socket_path
+            else:
+                log.debug(
+                    "Unable to parse DD_DOGSTATSD_URL, did you remember to prefix the url "
+                    "with 'unix://' or 'udp://'? Falling back to alternate "
+                    "connection identifiers."
+                )
+
+        # We either have some non-default values or no DD_DOGSTATSD_URL
+        # Check host and port env vars
+        agent_host = os.environ.get("DD_AGENT_HOST")
+        if agent_host and host == DEFAULT_HOST:
+            host = agent_host
+
+        dogstatsd_port = os.environ.get("DD_DOGSTATSD_PORT")
+        if dogstatsd_port and port == DEFAULT_PORT:
+            try:
+                port = int(dogstatsd_port)
+            except ValueError:
+                log.warning(
+                    "Port number provided in DD_DOGSTATSD_PORT env var is not an integer: \
+                %s, using %s as port number",
+                    dogstatsd_port,
+                    port,
+                )
+        return host, port, socket_path
+
     # Note: Invocations of this method should be thread-safe
     def _start_flush_thread(self):
         if self._disable_aggregation and self.disable_buffering:

doc/source/index.rst

@@ -40,6 +40,15 @@ Here's an example where the statsd host and port are configured as well::
     )
 
 
+If statsd_host and statsd_port are left at their default values and no socket_path alternative is supplied, 
+the DD_DOGSTATSD_URL environment variable, if it exists, will be used to determine the connection
+information. This must be a URL that start with either `udp://` (to connect using UDP) or with `unix://` 
+(to use a Unix Domain Socket).
+
+* Example for UDP url: `DD_DOGSTATSD_URL=udp://localhost:8125`
+* Example for UDS: `DD_DOGSTATSD_URL=unix:///var/run/datadog/dsd.socket`
+
+
 .. autofunction:: datadog.initialize
 
 

tests/unit/dogstatsd/test_statsd.py

@@ -30,7 +30,7 @@
 # Datadog libraries
 from datadog import initialize, statsd
 from datadog import __version__ as version
-from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DogStatsd, MIN_SEND_BUFFER_SIZE, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_OPTIMAL_PAYLOAD_LENGTH
+from datadog.dogstatsd.base import DEFAULT_BUFFERING_FLUSH_INTERVAL, DEFAULT_HOST, DEFAULT_PORT, DogStatsd, MIN_SEND_BUFFER_SIZE, UDP_OPTIMAL_PAYLOAD_LENGTH, UDS_OPTIMAL_PAYLOAD_LENGTH
 from datadog.dogstatsd.context import TimedContextManagerDecorator
 from datadog.util.compat import is_higher_py35, is_p3k
 from tests.util.contextmanagers import preserve_environment_variable, EnvVars
@@ -296,7 +296,7 @@ def test_initialization(self):
         initialize(**options)
         self.assertEqual(statsd.cardinality, 'none')
 
-    def test_dogstatsd_initialization_with_env_vars(self):
+    def test_dogstatsd_initialization_with_env_vars_agent_host(self):
         """
         Dogstatsd can retrieve its config from env vars when
         not provided in constructor.
@@ -312,6 +312,54 @@ def test_dogstatsd_initialization_with_env_vars(self):
         self.assertEqual(dogstatsd.host, "myenvvarhost")
         self.assertEqual(dogstatsd.port, 4321)
 
+
+    def test_dogstatsd_initialization_with_env_vars_dogstatsd_url(self):
+        """
+        Dogstatsd can retrieve its config from env vars when
+        not provided in constructor.
+        """
+        # Setup UDP
+        with preserve_environment_variable('DD_DOGSTATSD_URL'):
+            os.environ['DD_DOGSTATSD_URL'] = 'udp://myenvvarhost:4321'
+            dogstatsd = DogStatsd()
+
+        # Assert
+        self.assertEqual(dogstatsd.host, "myenvvarhost")
+        self.assertEqual(dogstatsd.port, 4321)
+        self.assertEqual(dogstatsd.socket_path, None)
+
+        # Test UDS
+        with preserve_environment_variable('DD_DOGSTATSD_URL'):
+            os.environ['DD_DOGSTATSD_URL'] = 'unix:///hello/world.sock'
+            dogstatsd = DogStatsd()
+        self.assertEqual(dogstatsd.socket_path, 'unix:///hello/world.sock')
+        self.assertEqual(dogstatsd.host, None)
+        self.assertEqual(dogstatsd.port, None)
+
+        # Test non-default host
+        with preserve_environment_variable('DD_DOGSTATSD_URL'):
+            os.environ['DD_DOGSTATSD_URL'] = 'unix:///hello/world.sock'
+            dogstatsd = DogStatsd(host="myhost")
+        self.assertEqual(dogstatsd.socket_path, None)
+        self.assertEqual(dogstatsd.host, 'myhost')
+        self.assertEqual(dogstatsd.port, DEFAULT_PORT)
+
+        # Test non-default port
+        with preserve_environment_variable('DD_DOGSTATSD_URL'):
+            os.environ['DD_DOGSTATSD_URL'] = 'unix:///hello/world.sock'
+            dogstatsd = DogStatsd(port=8240)
+        self.assertEqual(dogstatsd.socket_path, None)
+        self.assertEqual(dogstatsd.host, DEFAULT_HOST)
+        self.assertEqual(dogstatsd.port, 8240)
+
+        # Test non-default socket_path
+        with preserve_environment_variable('DD_DOGSTATSD_URL'):
+            os.environ['DD_DOGSTATSD_URL'] = 'unix:///hello/world.sock'
+            dogstatsd = DogStatsd(socket_path='unix:///var/run/datadog/dsd.sock')
+        self.assertEqual(dogstatsd.socket_path, 'unix:///var/run/datadog/dsd.sock')
+        self.assertEqual(dogstatsd.host, None)
+        self.assertEqual(dogstatsd.port, None)
+
     def test_initialization_closes_socket(self):
         statsd.socket = FakeSocket()
         self.assertIsNotNone(statsd.socket)
@@ -2178,13 +2226,13 @@ def test_fake_sockets(self):
         """
         statsd = DogStatsd(disable_buffering=True)
 
-        class fakeSock:
+        class FakeSock:
             def __init__(self, id):
                 self.id = id
             def send(self, _):
                 pass
-        statsd.socket = fakeSock(5)
-        statsd.telemetry_socket = fakeSock(10)
+        statsd.socket = FakeSock(5)
+        statsd.telemetry_socket = FakeSock(10)
 
         assert statsd.socket.id == 5
         assert statsd.telemetry_socket.id == 10