diff --git a/.ddev/config.toml b/.ddev/config.toml index 17e00efe4d138..c0e871ceb6fa6 100644 --- a/.ddev/config.toml +++ b/.ddev/config.toml @@ -70,9 +70,8 @@ oauthlib = ['BSD-3-Clause'] mmh3 = ['CC0-1.0'] # https://github.com/paramiko/paramiko/blob/master/LICENSE paramiko = ['LGPL-2.1-only'] -# https://github.com/psycopg/psycopg2/blob/master/LICENSE -# https://github.com/psycopg/psycopg2/blob/master/doc/COPYING.LESSER -psycopg2-binary = ['LGPL-3.0-only', 'BSD-3-Clause'] +# https://github.com/psycopg/psycopg/blob/master/LICENSE.txt +psycopg = ['LGPL-3.0-only'] # https://github.com/Legrandin/pycryptodome/blob/master/LICENSE.rst pycryptodomex = ['Unlicense', 'BSD-2-Clause'] # https://github.com/mongodb/mongo-python-driver/blob/master/LICENSE @@ -105,7 +104,6 @@ lxml = 'https://github.com/lxml/lxml' packaging = 'https://github.com/pypa/packaging' paramiko = 'https://github.com/paramiko/paramiko' protobuf = 'https://github.com/protocolbuffers/protobuf' -psycopg2-binary = 'https://github.com/psycopg/psycopg2' pycryptodomex = 'https://github.com/Legrandin/pycryptodome' redis = 'https://github.com/redis/redis-py' requests = 'https://github.com/psf/requests' @@ -160,9 +158,7 @@ exclude = [ 'aerospike', # v8+ breaks agent build. # https://github.com/DataDog/integrations-core/pull/16080 'lxml', - # We're not ready to switch to v3 of the postgres library, see: - # https://github.com/DataDog/integrations-core/pull/15859 - 'psycopg2-binary', + 'psycopg', # Pinning psycopg binary to 3.2.7 to align OpenSSL version with the agent until we build wheels ourselves. 'psutil', 'pyvmomi', # 9+ has breaking changes 'pymongo', # Upgrade from 4.8.0 to 4.10.1 causes "AttributeError: module 'pymongo' has no attribute 'mongo_client'" diff --git a/.deps/metadata.json b/.deps/metadata.json index 67721269340d9..3b4bc45da588a 100644 --- a/.deps/metadata.json +++ b/.deps/metadata.json @@ -1,3 +1,3 @@ { - "sha256": "17205e7b1857d377d70d3e0bb78e6471ae9048adddaac061bc8384c8e94b4f53" + "sha256": "13e691792b63fe5ab1502a5b4ca8b87ca957ffc7fcf1666cd8e4a4385a2a5da8" } diff --git a/.deps/resolved/linux-aarch64_3.12.txt b/.deps/resolved/linux-aarch64_3.12.txt index 1280d13efe17f..5a80dc7a1310c 100644 --- a/.deps/resolved/linux-aarch64_3.12.txt +++ b/.deps/resolved/linux-aarch64_3.12.txt @@ -18,8 +18,8 @@ charset-normalizer @ https://agent-int-packages.datadoghq.com/external/charset-n clickhouse-cityhash @ https://agent-int-packages.datadoghq.com/external/clickhouse-cityhash/clickhouse_cityhash-1.0.2.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=bbfd83713e5a7a700c4a8200e921bc580fd7cba5f3b9d732172a5d82b12b3e20 clickhouse-driver @ https://agent-int-packages.datadoghq.com/external/clickhouse-driver/clickhouse_driver-0.2.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=4a8d8e2888a857d8db3d98765a5ad23ab561241feaef68bbffc5a0bd9c142342 cm-client @ https://agent-int-packages.datadoghq.com/built/cm-client/cm_client-45.0.4-20250218143931-py3-none-manylinux2014_aarch64.whl#sha256=72f55306e2e3df9291ee55e3a6b2f6698fe3999db9570a14da0ea56bbf51e5a9 -confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250813135016-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=27bcf8f2467efe1ce1ca9a97a3141c635780571b13ae7935119bb11e87021f7c -cryptography @ https://agent-int-packages.datadoghq.com/built/cryptography/cryptography-45.0.4-20250818094325-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=de879f8a3ae92df60cf8caf3bac544e1d856518a58838f0260a37c3e4ea7f46c +confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250818110700-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=27bcf8f2467efe1ce1ca9a97a3141c635780571b13ae7935119bb11e87021f7c +cryptography @ https://agent-int-packages.datadoghq.com/built/cryptography/cryptography-45.0.4-20250827192034-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=2d7c3536ff5b25652b59d924d70352fed24f5affb8f087deb026927192e4fbe2 ddtrace @ https://agent-int-packages.datadoghq.com/external/ddtrace/ddtrace-3.9.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=9d5ef534db804ab4f84ef1fa30ad6144695eefadf7a0263f14b69418a4a5f2b6 decorator @ https://agent-int-packages.datadoghq.com/external/decorator/decorator-5.2.1-py3-none-any.whl#sha256=d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a dnspython @ https://agent-int-packages.datadoghq.com/external/dnspython/dnspython-2.7.0-py3-none-any.whl#sha256=b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86 @@ -30,7 +30,7 @@ fastavro @ https://agent-int-packages.datadoghq.com/external/fastavro/fastavro-1 filelock @ https://agent-int-packages.datadoghq.com/external/filelock/filelock-3.19.1-py3-none-any.whl#sha256=d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d foundationdb @ https://agent-int-packages.datadoghq.com/built/foundationdb/foundationdb-6.3.25-20250513155638-py3-none-manylinux2014_aarch64.whl#sha256=cc09ffb2fb9b2a3f7cdb28669648c42bda418f0af83dacb91077c48fc1b06ae6 google-auth @ https://agent-int-packages.datadoghq.com/external/google-auth/google_auth-1.6.3-py2.py3-none-any.whl#sha256=20705f6803fd2c4d1cc2dcb0df09d4dfcb9a7d51fd59e94a3a28231fd93119ed -gssapi @ https://agent-int-packages.datadoghq.com/built/gssapi/gssapi-1.9.0-20250818094326-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=7e09a1ddc0f216c486d9828be86f4bb3c582e66b233d6a72c07c66be285c7bd0 +gssapi @ https://agent-int-packages.datadoghq.com/built/gssapi/gssapi-1.9.0-20250827192035-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl#sha256=7e09a1ddc0f216c486d9828be86f4bb3c582e66b233d6a72c07c66be285c7bd0 hazelcast-python-client @ https://agent-int-packages.datadoghq.com/external/hazelcast-python-client/hazelcast_python_client-5.5.0-py3-none-any.whl#sha256=c797c23c219971d225f8590f6359692c14059c26baa15c2762c95667ae38b90a idna @ https://agent-int-packages.datadoghq.com/external/idna/idna-3.10-py3-none-any.whl#sha256=946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 importlib-metadata @ https://agent-int-packages.datadoghq.com/external/importlib-metadata/importlib_metadata-8.7.0-py3-none-any.whl#sha256=e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd @@ -40,8 +40,8 @@ jellyfish @ https://agent-int-packages.datadoghq.com/external/jellyfish/jellyfis jmespath @ https://agent-int-packages.datadoghq.com/external/jmespath/jmespath-1.0.1-py3-none-any.whl#sha256=02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 jsonpatch @ https://agent-int-packages.datadoghq.com/external/jsonpatch/jsonpatch-1.33-py2.py3-none-any.whl#sha256=0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade jsonpointer @ https://agent-int-packages.datadoghq.com/external/jsonpointer/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 -keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.11.1-py3-none-any.whl#sha256=4525adf03b6e591f4b9b8a72c3b14f6510a04816dd5a7aca6ebaa6dfc90b69e6 -krb5 @ https://agent-int-packages.datadoghq.com/built/krb5/krb5-0.7.1-20250818094326-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=91bf30b0a956a0b7f0109094284958ba1a25c8f44497bbeca6d0042ad0596884 +keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.12.0-py3-none-any.whl#sha256=2e514b03615e2d9162f0c07c823a61a636e6d4df38ff4a34b7511a04e8a4166a +krb5 @ https://agent-int-packages.datadoghq.com/built/krb5/krb5-0.7.1-20250827192035-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl#sha256=91bf30b0a956a0b7f0109094284958ba1a25c8f44497bbeca6d0042ad0596884 kubernetes @ https://agent-int-packages.datadoghq.com/external/kubernetes/kubernetes-33.1.0-py2.py3-none-any.whl#sha256=544de42b24b64287f7e0aa9513c93cb503f7f40eea39b20f66810011a86eabc5 lazy-loader @ https://agent-int-packages.datadoghq.com/external/lazy-loader/lazy_loader-0.4-py3-none-any.whl#sha256=342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc ldap3 @ https://agent-int-packages.datadoghq.com/external/ldap3/ldap3-2.9.1-py2.py3-none-any.whl#sha256=5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70 @@ -59,13 +59,15 @@ os-service-types @ https://agent-int-packages.datadoghq.com/external/os-service- packaging @ https://agent-int-packages.datadoghq.com/external/packaging/packaging-25.0-py3-none-any.whl#sha256=29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 paramiko @ https://agent-int-packages.datadoghq.com/external/paramiko/paramiko-3.5.1-py3-none-any.whl#sha256=43b9a0501fc2b5e70680388d9346cf252cfb7d00b0667c39e80eb43a408b8f61 pathspec @ https://agent-int-packages.datadoghq.com/external/pathspec/pathspec-0.12.1-py3-none-any.whl#sha256=a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 -pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.0-py2.py3-none-any.whl#sha256=b447e63a2bc04fd975fc0480b8d5ebf979179e2c0ae203bf1eff9ea20073bc38 -platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.3.8-py3-none-any.whl#sha256=ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4 +pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.1-py2.py3-none-any.whl#sha256=32df5156fbeccb6f8a858d1ebc4e465dcf47d6cc7a4895d5df9aa951c712fc35 +platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.4.0-py3-none-any.whl#sha256=abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85 ply @ https://agent-int-packages.datadoghq.com/external/ply/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce prometheus-client @ https://agent-int-packages.datadoghq.com/external/prometheus-client/prometheus_client-0.22.1-py3-none-any.whl#sha256=cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094 protobuf @ https://agent-int-packages.datadoghq.com/external/protobuf/protobuf-6.31.1-cp39-abi3-manylinux2014_aarch64.whl#sha256=a40fc12b84c154884d7d4c4ebd675d5b3b5283e155f324049ae396b95ddebc39 psutil @ https://agent-int-packages.datadoghq.com/external/psutil/psutil-6.0.0-cp36-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=e2e8d0054fc88153ca0544f5c4d554d42e33df2e009c4ff42284ac9ebdef4132 -psycopg2-binary @ https://agent-int-packages.datadoghq.com/external/psycopg2-binary/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=8f8544b092a29a6ddd72f3556a9fcf249ec412e10ad28be6a0c0d948924f2212 +psycopg @ https://agent-int-packages.datadoghq.com/external/psycopg/psycopg-3.2.7-py3-none-any.whl#sha256=d39747d2d5b9658b69fa462ad21d31f1ba4a5722ad1d0cb952552bc0b4125451 +psycopg-binary @ https://agent-int-packages.datadoghq.com/external/psycopg-binary/psycopg_binary-3.2.7-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=5d1c97a7c57e83b55172b585702744cd6bdad37c7a18cabdf55ba1e5a66ce476 +psycopg-pool @ https://agent-int-packages.datadoghq.com/external/psycopg-pool/psycopg_pool-3.2.6-py3-none-any.whl#sha256=5887318a9f6af906d041a0b1dc1c60f8f0dda8340c2572b74e10907b51ed5da7 pyasn1 @ https://agent-int-packages.datadoghq.com/external/pyasn1/pyasn1-0.4.8-py2.py3-none-any.whl#sha256=39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d pyasn1-modules @ https://agent-int-packages.datadoghq.com/external/pyasn1-modules/pyasn1_modules-0.4.1-py3-none-any.whl#sha256=49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd pyasyncore @ https://agent-int-packages.datadoghq.com/external/pyasyncore/pyasyncore-1.0.4-py3-none-any.whl#sha256=9e5f6dc9dc057c56370b7a5cdb4c4670fd4b0556de2913ed1f428cd6a5366895 @@ -110,12 +112,12 @@ simplejson @ https://agent-int-packages.datadoghq.com/external/simplejson/simple six @ https://agent-int-packages.datadoghq.com/external/six/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 snowflake-connector-python @ https://agent-int-packages.datadoghq.com/external/snowflake-connector-python/snowflake_connector_python-3.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=457408eeb4116f73df60bb912fdeb7f6d4849e2cfd7e282b2afb25f080f20188 sortedcontainers @ https://agent-int-packages.datadoghq.com/external/sortedcontainers/sortedcontainers-2.4.0-py2.py3-none-any.whl#sha256=a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 -soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.7-py3-none-any.whl#sha256=6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4 -stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.4.1-py3-none-any.whl#sha256=d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe +soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.8-py3-none-any.whl#sha256=0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c +stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.5.0-py3-none-any.whl#sha256=18363d4d268181e8e8452e71a38cd77630f345b2ef6b4a8d5614dac5ee0d18cf supervisor @ https://agent-int-packages.datadoghq.com/external/supervisor/supervisor-4.2.5-py2.py3-none-any.whl#sha256=2ecaede32fc25af814696374b79e42644ecaba5c09494c51016ffda9602d0f08 tomlkit @ https://agent-int-packages.datadoghq.com/external/tomlkit/tomlkit-0.13.3-py3-none-any.whl#sha256=c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0 tuf @ https://agent-int-packages.datadoghq.com/external/tuf/tuf-4.0.0-py3-none-any.whl#sha256=a22ab5fa6daf910b3052929fdce42ccad8a300e5e85715daaff9592aed980f7a -typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.14.1-py3-none-any.whl#sha256=d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 +typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.15.0-py3-none-any.whl#sha256=f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 typing-inspection @ https://agent-int-packages.datadoghq.com/external/typing-inspection/typing_inspection-0.4.1-py3-none-any.whl#sha256=389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 tzlocal @ https://agent-int-packages.datadoghq.com/external/tzlocal/tzlocal-5.3.1-py3-none-any.whl#sha256=eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d uhashring @ https://agent-int-packages.datadoghq.com/external/uhashring/uhashring-2.4-py3-none-any.whl#sha256=0d6cae4ac3205ef039860b0befd6bc762f1686a276805bf1b998c8657124df62 diff --git a/.deps/resolved/linux-x86_64_3.12.txt b/.deps/resolved/linux-x86_64_3.12.txt index 635dfc5a0facf..d53e8bb7ce362 100644 --- a/.deps/resolved/linux-x86_64_3.12.txt +++ b/.deps/resolved/linux-x86_64_3.12.txt @@ -18,8 +18,8 @@ charset-normalizer @ https://agent-int-packages.datadoghq.com/external/charset-n clickhouse-cityhash @ https://agent-int-packages.datadoghq.com/external/clickhouse-cityhash/clickhouse_cityhash-1.0.2.4-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=f1f8fec4027cd648f72009ef59c9b76c5a27a33ca166b4e79e46542009429813 clickhouse-driver @ https://agent-int-packages.datadoghq.com/external/clickhouse-driver/clickhouse_driver-0.2.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=6dbcee870c60d9835e5dce1456ab6b9d807e6669246357f4b321ef747b90fa43 cm-client @ https://agent-int-packages.datadoghq.com/built/cm-client/cm_client-45.0.4-20250218143942-py3-none-manylinux2014_x86_64.whl#sha256=72f55306e2e3df9291ee55e3a6b2f6698fe3999db9570a14da0ea56bbf51e5a9 -confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250813135022-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a9c6a710617a52e0d5b799c0877a3b2734ab0a32f8009a6d97075e7ab39cdf13 -cryptography @ https://agent-int-packages.datadoghq.com/built/cryptography/cryptography-45.0.4-20250818094330-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=1a9f56e964dac718b36c62d063fa20d2bbde46920d8f57092037e1ee2fb3da63 +confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250818110708-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=a9c6a710617a52e0d5b799c0877a3b2734ab0a32f8009a6d97075e7ab39cdf13 +cryptography @ https://agent-int-packages.datadoghq.com/built/cryptography/cryptography-45.0.4-20250827192042-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=20e2e207d6981b4d43a912fe1c95ccdab85750bcef9bc0e8cbfc3a2cdc9544f4 ddtrace @ https://agent-int-packages.datadoghq.com/external/ddtrace/ddtrace-3.9.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=30f5fbcd0940b35072c6d7fbfef810a362a3d580e9f2001dd1eb3f39527f7f83 decorator @ https://agent-int-packages.datadoghq.com/external/decorator/decorator-5.2.1-py3-none-any.whl#sha256=d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a dnspython @ https://agent-int-packages.datadoghq.com/external/dnspython/dnspython-2.7.0-py3-none-any.whl#sha256=b4c34b7d10b51bcc3a5071e7b8dee77939f1e878477eeecc965e9835f63c6c86 @@ -30,7 +30,7 @@ fastavro @ https://agent-int-packages.datadoghq.com/external/fastavro/fastavro-1 filelock @ https://agent-int-packages.datadoghq.com/external/filelock/filelock-3.19.1-py3-none-any.whl#sha256=d38e30481def20772f5baf097c122c3babc4fcdb7e14e57049eb9d88c6dc017d foundationdb @ https://agent-int-packages.datadoghq.com/built/foundationdb/foundationdb-6.3.25-20250513155645-py3-none-manylinux2014_x86_64.whl#sha256=cc09ffb2fb9b2a3f7cdb28669648c42bda418f0af83dacb91077c48fc1b06ae6 google-auth @ https://agent-int-packages.datadoghq.com/external/google-auth/google_auth-1.6.3-py2.py3-none-any.whl#sha256=20705f6803fd2c4d1cc2dcb0df09d4dfcb9a7d51fd59e94a3a28231fd93119ed -gssapi @ https://agent-int-packages.datadoghq.com/built/gssapi/gssapi-1.9.0-20250818094330-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=ebb9c8abe80f3d7e014d96b7516c59f7c9d9818e66f35f67e4ecf55843804757 +gssapi @ https://agent-int-packages.datadoghq.com/built/gssapi/gssapi-1.9.0-20250827192042-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl#sha256=ebb9c8abe80f3d7e014d96b7516c59f7c9d9818e66f35f67e4ecf55843804757 hazelcast-python-client @ https://agent-int-packages.datadoghq.com/external/hazelcast-python-client/hazelcast_python_client-5.5.0-py3-none-any.whl#sha256=c797c23c219971d225f8590f6359692c14059c26baa15c2762c95667ae38b90a idna @ https://agent-int-packages.datadoghq.com/external/idna/idna-3.10-py3-none-any.whl#sha256=946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 importlib-metadata @ https://agent-int-packages.datadoghq.com/external/importlib-metadata/importlib_metadata-8.7.0-py3-none-any.whl#sha256=e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd @@ -40,8 +40,8 @@ jellyfish @ https://agent-int-packages.datadoghq.com/external/jellyfish/jellyfis jmespath @ https://agent-int-packages.datadoghq.com/external/jmespath/jmespath-1.0.1-py3-none-any.whl#sha256=02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 jsonpatch @ https://agent-int-packages.datadoghq.com/external/jsonpatch/jsonpatch-1.33-py2.py3-none-any.whl#sha256=0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade jsonpointer @ https://agent-int-packages.datadoghq.com/external/jsonpointer/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 -keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.11.1-py3-none-any.whl#sha256=4525adf03b6e591f4b9b8a72c3b14f6510a04816dd5a7aca6ebaa6dfc90b69e6 -krb5 @ https://agent-int-packages.datadoghq.com/built/krb5/krb5-0.7.1-20250818094331-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=7c609256d54324c3e1e1b494311c537f10108a1a124fcccc4cff04ad598814c7 +keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.12.0-py3-none-any.whl#sha256=2e514b03615e2d9162f0c07c823a61a636e6d4df38ff4a34b7511a04e8a4166a +krb5 @ https://agent-int-packages.datadoghq.com/built/krb5/krb5-0.7.1-20250827192043-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl#sha256=7c609256d54324c3e1e1b494311c537f10108a1a124fcccc4cff04ad598814c7 kubernetes @ https://agent-int-packages.datadoghq.com/external/kubernetes/kubernetes-33.1.0-py2.py3-none-any.whl#sha256=544de42b24b64287f7e0aa9513c93cb503f7f40eea39b20f66810011a86eabc5 lazy-loader @ https://agent-int-packages.datadoghq.com/external/lazy-loader/lazy_loader-0.4-py3-none-any.whl#sha256=342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc ldap3 @ https://agent-int-packages.datadoghq.com/external/ldap3/ldap3-2.9.1-py2.py3-none-any.whl#sha256=5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70 @@ -59,13 +59,15 @@ os-service-types @ https://agent-int-packages.datadoghq.com/external/os-service- packaging @ https://agent-int-packages.datadoghq.com/external/packaging/packaging-25.0-py3-none-any.whl#sha256=29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 paramiko @ https://agent-int-packages.datadoghq.com/external/paramiko/paramiko-3.5.1-py3-none-any.whl#sha256=43b9a0501fc2b5e70680388d9346cf252cfb7d00b0667c39e80eb43a408b8f61 pathspec @ https://agent-int-packages.datadoghq.com/external/pathspec/pathspec-0.12.1-py3-none-any.whl#sha256=a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 -pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.0-py2.py3-none-any.whl#sha256=b447e63a2bc04fd975fc0480b8d5ebf979179e2c0ae203bf1eff9ea20073bc38 -platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.3.8-py3-none-any.whl#sha256=ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4 +pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.1-py2.py3-none-any.whl#sha256=32df5156fbeccb6f8a858d1ebc4e465dcf47d6cc7a4895d5df9aa951c712fc35 +platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.4.0-py3-none-any.whl#sha256=abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85 ply @ https://agent-int-packages.datadoghq.com/external/ply/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce prometheus-client @ https://agent-int-packages.datadoghq.com/external/prometheus-client/prometheus_client-0.22.1-py3-none-any.whl#sha256=cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094 protobuf @ https://agent-int-packages.datadoghq.com/external/protobuf/protobuf-6.31.1-cp39-abi3-manylinux2014_x86_64.whl#sha256=4ee898bf66f7a8b0bd21bce523814e6fbd8c6add948045ce958b73af7e8878c6 psutil @ https://agent-int-packages.datadoghq.com/external/psutil/psutil-6.0.0-cp36-abi3-manylinux_2_12_x86_64.manylinux2010_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=5fd9a97c8e94059b0ef54a7d4baf13b405011176c3b6ff257c247cae0d560ecd -psycopg2-binary @ https://agent-int-packages.datadoghq.com/external/psycopg2-binary/psycopg2_binary-2.9.9-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=6e6f98446430fdf41bd36d4faa6cb409f5140c1c2cf58ce0bbdaf16af7d3f119 +psycopg @ https://agent-int-packages.datadoghq.com/external/psycopg/psycopg-3.2.7-py3-none-any.whl#sha256=d39747d2d5b9658b69fa462ad21d31f1ba4a5722ad1d0cb952552bc0b4125451 +psycopg-binary @ https://agent-int-packages.datadoghq.com/external/psycopg-binary/psycopg_binary-3.2.7-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=8eee57667fdd8a1cd8c4c2dc7350914267baf4d699690d44e439df9ae9148e7a +psycopg-pool @ https://agent-int-packages.datadoghq.com/external/psycopg-pool/psycopg_pool-3.2.6-py3-none-any.whl#sha256=5887318a9f6af906d041a0b1dc1c60f8f0dda8340c2572b74e10907b51ed5da7 pyasn1 @ https://agent-int-packages.datadoghq.com/external/pyasn1/pyasn1-0.4.8-py2.py3-none-any.whl#sha256=39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d pyasn1-modules @ https://agent-int-packages.datadoghq.com/external/pyasn1-modules/pyasn1_modules-0.4.1-py3-none-any.whl#sha256=49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd pyasyncore @ https://agent-int-packages.datadoghq.com/external/pyasyncore/pyasyncore-1.0.4-py3-none-any.whl#sha256=9e5f6dc9dc057c56370b7a5cdb4c4670fd4b0556de2913ed1f428cd6a5366895 @@ -111,12 +113,12 @@ simplejson @ https://agent-int-packages.datadoghq.com/external/simplejson/simple six @ https://agent-int-packages.datadoghq.com/external/six/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 snowflake-connector-python @ https://agent-int-packages.datadoghq.com/external/snowflake-connector-python/snowflake_connector_python-3.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl#sha256=63e93f67712260a11ae3bb858c567be805aca5f96d042ff5feacea102a33b49f sortedcontainers @ https://agent-int-packages.datadoghq.com/external/sortedcontainers/sortedcontainers-2.4.0-py2.py3-none-any.whl#sha256=a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 -soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.7-py3-none-any.whl#sha256=6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4 -stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.4.1-py3-none-any.whl#sha256=d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe +soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.8-py3-none-any.whl#sha256=0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c +stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.5.0-py3-none-any.whl#sha256=18363d4d268181e8e8452e71a38cd77630f345b2ef6b4a8d5614dac5ee0d18cf supervisor @ https://agent-int-packages.datadoghq.com/external/supervisor/supervisor-4.2.5-py2.py3-none-any.whl#sha256=2ecaede32fc25af814696374b79e42644ecaba5c09494c51016ffda9602d0f08 tomlkit @ https://agent-int-packages.datadoghq.com/external/tomlkit/tomlkit-0.13.3-py3-none-any.whl#sha256=c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0 tuf @ https://agent-int-packages.datadoghq.com/external/tuf/tuf-4.0.0-py3-none-any.whl#sha256=a22ab5fa6daf910b3052929fdce42ccad8a300e5e85715daaff9592aed980f7a -typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.14.1-py3-none-any.whl#sha256=d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 +typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.15.0-py3-none-any.whl#sha256=f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 typing-inspection @ https://agent-int-packages.datadoghq.com/external/typing-inspection/typing_inspection-0.4.1-py3-none-any.whl#sha256=389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 tzlocal @ https://agent-int-packages.datadoghq.com/external/tzlocal/tzlocal-5.3.1-py3-none-any.whl#sha256=eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d uhashring @ https://agent-int-packages.datadoghq.com/external/uhashring/uhashring-2.4-py3-none-any.whl#sha256=0d6cae4ac3205ef039860b0befd6bc762f1686a276805bf1b998c8657124df62 diff --git a/.deps/resolved/macos-aarch64_3.12.txt b/.deps/resolved/macos-aarch64_3.12.txt index d59f7815c32c0..3503d26336dad 100644 --- a/.deps/resolved/macos-aarch64_3.12.txt +++ b/.deps/resolved/macos-aarch64_3.12.txt @@ -17,7 +17,7 @@ charset-normalizer @ https://agent-int-packages.datadoghq.com/external/charset-n clickhouse-cityhash @ https://agent-int-packages.datadoghq.com/external/clickhouse-cityhash/clickhouse_cityhash-1.0.2.4-cp312-cp312-macosx_11_0_arm64.whl#sha256=acfa79048ac3b8203feba108c2d637d89ce1dfeaefabc1272a5c4e2dab716314 clickhouse-driver @ https://agent-int-packages.datadoghq.com/external/clickhouse-driver/clickhouse_driver-0.2.9-cp312-cp312-macosx_11_0_arm64.whl#sha256=b7a3e6b0a1eb218e3d870a94c76daaf65da46dca8f6888ea6542f94905c24d88 cm-client @ https://agent-int-packages.datadoghq.com/built/cm-client/cm_client-45.0.4-20241216144620-py3-none-macosx_10_12_universal2.whl#sha256=72f55306e2e3df9291ee55e3a6b2f6698fe3999db9570a14da0ea56bbf51e5a9 -confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250818094321-cp312-cp312-macosx_11_0_arm64.whl#sha256=0b88faf5ed33fc957de0cac3bbb24f7e3900349d4c53d838fe04740d2d7b9af8 +confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250827192027-cp312-cp312-macosx_11_0_arm64.whl#sha256=323424663693c7bffc11fab832c3964abe15c8bfcd16cda40bdf4d8d137e0b7e cryptography @ https://agent-int-packages.datadoghq.com/external/cryptography/cryptography-45.0.4-cp311-abi3-macosx_10_9_universal2.whl#sha256=425a9a6ac2823ee6e46a76a21a4e8342d8fa5c01e08b823c1f19a8b74f096069 ddtrace @ https://agent-int-packages.datadoghq.com/external/ddtrace/ddtrace-3.9.3-cp312-cp312-macosx_12_0_arm64.whl#sha256=ee54a29c53e2cbf8d07f57798f185280409415a249a968e904c887a12bbcf45e decorator @ https://agent-int-packages.datadoghq.com/external/decorator/decorator-5.2.1-py3-none-any.whl#sha256=d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a @@ -39,7 +39,7 @@ jellyfish @ https://agent-int-packages.datadoghq.com/external/jellyfish/jellyfis jmespath @ https://agent-int-packages.datadoghq.com/external/jmespath/jmespath-1.0.1-py3-none-any.whl#sha256=02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 jsonpatch @ https://agent-int-packages.datadoghq.com/external/jsonpatch/jsonpatch-1.33-py2.py3-none-any.whl#sha256=0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade jsonpointer @ https://agent-int-packages.datadoghq.com/external/jsonpointer/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 -keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.11.1-py3-none-any.whl#sha256=4525adf03b6e591f4b9b8a72c3b14f6510a04816dd5a7aca6ebaa6dfc90b69e6 +keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.12.0-py3-none-any.whl#sha256=2e514b03615e2d9162f0c07c823a61a636e6d4df38ff4a34b7511a04e8a4166a krb5 @ https://agent-int-packages.datadoghq.com/external/krb5/krb5-0.7.1-cp312-cp312-macosx_11_0_arm64.whl#sha256=af1932778cd462852e2a25596737cf0ae4e361f69e892b6c3ef3a29c960de3a0 kubernetes @ https://agent-int-packages.datadoghq.com/external/kubernetes/kubernetes-33.1.0-py2.py3-none-any.whl#sha256=544de42b24b64287f7e0aa9513c93cb503f7f40eea39b20f66810011a86eabc5 lazy-loader @ https://agent-int-packages.datadoghq.com/external/lazy-loader/lazy_loader-0.4-py3-none-any.whl#sha256=342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc @@ -58,13 +58,15 @@ os-service-types @ https://agent-int-packages.datadoghq.com/external/os-service- packaging @ https://agent-int-packages.datadoghq.com/external/packaging/packaging-25.0-py3-none-any.whl#sha256=29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 paramiko @ https://agent-int-packages.datadoghq.com/external/paramiko/paramiko-3.5.1-py3-none-any.whl#sha256=43b9a0501fc2b5e70680388d9346cf252cfb7d00b0667c39e80eb43a408b8f61 pathspec @ https://agent-int-packages.datadoghq.com/external/pathspec/pathspec-0.12.1-py3-none-any.whl#sha256=a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 -pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.0-py2.py3-none-any.whl#sha256=b447e63a2bc04fd975fc0480b8d5ebf979179e2c0ae203bf1eff9ea20073bc38 -platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.3.8-py3-none-any.whl#sha256=ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4 +pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.1-py2.py3-none-any.whl#sha256=32df5156fbeccb6f8a858d1ebc4e465dcf47d6cc7a4895d5df9aa951c712fc35 +platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.4.0-py3-none-any.whl#sha256=abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85 ply @ https://agent-int-packages.datadoghq.com/external/ply/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce prometheus-client @ https://agent-int-packages.datadoghq.com/external/prometheus-client/prometheus_client-0.22.1-py3-none-any.whl#sha256=cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094 protobuf @ https://agent-int-packages.datadoghq.com/external/protobuf/protobuf-6.31.1-cp39-abi3-macosx_10_9_universal2.whl#sha256=6f1227473dc43d44ed644425268eb7c2e488ae245d51c6866d19fe158e207402 psutil @ https://agent-int-packages.datadoghq.com/external/psutil/psutil-6.0.0-cp38-abi3-macosx_11_0_arm64.whl#sha256=ffe7fc9b6b36beadc8c322f84e1caff51e8703b88eee1da46d1e3a6ae11b4fd0 -psycopg2-binary @ https://agent-int-packages.datadoghq.com/external/psycopg2-binary/psycopg2_binary-2.9.9-cp312-cp312-macosx_11_0_arm64.whl#sha256=b0605eaed3eb239e87df0d5e3c6489daae3f7388d455d0c0b4df899519c6a38d +psycopg @ https://agent-int-packages.datadoghq.com/external/psycopg/psycopg-3.2.7-py3-none-any.whl#sha256=d39747d2d5b9658b69fa462ad21d31f1ba4a5722ad1d0cb952552bc0b4125451 +psycopg-binary @ https://agent-int-packages.datadoghq.com/external/psycopg-binary/psycopg_binary-3.2.7-cp312-cp312-macosx_11_0_arm64.whl#sha256=5ff4c97a04eeb11d54d4c8ca22459e2cca9a423e7f397c29ae311c6e7c784d49 +psycopg-pool @ https://agent-int-packages.datadoghq.com/external/psycopg-pool/psycopg_pool-3.2.6-py3-none-any.whl#sha256=5887318a9f6af906d041a0b1dc1c60f8f0dda8340c2572b74e10907b51ed5da7 pyasn1 @ https://agent-int-packages.datadoghq.com/external/pyasn1/pyasn1-0.4.8-py2.py3-none-any.whl#sha256=39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d pyasn1-modules @ https://agent-int-packages.datadoghq.com/external/pyasn1-modules/pyasn1_modules-0.4.1-py3-none-any.whl#sha256=49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd pyasyncore @ https://agent-int-packages.datadoghq.com/external/pyasyncore/pyasyncore-1.0.4-py3-none-any.whl#sha256=9e5f6dc9dc057c56370b7a5cdb4c4670fd4b0556de2913ed1f428cd6a5366895 @@ -74,7 +76,7 @@ pydantic @ https://agent-int-packages.datadoghq.com/external/pydantic/pydantic-2 pydantic-core @ https://agent-int-packages.datadoghq.com/external/pydantic-core/pydantic_core-2.33.2-cp312-cp312-macosx_11_0_arm64.whl#sha256=3c6db6e52c6d70aa0d00d45cdb9b40f0433b96380071ea80b09277dba021ddf7 pyjwt @ https://agent-int-packages.datadoghq.com/external/pyjwt/PyJWT-2.10.1-py3-none-any.whl#sha256=dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb pymongo @ https://agent-int-packages.datadoghq.com/external/pymongo/pymongo-4.8.0-cp312-cp312-macosx_11_0_arm64.whl#sha256=31e4d21201bdf15064cf47ce7b74722d3e1aea2597c6785882244a3bb58c7eab -pymqi @ https://agent-int-packages.datadoghq.com/built/pymqi/pymqi-1.12.11-20250818094321-cp312-cp312-macosx_11_0_arm64.whl#sha256=9f7d83e9545aa76650fe5e74b692d6705fa2e5a43a4808180e70bcf16714301d +pymqi @ https://agent-int-packages.datadoghq.com/built/pymqi/pymqi-1.12.11-20250827192028-cp312-cp312-macosx_11_0_arm64.whl#sha256=ddbd8999fa83a486af03e3889d83e56959e25d694ce257b21b618fe17faac68b pymysql @ https://agent-int-packages.datadoghq.com/external/pymysql/PyMySQL-1.1.1-py3-none-any.whl#sha256=4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c pynacl @ https://agent-int-packages.datadoghq.com/external/pynacl/PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl#sha256=401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1 pyodbc @ https://agent-int-packages.datadoghq.com/external/pyodbc/pyodbc-5.2.0-cp312-cp312-macosx_11_0_arm64.whl#sha256=9f7badd0055221a744d76c11440c0856fd2846ed53b6555cf8f0a8893a3e4b03 @@ -110,12 +112,12 @@ simplejson @ https://agent-int-packages.datadoghq.com/external/simplejson/simple six @ https://agent-int-packages.datadoghq.com/external/six/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 snowflake-connector-python @ https://agent-int-packages.datadoghq.com/external/snowflake-connector-python/snowflake_connector_python-3.15.0-cp312-cp312-macosx_11_0_arm64.whl#sha256=8f41ab707792b07c58214b13e7a6b50d4460f7151e19ea80df3612c92d3a7b76 sortedcontainers @ https://agent-int-packages.datadoghq.com/external/sortedcontainers/sortedcontainers-2.4.0-py2.py3-none-any.whl#sha256=a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 -soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.7-py3-none-any.whl#sha256=6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4 -stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.4.1-py3-none-any.whl#sha256=d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe +soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.8-py3-none-any.whl#sha256=0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c +stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.5.0-py3-none-any.whl#sha256=18363d4d268181e8e8452e71a38cd77630f345b2ef6b4a8d5614dac5ee0d18cf supervisor @ https://agent-int-packages.datadoghq.com/external/supervisor/supervisor-4.2.5-py2.py3-none-any.whl#sha256=2ecaede32fc25af814696374b79e42644ecaba5c09494c51016ffda9602d0f08 tomlkit @ https://agent-int-packages.datadoghq.com/external/tomlkit/tomlkit-0.13.3-py3-none-any.whl#sha256=c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0 tuf @ https://agent-int-packages.datadoghq.com/external/tuf/tuf-4.0.0-py3-none-any.whl#sha256=a22ab5fa6daf910b3052929fdce42ccad8a300e5e85715daaff9592aed980f7a -typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.14.1-py3-none-any.whl#sha256=d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 +typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.15.0-py3-none-any.whl#sha256=f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 typing-inspection @ https://agent-int-packages.datadoghq.com/external/typing-inspection/typing_inspection-0.4.1-py3-none-any.whl#sha256=389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 tzlocal @ https://agent-int-packages.datadoghq.com/external/tzlocal/tzlocal-5.3.1-py3-none-any.whl#sha256=eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d uhashring @ https://agent-int-packages.datadoghq.com/external/uhashring/uhashring-2.4-py3-none-any.whl#sha256=0d6cae4ac3205ef039860b0befd6bc762f1686a276805bf1b998c8657124df62 diff --git a/.deps/resolved/macos-x86_64_3.12.txt b/.deps/resolved/macos-x86_64_3.12.txt index b15bb39520238..5f87de9af03da 100644 --- a/.deps/resolved/macos-x86_64_3.12.txt +++ b/.deps/resolved/macos-x86_64_3.12.txt @@ -17,7 +17,7 @@ charset-normalizer @ https://agent-int-packages.datadoghq.com/external/charset-n clickhouse-cityhash @ https://agent-int-packages.datadoghq.com/external/clickhouse-cityhash/clickhouse_cityhash-1.0.2.4-cp312-cp312-macosx_10_9_x86_64.whl#sha256=261fc1b0bf349de66b2d9e3d367879a561b516ca8e54e85e0c27b7c1a4f639b4 clickhouse-driver @ https://agent-int-packages.datadoghq.com/external/clickhouse-driver/clickhouse_driver-0.2.9-cp312-cp312-macosx_10_9_x86_64.whl#sha256=fcb2fd00e58650ae206a6d5dbc83117240e622471aa5124733fbf2805eb8bda0 cm-client @ https://agent-int-packages.datadoghq.com/built/cm-client/cm_client-45.0.4-20241216144620-py3-none-macosx_10_12_universal2.whl#sha256=72f55306e2e3df9291ee55e3a6b2f6698fe3999db9570a14da0ea56bbf51e5a9 -confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250818094311-cp312-cp312-macosx_10_12_x86_64.whl#sha256=8b8ef62af0f12bb81fe5cb39f660cacc2aff072f4217fdf9e9e3fca47634703f +confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250827192010-cp312-cp312-macosx_10_12_x86_64.whl#sha256=9d583f2cdb5688a6c59c3bf123a1203e3ed38e20026482f1fb39a623f57867c7 cryptography @ https://agent-int-packages.datadoghq.com/external/cryptography/cryptography-45.0.4-cp311-abi3-macosx_10_9_universal2.whl#sha256=425a9a6ac2823ee6e46a76a21a4e8342d8fa5c01e08b823c1f19a8b74f096069 ddtrace @ https://agent-int-packages.datadoghq.com/external/ddtrace/ddtrace-3.9.3-cp312-cp312-macosx_12_0_x86_64.whl#sha256=b2553355df4c8bac52041b2f08e1b692b03fe1068f9686fb400ae188356c303c decorator @ https://agent-int-packages.datadoghq.com/external/decorator/decorator-5.2.1-py3-none-any.whl#sha256=d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a @@ -39,7 +39,7 @@ jellyfish @ https://agent-int-packages.datadoghq.com/external/jellyfish/jellyfis jmespath @ https://agent-int-packages.datadoghq.com/external/jmespath/jmespath-1.0.1-py3-none-any.whl#sha256=02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 jsonpatch @ https://agent-int-packages.datadoghq.com/external/jsonpatch/jsonpatch-1.33-py2.py3-none-any.whl#sha256=0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade jsonpointer @ https://agent-int-packages.datadoghq.com/external/jsonpointer/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 -keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.11.1-py3-none-any.whl#sha256=4525adf03b6e591f4b9b8a72c3b14f6510a04816dd5a7aca6ebaa6dfc90b69e6 +keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.12.0-py3-none-any.whl#sha256=2e514b03615e2d9162f0c07c823a61a636e6d4df38ff4a34b7511a04e8a4166a krb5 @ https://agent-int-packages.datadoghq.com/external/krb5/krb5-0.7.1-cp312-cp312-macosx_10_13_x86_64.whl#sha256=a075da3721b188070d801814c58652d04d3f37ccbf399dee63251f5ff27d2987 kubernetes @ https://agent-int-packages.datadoghq.com/external/kubernetes/kubernetes-33.1.0-py2.py3-none-any.whl#sha256=544de42b24b64287f7e0aa9513c93cb503f7f40eea39b20f66810011a86eabc5 lazy-loader @ https://agent-int-packages.datadoghq.com/external/lazy-loader/lazy_loader-0.4-py3-none-any.whl#sha256=342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc @@ -58,13 +58,15 @@ os-service-types @ https://agent-int-packages.datadoghq.com/external/os-service- packaging @ https://agent-int-packages.datadoghq.com/external/packaging/packaging-25.0-py3-none-any.whl#sha256=29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 paramiko @ https://agent-int-packages.datadoghq.com/external/paramiko/paramiko-3.5.1-py3-none-any.whl#sha256=43b9a0501fc2b5e70680388d9346cf252cfb7d00b0667c39e80eb43a408b8f61 pathspec @ https://agent-int-packages.datadoghq.com/external/pathspec/pathspec-0.12.1-py3-none-any.whl#sha256=a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 -pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.0-py2.py3-none-any.whl#sha256=b447e63a2bc04fd975fc0480b8d5ebf979179e2c0ae203bf1eff9ea20073bc38 -platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.3.8-py3-none-any.whl#sha256=ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4 +pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.1-py2.py3-none-any.whl#sha256=32df5156fbeccb6f8a858d1ebc4e465dcf47d6cc7a4895d5df9aa951c712fc35 +platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.4.0-py3-none-any.whl#sha256=abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85 ply @ https://agent-int-packages.datadoghq.com/external/ply/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce prometheus-client @ https://agent-int-packages.datadoghq.com/external/prometheus-client/prometheus_client-0.22.1-py3-none-any.whl#sha256=cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094 protobuf @ https://agent-int-packages.datadoghq.com/external/protobuf/protobuf-6.31.1-cp39-abi3-macosx_10_9_universal2.whl#sha256=6f1227473dc43d44ed644425268eb7c2e488ae245d51c6866d19fe158e207402 psutil @ https://agent-int-packages.datadoghq.com/external/psutil/psutil-6.0.0-cp36-abi3-macosx_10_9_x86_64.whl#sha256=c588a7e9b1173b6e866756dde596fd4cad94f9399daf99ad8c3258b3cb2b47a0 -psycopg2-binary @ https://agent-int-packages.datadoghq.com/external/psycopg2-binary/psycopg2_binary-2.9.9-cp312-cp312-macosx_10_9_x86_64.whl#sha256=8532fd6e6e2dc57bcb3bc90b079c60de896d2128c5d9d6f24a63875a95a088cf +psycopg @ https://agent-int-packages.datadoghq.com/external/psycopg/psycopg-3.2.7-py3-none-any.whl#sha256=d39747d2d5b9658b69fa462ad21d31f1ba4a5722ad1d0cb952552bc0b4125451 +psycopg-binary @ https://agent-int-packages.datadoghq.com/external/psycopg-binary/psycopg_binary-3.2.7-cp312-cp312-macosx_10_13_x86_64.whl#sha256=76e55ec30b3947b921f267795ffd2f433c65fc8a41adc4939fd9ccfb0f5b0322 +psycopg-pool @ https://agent-int-packages.datadoghq.com/external/psycopg-pool/psycopg_pool-3.2.6-py3-none-any.whl#sha256=5887318a9f6af906d041a0b1dc1c60f8f0dda8340c2572b74e10907b51ed5da7 pyasn1 @ https://agent-int-packages.datadoghq.com/external/pyasn1/pyasn1-0.4.8-py2.py3-none-any.whl#sha256=39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d pyasn1-modules @ https://agent-int-packages.datadoghq.com/external/pyasn1-modules/pyasn1_modules-0.4.1-py3-none-any.whl#sha256=49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd pyasyncore @ https://agent-int-packages.datadoghq.com/external/pyasyncore/pyasyncore-1.0.4-py3-none-any.whl#sha256=9e5f6dc9dc057c56370b7a5cdb4c4670fd4b0556de2913ed1f428cd6a5366895 @@ -74,7 +76,7 @@ pydantic @ https://agent-int-packages.datadoghq.com/external/pydantic/pydantic-2 pydantic-core @ https://agent-int-packages.datadoghq.com/external/pydantic-core/pydantic_core-2.33.2-cp312-cp312-macosx_10_12_x86_64.whl#sha256=a7ec89dc587667f22b6a0b6579c249fca9026ce7c333fc142ba42411fa243cdc pyjwt @ https://agent-int-packages.datadoghq.com/external/pyjwt/PyJWT-2.10.1-py3-none-any.whl#sha256=dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb pymongo @ https://agent-int-packages.datadoghq.com/external/pymongo/pymongo-4.8.0-cp312-cp312-macosx_10_9_x86_64.whl#sha256=e6a720a3d22b54183352dc65f08cd1547204d263e0651b213a0a2e577e838526 -pymqi @ https://agent-int-packages.datadoghq.com/built/pymqi/pymqi-1.12.11-20250818094312-cp312-cp312-macosx_10_12_x86_64.whl#sha256=6291de32095463dbbeadb06aadb2027772438f7b4a97c714d3d465ac3a870eb7 +pymqi @ https://agent-int-packages.datadoghq.com/built/pymqi/pymqi-1.12.11-20250827192011-cp312-cp312-macosx_10_12_x86_64.whl#sha256=604b1aa05f6220c321b86b1156e7b5d641831eea0ca12224ffda02194a61b5be pymysql @ https://agent-int-packages.datadoghq.com/external/pymysql/PyMySQL-1.1.1-py3-none-any.whl#sha256=4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c pynacl @ https://agent-int-packages.datadoghq.com/external/pynacl/PyNaCl-1.5.0-cp36-abi3-macosx_10_10_universal2.whl#sha256=401002a4aaa07c9414132aaed7f6836ff98f59277a234704ff66878c2ee4a0d1 pyodbc @ https://agent-int-packages.datadoghq.com/external/pyodbc/pyodbc-5.2.0-cp312-cp312-macosx_10_13_x86_64.whl#sha256=be43d1ece4f2cf4d430996689d89a1a15aeb3a8da8262527e5ced5aee27e89c3 @@ -110,12 +112,12 @@ simplejson @ https://agent-int-packages.datadoghq.com/external/simplejson/simple six @ https://agent-int-packages.datadoghq.com/external/six/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 snowflake-connector-python @ https://agent-int-packages.datadoghq.com/external/snowflake-connector-python/snowflake_connector_python-3.15.0-cp312-cp312-macosx_11_0_x86_64.whl#sha256=d2f0bb00a48c4125090dfe2e8bd15f9389be7867fd9e3c118ecb8dc87c69bcb8 sortedcontainers @ https://agent-int-packages.datadoghq.com/external/sortedcontainers/sortedcontainers-2.4.0-py2.py3-none-any.whl#sha256=a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 -soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.7-py3-none-any.whl#sha256=6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4 -stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.4.1-py3-none-any.whl#sha256=d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe +soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.8-py3-none-any.whl#sha256=0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c +stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.5.0-py3-none-any.whl#sha256=18363d4d268181e8e8452e71a38cd77630f345b2ef6b4a8d5614dac5ee0d18cf supervisor @ https://agent-int-packages.datadoghq.com/external/supervisor/supervisor-4.2.5-py2.py3-none-any.whl#sha256=2ecaede32fc25af814696374b79e42644ecaba5c09494c51016ffda9602d0f08 tomlkit @ https://agent-int-packages.datadoghq.com/external/tomlkit/tomlkit-0.13.3-py3-none-any.whl#sha256=c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0 tuf @ https://agent-int-packages.datadoghq.com/external/tuf/tuf-4.0.0-py3-none-any.whl#sha256=a22ab5fa6daf910b3052929fdce42ccad8a300e5e85715daaff9592aed980f7a -typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.14.1-py3-none-any.whl#sha256=d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 +typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.15.0-py3-none-any.whl#sha256=f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 typing-inspection @ https://agent-int-packages.datadoghq.com/external/typing-inspection/typing_inspection-0.4.1-py3-none-any.whl#sha256=389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 tzlocal @ https://agent-int-packages.datadoghq.com/external/tzlocal/tzlocal-5.3.1-py3-none-any.whl#sha256=eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d uhashring @ https://agent-int-packages.datadoghq.com/external/uhashring/uhashring-2.4-py3-none-any.whl#sha256=0d6cae4ac3205ef039860b0befd6bc762f1686a276805bf1b998c8657124df62 diff --git a/.deps/resolved/windows-x86_64_3.12.txt b/.deps/resolved/windows-x86_64_3.12.txt index dc8d285b10e02..c915c9df1aba9 100644 --- a/.deps/resolved/windows-x86_64_3.12.txt +++ b/.deps/resolved/windows-x86_64_3.12.txt @@ -17,7 +17,7 @@ charset-normalizer @ https://agent-int-packages.datadoghq.com/external/charset-n clickhouse-cityhash @ https://agent-int-packages.datadoghq.com/external/clickhouse-cityhash/clickhouse_cityhash-1.0.2.4-cp312-cp312-win_amd64.whl#sha256=0409917be29f5ad80a6772712fce954b5e81450555636e8523290ee9740a2dbb clickhouse-driver @ https://agent-int-packages.datadoghq.com/external/clickhouse-driver/clickhouse_driver-0.2.9-cp312-cp312-win_amd64.whl#sha256=de6624e28eeffd01668803d28ae89e3d4e359b1bff8b60e4933e1cb3c6f86f18 cm-client @ https://agent-int-packages.datadoghq.com/built/cm-client/cm_client-45.0.4-20250218143924-py3-none-win_amd64.whl#sha256=02165c52665dfa139b6606bd802b996c51bcbc8c89f3c79801833d53e838d735 -confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250818094316-cp312-cp312-win_amd64.whl#sha256=599e3f2e34e7fab40643d27561a374b093dbe789d217b0789699d2b5aabf8b01 +confluent-kafka @ https://agent-int-packages.datadoghq.com/built/confluent-kafka/confluent_kafka-2.8.0-20250827192020-cp312-cp312-win_amd64.whl#sha256=8df64225c61c04950e2d4e40f93f0176829866549b7e3ee38d9cd060edd291d9 cryptography @ https://agent-int-packages.datadoghq.com/external/cryptography/cryptography-45.0.4-cp311-abi3-win_amd64.whl#sha256=817ee05c6c9f7a69a16200f0c90ab26d23a87701e2a284bd15156783e46dbcc8 ddtrace @ https://agent-int-packages.datadoghq.com/external/ddtrace/ddtrace-3.9.3-cp312-cp312-win_amd64.whl#sha256=5d8171be0bf47928037a5844bb0ca7820935b465d7bcec9fe022455f00b0f690 decorator @ https://agent-int-packages.datadoghq.com/external/decorator/decorator-5.2.1-py3-none-any.whl#sha256=d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a @@ -38,7 +38,7 @@ jellyfish @ https://agent-int-packages.datadoghq.com/external/jellyfish/jellyfis jmespath @ https://agent-int-packages.datadoghq.com/external/jmespath/jmespath-1.0.1-py3-none-any.whl#sha256=02e2e4cc71b5bcab88332eebf907519190dd9e6e82107fa7f83b1003a6252980 jsonpatch @ https://agent-int-packages.datadoghq.com/external/jsonpatch/jsonpatch-1.33-py2.py3-none-any.whl#sha256=0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade jsonpointer @ https://agent-int-packages.datadoghq.com/external/jsonpointer/jsonpointer-3.0.0-py2.py3-none-any.whl#sha256=13e088adc14fca8b6aa8177c044e12701e6ad4b28ff10e65f2267a90109c9942 -keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.11.1-py3-none-any.whl#sha256=4525adf03b6e591f4b9b8a72c3b14f6510a04816dd5a7aca6ebaa6dfc90b69e6 +keystoneauth1 @ https://agent-int-packages.datadoghq.com/external/keystoneauth1/keystoneauth1-5.12.0-py3-none-any.whl#sha256=2e514b03615e2d9162f0c07c823a61a636e6d4df38ff4a34b7511a04e8a4166a kubernetes @ https://agent-int-packages.datadoghq.com/external/kubernetes/kubernetes-33.1.0-py2.py3-none-any.whl#sha256=544de42b24b64287f7e0aa9513c93cb503f7f40eea39b20f66810011a86eabc5 lazy-loader @ https://agent-int-packages.datadoghq.com/external/lazy-loader/lazy_loader-0.4-py3-none-any.whl#sha256=342aa8e14d543a154047afb4ba8ef17f5563baad3fc610d7b15b213b0f119efc ldap3 @ https://agent-int-packages.datadoghq.com/external/ldap3/ldap3-2.9.1-py2.py3-none-any.whl#sha256=5869596fc4948797020d3f03b7939da938778a0f9e2009f7a072ccf92b8e8d70 @@ -56,13 +56,15 @@ os-service-types @ https://agent-int-packages.datadoghq.com/external/os-service- packaging @ https://agent-int-packages.datadoghq.com/external/packaging/packaging-25.0-py3-none-any.whl#sha256=29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484 paramiko @ https://agent-int-packages.datadoghq.com/external/paramiko/paramiko-3.5.1-py3-none-any.whl#sha256=43b9a0501fc2b5e70680388d9346cf252cfb7d00b0667c39e80eb43a408b8f61 pathspec @ https://agent-int-packages.datadoghq.com/external/pathspec/pathspec-0.12.1-py3-none-any.whl#sha256=a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08 -pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.0-py2.py3-none-any.whl#sha256=b447e63a2bc04fd975fc0480b8d5ebf979179e2c0ae203bf1eff9ea20073bc38 -platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.3.8-py3-none-any.whl#sha256=ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4 +pbr @ https://agent-int-packages.datadoghq.com/external/pbr/pbr-7.0.1-py2.py3-none-any.whl#sha256=32df5156fbeccb6f8a858d1ebc4e465dcf47d6cc7a4895d5df9aa951c712fc35 +platformdirs @ https://agent-int-packages.datadoghq.com/external/platformdirs/platformdirs-4.4.0-py3-none-any.whl#sha256=abd01743f24e5287cd7a5db3752faf1a2d65353f38ec26d98e25a6db65958c85 ply @ https://agent-int-packages.datadoghq.com/external/ply/ply-3.11-py2.py3-none-any.whl#sha256=096f9b8350b65ebd2fd1346b12452efe5b9607f7482813ffca50c22722a807ce prometheus-client @ https://agent-int-packages.datadoghq.com/external/prometheus-client/prometheus_client-0.22.1-py3-none-any.whl#sha256=cca895342e308174341b2cbf99a56bef291fbc0ef7b9e5412a0f26d653ba7094 protobuf @ https://agent-int-packages.datadoghq.com/external/protobuf/protobuf-6.31.1-cp310-abi3-win_amd64.whl#sha256=426f59d2964864a1a366254fa703b8632dcec0790d8862d30034d8245e1cd447 psutil @ https://agent-int-packages.datadoghq.com/external/psutil/psutil-6.0.0-cp37-abi3-win_amd64.whl#sha256=33ea5e1c975250a720b3a6609c490db40dae5d83a4eb315170c4fe0d8b1f34b3 -psycopg2-binary @ https://agent-int-packages.datadoghq.com/external/psycopg2-binary/psycopg2_binary-2.9.9-cp312-cp312-win_amd64.whl#sha256=81ff62668af011f9a48787564ab7eded4e9fb17a4a6a74af5ffa6a457400d2ab +psycopg @ https://agent-int-packages.datadoghq.com/external/psycopg/psycopg-3.2.7-py3-none-any.whl#sha256=d39747d2d5b9658b69fa462ad21d31f1ba4a5722ad1d0cb952552bc0b4125451 +psycopg-binary @ https://agent-int-packages.datadoghq.com/external/psycopg-binary/psycopg_binary-3.2.7-cp312-cp312-win_amd64.whl#sha256=47e9d09b4f898eaf46cd2b7433f9e6faa935246a9d8983b4f88f0a46809abbd2 +psycopg-pool @ https://agent-int-packages.datadoghq.com/external/psycopg-pool/psycopg_pool-3.2.6-py3-none-any.whl#sha256=5887318a9f6af906d041a0b1dc1c60f8f0dda8340c2572b74e10907b51ed5da7 pyasn1 @ https://agent-int-packages.datadoghq.com/external/pyasn1/pyasn1-0.4.8-py2.py3-none-any.whl#sha256=39c7e2ec30515947ff4e87fb6f456dfc6e84857d34be479c9d4a4ba4bf46aa5d pyasn1-modules @ https://agent-int-packages.datadoghq.com/external/pyasn1-modules/pyasn1_modules-0.4.1-py3-none-any.whl#sha256=49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd pyasyncore @ https://agent-int-packages.datadoghq.com/external/pyasyncore/pyasyncore-1.0.4-py3-none-any.whl#sha256=9e5f6dc9dc057c56370b7a5cdb4c4670fd4b0556de2913ed1f428cd6a5366895 @@ -72,7 +74,7 @@ pydantic @ https://agent-int-packages.datadoghq.com/external/pydantic/pydantic-2 pydantic-core @ https://agent-int-packages.datadoghq.com/external/pydantic-core/pydantic_core-2.33.2-cp312-cp312-win_amd64.whl#sha256=f941635f2a3d96b2973e867144fde513665c87f13fe0e193c158ac51bfaaa7b2 pyjwt @ https://agent-int-packages.datadoghq.com/external/pyjwt/PyJWT-2.10.1-py3-none-any.whl#sha256=dcdd193e30abefd5debf142f9adfcdd2b58004e644f25406ffaebd50bd98dacb pymongo @ https://agent-int-packages.datadoghq.com/external/pymongo/pymongo-4.8.0-cp312-cp312-win_amd64.whl#sha256=e84bc7707492f06fbc37a9f215374d2977d21b72e10a67f1b31893ec5a140ad8 -pymqi @ https://agent-int-packages.datadoghq.com/built/pymqi/pymqi-1.12.11-20250818094317-cp312-cp312-win_amd64.whl#sha256=5af3bf4c8ca731462504db72bdeb13e0436f22ba03b02475264b8b25c74e94f2 +pymqi @ https://agent-int-packages.datadoghq.com/built/pymqi/pymqi-1.12.11-20250818110648-cp312-cp312-win_amd64.whl#sha256=6dfad4d0772cb3dfb37424ffdad6ddf939524b414907167f43a35c29db070a76 pymysql @ https://agent-int-packages.datadoghq.com/external/pymysql/PyMySQL-1.1.1-py3-none-any.whl#sha256=4de15da4c61dc132f4fb9ab763063e693d521a80fd0e87943b9a453dd4c19d6c pynacl @ https://agent-int-packages.datadoghq.com/external/pynacl/PyNaCl-1.5.0-cp36-abi3-win_amd64.whl#sha256=20f42270d27e1b6a29f54032090b972d97f0a1b0948cc52392041ef7831fee93 pyodbc @ https://agent-int-packages.datadoghq.com/external/pyodbc/pyodbc-5.2.0-cp312-cp312-win_amd64.whl#sha256=b1f5686b142759c5b2bdbeaa0692622c2ebb1f10780eb3c174b85f5607fbcf55 @@ -107,13 +109,13 @@ simplejson @ https://agent-int-packages.datadoghq.com/external/simplejson/simple six @ https://agent-int-packages.datadoghq.com/external/six/six-1.17.0-py2.py3-none-any.whl#sha256=4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 snowflake-connector-python @ https://agent-int-packages.datadoghq.com/external/snowflake-connector-python/snowflake_connector_python-3.15.0-cp312-cp312-win_amd64.whl#sha256=acbceae9120502613991b9997d94cc23ea7e76cdfc3601718025f9f0859cc21e sortedcontainers @ https://agent-int-packages.datadoghq.com/external/sortedcontainers/sortedcontainers-2.4.0-py2.py3-none-any.whl#sha256=a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0 -soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.7-py3-none-any.whl#sha256=6e60cc5c1ffaf1cebcc12e8188320b72071e922c2e897f737cadce79ad5d30c4 +soupsieve @ https://agent-int-packages.datadoghq.com/external/soupsieve/soupsieve-2.8-py3-none-any.whl#sha256=0cc76456a30e20f5d7f2e14a98a4ae2ee4e5abdc7c5ea0aafe795f344bc7984c sspilib @ https://agent-int-packages.datadoghq.com/external/sspilib/sspilib-0.3.1-cp312-cp312-win_amd64.whl#sha256=3355cfc5f3d5c257dbab2396d83493330ca952f9c28f3fe964193ababcc8c293 -stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.4.1-py3-none-any.whl#sha256=d10a31c7b86cba16c1f6e8d15416955fc797052351a56af15e608ad20811fcfe +stevedore @ https://agent-int-packages.datadoghq.com/external/stevedore/stevedore-5.5.0-py3-none-any.whl#sha256=18363d4d268181e8e8452e71a38cd77630f345b2ef6b4a8d5614dac5ee0d18cf supervisor @ https://agent-int-packages.datadoghq.com/external/supervisor/supervisor-4.2.5-py2.py3-none-any.whl#sha256=2ecaede32fc25af814696374b79e42644ecaba5c09494c51016ffda9602d0f08 tomlkit @ https://agent-int-packages.datadoghq.com/external/tomlkit/tomlkit-0.13.3-py3-none-any.whl#sha256=c89c649d79ee40629a9fda55f8ace8c6a1b42deb912b2a8fd8d942ddadb606b0 tuf @ https://agent-int-packages.datadoghq.com/external/tuf/tuf-4.0.0-py3-none-any.whl#sha256=a22ab5fa6daf910b3052929fdce42ccad8a300e5e85715daaff9592aed980f7a -typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.14.1-py3-none-any.whl#sha256=d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 +typing-extensions @ https://agent-int-packages.datadoghq.com/external/typing-extensions/typing_extensions-4.15.0-py3-none-any.whl#sha256=f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548 typing-inspection @ https://agent-int-packages.datadoghq.com/external/typing-inspection/typing_inspection-0.4.1-py3-none-any.whl#sha256=389055682238f53b04f7badcb49b989835495a96700ced5dab2d8feae4b26f51 tzdata @ https://agent-int-packages.datadoghq.com/external/tzdata/tzdata-2025.2-py2.py3-none-any.whl#sha256=1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8 tzlocal @ https://agent-int-packages.datadoghq.com/external/tzlocal/tzlocal-5.3.1-py3-none-any.whl#sha256=eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d diff --git a/LICENSE-3rdparty.csv b/LICENSE-3rdparty.csv index f4164f44dc651..ad099b5f9c61a 100644 --- a/LICENSE-3rdparty.csv +++ b/LICENSE-3rdparty.csv @@ -47,8 +47,7 @@ ply,PyPI,BSD-3-Clause,Copyright (C) 2001-2018 prometheus-client,PyPI,Apache-2.0,Copyright 2015 The Prometheus Authors protobuf,PyPI,BSD-3-Clause,Copyright 2008 Google Inc. All rights reserved. psutil,PyPI,BSD-3-Clause,"Copyright (c) 2009, Jay Loden, Dave Daeschler, Giampaolo Rodola" -psycopg2-binary,PyPI,BSD-3-Clause,Copyright 2013 Federico Di Gregorio -psycopg2-binary,PyPI,LGPL-3.0-only,Copyright (C) 2013 Federico Di Gregorio +psycopg,PyPI,LGPL-3.0-only,Copyright (C) 2020 The Psycopg Team pyOpenSSL,PyPI,Apache-2.0,Copyright The pyOpenSSL developers pyasn1,PyPI,BSD-3-Clause,"Copyright (c) 2005-2019, Ilya Etingof " pycryptodomex,PyPI,BSD-2-Clause,Copyright 2014 Helder Eijs diff --git a/agent_requirements.in b/agent_requirements.in index 07b5c82970558..5a9cc28f0d8a1 100644 --- a/agent_requirements.in +++ b/agent_requirements.in @@ -33,7 +33,7 @@ ply==3.11 prometheus-client==0.22.1 protobuf==6.31.1 psutil==6.0.0 -psycopg2-binary==2.9.9 +psycopg[binary,pool]==3.2.7 pyasn1==0.4.8 pycryptodomex==3.23.0 pydantic==2.11.7 diff --git a/databricks/assets/dashboards/databricks_cost_overview.json b/databricks/assets/dashboards/databricks_cost_overview.json index 63f2de8ea9c2a..825f3162e9d4c 100644 --- a/databricks/assets/dashboards/databricks_cost_overview.json +++ b/databricks/assets/dashboards/databricks_cost_overview.json @@ -1,747 +1,985 @@ { "title": "Databricks Cost Overview", "description": "This dashboard provides insights into various Databricks costs and their sources of attribution.", - "widgets": [{ - "id": 8347049005274980, - "definition": { - "title": "Databricks Cost Management", - "background_color": "vivid_green", - "show_title": true, - "type": "group", - "layout_type": "ordered", - "widgets": [{ - "id": 409821317360376, - "definition": { - "type": "image", - "url": "/static/images/logos/databricks_small.svg", - "url_dark_theme": "", - "sizing": "cover", - "has_background": true, - "has_border": true, - "vertical_align": "center", - "horizontal_align": "center" + "widgets": [ + { + "id": 8738438501262309, + "definition": { + "title": "Total Databricks Spend", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true }, - "layout": { - "x": 0, - "y": 0, - "width": 3, - "height": 2 - } - }, { - "id": 1457644073160806, - "definition": { - "title": "Total Costs (past 30d)", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1 - }, - "type": "query_value", - "requests": [{ - "formulas": [{ - "formula": "query1" - }], - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:custom.cost.amortized{provider_name:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id}", - "aggregator": "sum" - }], + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:custom.cost.amortized{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id}", + "aggregator": "sum" + } + ], "response_format": "scalar" - }], - "autoscale": true, - "precision": 2 + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 0, + "width": 4, + "height": 2 + } + }, + { + "id": 7402251919033163, + "definition": { + "title": "Total Costs (past 30d)", + "title_size": "16", + "title_align": "left", + "time": { + "type": "fixed", + "from": 1751932800000, + "to": 1754611200000, + "hide_incomplete_cost_data": true }, - "layout": { - "x": 3, - "y": 0, - "width": 3, - "height": 2 - } - }, { - "id": 5140360304170010, - "definition": { - "title": "Total Costs (30d prior)", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1 - }, - "type": "query_value", - "requests": [{ - "formulas": [{ - "formula": "timeshift(query1, -2592000)" - }], - "queries": [{ - "query": "sum:custom.cost.amortized{provider_name:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id}", - "data_source": "cloud_cost", - "name": "query1", - "aggregator": "sum" - }], + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:custom.cost.amortized{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id}", + "aggregator": "sum" + } + ], "response_format": "scalar" - }], - "autoscale": true, - "precision": 2 + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 4, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 5674547883690658, + "definition": { + "title": "Total Costs (30d prior)", + "title_size": "16", + "title_align": "left", + "time": { + "type": "fixed", + "from": 1751932800000, + "to": 1754611200000, + "hide_incomplete_cost_data": true }, - "layout": { - "x": 6, - "y": 0, - "width": 3, - "height": 2 - } - }, { - "id": 8170994894343814, - "definition": { - "title": "% change (past 30d vs 30d prior)", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1 - }, - "type": "query_value", - "requests": [{ - "formulas": [{ - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "percent" - } + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "timeshift(query1, -2592000)" + } + ], + "queries": [ + { + "query": "sum:custom.cost.amortized{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id}", + "data_source": "cloud_cost", + "name": "query1", + "aggregator": "sum" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 7, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 3044766898652945, + "definition": { + "title": "MoM change", + "title_size": "16", + "title_align": "left", + "time": { + "type": "fixed", + "from": 1749826086865, + "to": 1752504486865, + "hide_incomplete_cost_data": true + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "(query1 - timeshift(query2, -2592000)) / timeshift(query2, -2592000) * 100" + } + ], + "queries": [ + { + "query": "sum:custom.cost.amortized{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id}", + "data_source": "cloud_cost", + "name": "query1", + "aggregator": "sum" }, - "formula": "(query1 - timeshift(query2, -2592000)) / timeshift(query2, -2592000) * 100" - }], - "queries": [{ - "query": "sum:custom.cost.amortized{provider_name:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id}", - "data_source": "cloud_cost", - "name": "query1", - "aggregator": "sum" - }, { - "query": "sum:custom.cost.amortized{provider_name:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id}", - "data_source": "cloud_cost", - "name": "query2", - "aggregator": "sum" - }], + { + "query": "sum:custom.cost.amortized{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id}", + "data_source": "cloud_cost", + "name": "query2", + "aggregator": "sum" + } + ], "response_format": "scalar", - "conditional_formats": [{ - "comparator": "<", - "value": 5, - "palette": "black_on_light_green" - }, { - "comparator": "<", - "value": 10, - "palette": "black_on_light_yellow" - }, { - "comparator": ">", - "value": 10, - "palette": "black_on_light_red" - }] - }], - "autoscale": true, - "precision": 2 + "conditional_formats": [ + { + "comparator": "<", + "value": 1, + "palette": "black_on_light_green" + }, + { + "comparator": "<", + "value": 10, + "palette": "black_on_light_yellow" + }, + { + "comparator": "<", + "value": 100, + "palette": "black_on_light_red" + }, + { + "comparator": ">=", + "value": 100, + "palette": "white_on_red" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 10, + "y": 0, + "width": 2, + "height": 2 + } + }, + { + "id": 5683139742669696, + "definition": { + "type": "note", + "content": "Key concepts:\n- **workspace_id** - unified environment for performing data-centric tasks\n- **servicename** - the billing origin product, such as all purpose, jobs, dlt, etc\n- **charge_description** - the SKU, or pricing model, based on usage type", + "background_color": "green", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 2, + "width": 3, + "height": 3 + } + }, + { + "id": 8604160612174419, + "definition": { + "title": "Daily Spend per Charge Description", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "hide_incomplete_cost_data": true }, - "layout": { - "x": 9, - "y": 0, - "width": 3, - "height": 2 - } - }, { - "id": 3631138712976072, - "definition": { - "type": "note", - "content": "Key concepts:\n- **workspace_id** - unified environment for performing data-centric tasks\n- **servicename** - the billing origin product, such as all purpose, jobs, dlt, etc\n- **charge_description** - the SKU, or pricing model, based on usage type", - "background_color": "green", - "font_size": "14", - "text_align": "left", - "vertical_align": "center", - "show_tick": false, - "tick_pos": "50%", - "tick_edge": "left", - "has_padding": true + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {servicename,charge_description}.rollup(sum, daily)" + } + ], + "response_format": "timeseries", + "style": { + "palette": "classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ] + }, + "layout": { + "x": 3, + "y": 2, + "width": 9, + "height": 3 + } + }, + { + "id": 7403434338270407, + "definition": { + "title": "Spend per ServiceName", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true }, - "layout": { - "x": 0, - "y": 2, - "width": 3, - "height": 3 - } - }, { - "id": 8525206763937204, - "definition": { - "title": "Cost per ServiceName", - "title_size": "16", - "title_align": "left", - "requests": [{ + "requests": [ + { "response_format": "scalar", - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {servicename}", - "aggregator": "sum" - }], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {servicename}", + "aggregator": "sum" + } + ], "style": { - "palette": "datadog16" + "palette": "classic" }, - "formulas": [{ - "formula": "query1" - }], + "formulas": [ + { + "formula": "query1" + } + ], "sort": { "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] } - }], - "type": "sunburst", - "legend": { - "type": "table" } - }, - "layout": { - "x": 3, - "y": 2, - "width": 4, - "height": 3 + ], + "type": "sunburst", + "legend": { + "type": "table" } - }, { - "id": 6811822208736696, - "definition": { - "title": "Cost by Workspace", - "title_size": "16", - "title_align": "left", - "type": "toplist", - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {workspace_id}", - "aggregator": "sum" - }], + }, + "layout": { + "x": 0, + "y": 5, + "width": 6, + "height": 4 + } + }, + { + "id": 6603688971086557, + "definition": { + "title": "Spend per Charge Description", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "requests": [ + { "response_format": "scalar", - "formulas": [{ - "formula": "query1" - }], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {charge_description}", + "aggregator": "sum" + } + ], + "style": { + "palette": "classic" + }, + "formulas": [ + { + "formula": "query1" + } + ], "sort": { "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] } - }], - "style": { - "display": { - "type": "stacked", - "legend": "automatic" - }, - "palette": "datadog16" } - }, - "layout": { - "x": 7, - "y": 2, - "width": 5, - "height": 3 + ], + "type": "sunburst", + "legend": { + "type": "table" } - }, { - "id": 7671209769837654, - "definition": { - "title": "Cost per SKU", - "title_size": "16", - "title_align": "left", - "type": "toplist", - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {charge_description}", - "aggregator": "sum" - }], + }, + "layout": { + "x": 6, + "y": 5, + "width": 6, + "height": 4 + } + }, + { + "id": 3011381139040291, + "definition": { + "title": "Cost by Workspace", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {workspace_id}", + "aggregator": "sum" + } + ], "response_format": "scalar", - "formulas": [{ - "formula": "query1" - }], + "style": { + "palette": "classic" + }, + "formulas": [ + { + "formula": "query1" + } + ], "sort": { - "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] } - }], - "style": { - "display": { - "type": "stacked", - "legend": "automatic" - }, - "palette": "datadog16" } - }, - "layout": { - "x": 0, - "y": 5, - "width": 5, - "height": 3 + ], + "type": "sunburst", + "legend": { + "type": "table" } - }, { - "id": 6000488160971600, - "definition": { - "title": "Cost per SKU Over Time", - "title_size": "16", - "title_align": "left", - "show_legend": true, - "legend_layout": "auto", - "legend_columns": ["avg", "min", "max", "value", "sum"], - "time": { - "hide_incomplete_cost_data": true - }, - "type": "timeseries", - "requests": [{ - "formulas": [{ - "formula": "query1" - }], - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {charge_description}.rollup(sum, daily)" - }], + }, + "layout": { + "x": 0, + "y": 9, + "width": 6, + "height": 4 + } + }, + { + "id": 2933324945167311, + "definition": { + "title": "Cost by Workspace", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "hide_incomplete_cost_data": true + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {workspace_id}.rollup(sum, daily)" + } + ], "response_format": "timeseries", "style": { - "palette": "datadog16", + "palette": "classic", "order_by": "values", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" - }] + } + ] + }, + "layout": { + "x": 6, + "y": 9, + "width": 6, + "height": 4 + } + }, + { + "id": 2469704178227516, + "definition": { + "title": "Usage Changes by ServiceName and SKU", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true }, - "layout": { - "x": 5, - "y": 5, - "width": 7, - "height": 3 - } - }, { - "id": 4629842272544944, - "definition": { - "title": "Usage Changes by ServiceName and SKU", - "title_size": "16", - "title_align": "left", - "type": "query_table", - "requests": [{ + "type": "query_table", + "requests": [ + { "response_format": "scalar", - "queries": [{ - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {servicename,charge_description}", - "data_source": "cloud_cost", - "name": "query1", - "aggregator": "sum" - }, { - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {servicename,charge_description}", - "data_source": "cloud_cost", - "name": "query2", - "aggregator": "sum" - }], + "queries": [ + { + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {servicename,charge_description}", + "data_source": "cloud_cost", + "name": "query1", + "aggregator": "sum" + }, + { + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {servicename,charge_description}", + "data_source": "cloud_cost", + "name": "query2", + "aggregator": "sum" + } + ], "sort": { "count": 500, - "order_by": [{ - "type": "formula", - "index": 2, - "order": "desc" - }] - }, - "formulas": [{ - "alias": "Cost past 30 days", - "cell_display_mode": "number", - "formula": "query1" - }, { - "alias": "Cost 30 days prior", - "cell_display_mode": "number", - "formula": "default_zero(timeshift(query2, -2592000))" - }, { - "alias": "$ Change", - "cell_display_mode": "number", - "formula": "query1 - default_zero(timeshift(query2, -2592000))" - }, { - "alias": "% Change", - "conditional_formats": [{ - "comparator": ">", - "value": 15, - "palette": "white_on_red" - }, { - "comparator": ">", - "value": 10, - "palette": "black_on_light_red" - }, { - "comparator": ">", - "value": 5, - "palette": "black_on_light_yellow" - }], - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "percent" + "order_by": [ + { + "type": "formula", + "index": 2, + "order": "desc" } + ] + }, + "formulas": [ + { + "alias": "Cost past 30 days", + "cell_display_mode": "number", + "formula": "query1" }, - "formula": "(query1 - default_zero(timeshift(query2, -2592000))) / default_zero(timeshift(query2, -2592000)) * 100" - }] - }], - "has_search_bar": "always" - }, - "layout": { - "x": 0, - "y": 8, - "width": 12, - "height": 4 - } - }] + { + "alias": "Cost 30 days prior", + "cell_display_mode": "number", + "formula": "default_zero(timeshift(query2, -2592000))" + }, + { + "alias": "$ Change", + "cell_display_mode": "number", + "formula": "query1 - default_zero(timeshift(query2, -2592000))" + }, + { + "alias": "% Change", + "conditional_formats": [ + { + "comparator": ">", + "value": 15, + "palette": "white_on_red" + }, + { + "comparator": ">", + "value": 10, + "palette": "black_on_light_red" + }, + { + "comparator": ">", + "value": 5, + "palette": "black_on_light_yellow" + } + ], + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "(query1 - default_zero(timeshift(query2, -2592000))) / default_zero(timeshift(query2, -2592000)) * 100" + } + ] + } + ], + "has_search_bar": "always" + }, + "layout": { + "x": 0, + "y": 13, + "width": 12, + "height": 4 + } }, - "layout": { - "x": 0, - "y": 0, - "width": 12, - "height": 13 - } - }, { - "id": 624632189314412, - "definition": { - "title": "Cluster and Job Cost", - "background_color": "vivid_orange", - "show_title": true, - "type": "group", - "layout_type": "ordered", - "widgets": [{ - "id": 1937051372425104, - "definition": { - "type": "note", - "content": "Check out [**Data Jobs Monitoring**](https://app.datadoghq.com/data-jobs/) to get insights into how you can reduce costs by optimizing overprovisioned Databricks clusters and inefficient jobs.", - "background_color": "orange", - "font_size": "14", - "text_align": "center", - "vertical_align": "center", - "show_tick": false, - "tick_pos": "50%", - "tick_edge": "left", - "has_padding": true - }, - "layout": { - "x": 0, - "y": 0, - "width": 12, - "height": 1 - } - }, { - "id": 4205062182087582, - "definition": { - "title": "Cost per Cluster", - "title_size": "16", - "title_align": "left", - "type": "toplist", - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", - "aggregator": "sum" - }], - "response_format": "scalar", - "formulas": [{ - "formula": "query1" - }], - "sort": { - "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + { + "id": 624632189314412, + "definition": { + "title": "Cluster and Job Cost", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 1937051372425104, + "definition": { + "type": "note", + "content": "Check out [**Data Jobs Monitoring**](https://app.datadoghq.com/data-jobs/) to get insights into how you can reduce costs by optimizing overprovisioned Databricks clusters and inefficient jobs.", + "background_color": "green", + "font_size": "14", + "text_align": "center", + "vertical_align": "center", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 1 } - }], - "style": { - "display": { - "type": "stacked", - "legend": "automatic" + }, + { + "id": 4205062182087582, + "definition": { + "title": "Cost per Cluster", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "classic" + } }, - "palette": "datadog16" - } - }, - "layout": { - "x": 0, - "y": 1, - "width": 6, - "height": 3 - } - }, { - "id": 427998264646722, - "definition": { - "title": "Cost per Cluster Over Time", - "title_size": "16", - "title_align": "left", - "show_legend": false, - "legend_layout": "auto", - "legend_columns": ["avg", "min", "max", "value", "sum"], - "time": { - "hide_incomplete_cost_data": true + "layout": { + "x": 0, + "y": 1, + "width": 6, + "height": 3 + } }, - "type": "timeseries", - "requests": [{ - "formulas": [{ - "formula": "query1" - }], - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}.rollup(sum, daily)" - }], - "response_format": "timeseries", - "style": { - "palette": "datadog16", - "order_by": "values", - "line_type": "solid", - "line_width": "normal" + { + "id": 427998264646722, + "definition": { + "title": "Cost per Cluster Over Time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "hide_incomplete_cost_data": true + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}.rollup(sum, daily)" + } + ], + "response_format": "timeseries", + "style": { + "palette": "classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] }, - "display_type": "line" - }] - }, - "layout": { - "x": 6, - "y": 1, - "width": 6, - "height": 3 - } - }, { - "id": 1397446942991366, - "definition": { - "title": "Cost per Job", - "title_size": "16", - "title_align": "left", - "type": "toplist", - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:Databricks,$workspace_id,$cluster_id,$job_id,$charge_description,$servicename} by {job_id}", - "aggregator": "sum" - }], - "response_format": "scalar", - "formulas": [{ - "formula": "query1" - }], - "sort": { - "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + "layout": { + "x": 6, + "y": 1, + "width": 6, + "height": 3 } - }], - "style": { - "display": { - "type": "stacked", - "legend": "automatic" + }, + { + "id": 1397446942991366, + "definition": { + "title": "Cost per Job", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$workspace_id,$cluster_id,$job_id,$charge_description,$servicename} by {job_id}", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "classic" + } }, - "palette": "datadog16" - } - }, - "layout": { - "x": 0, - "y": 4, - "width": 6, - "height": 3 - } - }, { - "id": 3818638263356756, - "definition": { - "title": "Cost per Job Over Time", - "title_size": "16", - "title_align": "left", - "show_legend": false, - "legend_layout": "auto", - "legend_columns": ["avg", "min", "max", "value", "sum"], - "time": { - "hide_incomplete_cost_data": true + "layout": { + "x": 0, + "y": 4, + "width": 6, + "height": 3 + } }, - "type": "timeseries", - "requests": [{ - "formulas": [{ - "formula": "query1" - }], - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {job_id}.rollup(sum, daily)" - }], - "response_format": "timeseries", - "style": { - "palette": "datadog16", - "order_by": "values", - "line_type": "solid", - "line_width": "normal" + { + "id": 3818638263356756, + "definition": { + "title": "Cost per Job Over Time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "hide_incomplete_cost_data": true + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {job_id}.rollup(sum, daily)" + } + ], + "response_format": "timeseries", + "style": { + "palette": "classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] }, - "display_type": "line" - }] - }, - "layout": { - "x": 6, - "y": 4, - "width": 6, - "height": 3 - } - }] + "layout": { + "x": 6, + "y": 4, + "width": 6, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 17, + "width": 12, + "height": 8, + "is_column_break": true + } }, - "layout": { - "x": 0, - "y": 13, - "width": 12, - "height": 8, - "is_column_break": true - } - }, { - "id": 3812785000101592, - "definition": { - "title": "Cluster Cost & Resource Utilization ", - "background_color": "vivid_blue", - "show_title": true, - "type": "group", - "layout_type": "ordered", - "widgets": [{ - "id": 4442505189704330, - "definition": { - "type": "note", - "content": "Set up [Data Jobs Monitoring **(recommended)**](https://docs.datadoghq.com/data_jobs/) or our [Databricks Integration](https://docs.datadoghq.com/integrations/databricks/?tab=driveronly) to see data below. Dive deeper in [Databricks Overview](https://app.datadoghq.com/dash/integration/30437/databricks-overview?fromUser=false&refresh_mode=sliding&from_ts=1725645420213&to_ts=1725649020213&live=true).", - "background_color": "blue", - "font_size": "14", - "text_align": "center", - "vertical_align": "center", - "show_tick": false, - "tick_pos": "50%", - "tick_edge": "left", - "has_padding": true - }, - "layout": { - "x": 0, - "y": 0, - "width": 12, - "height": 1 - } - }, { - "id": 2449898261328074, - "definition": { - "title": "Cluster Cost and Observability Data", - "title_size": "16", - "title_align": "left", - "type": "query_table", - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query2", - "query": "avg:system.cpu.system{$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", - "aggregator": "avg" - }, { - "data_source": "metrics", - "name": "query3", - "query": "avg:system.mem.free{$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", - "aggregator": "avg" - }, { - "data_source": "metrics", - "name": "query4", - "query": "avg:system.mem.total{$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", - "aggregator": "avg" - }], - "response_format": "scalar", - "sort": { - "count": 50, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + { + "id": 3812785000101592, + "definition": { + "title": "Cluster Cost and Resource Utilization", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 4442505189704330, + "definition": { + "type": "note", + "content": "Set up [Data Jobs Monitoring **(recommended)**](https://docs.datadoghq.com/data_jobs/) or our [Databricks Integration](https://docs.datadoghq.com/integrations/databricks/?tab=driveronly) to see data below. Dive deeper in [Databricks Overview](/dash/integration/Databricks%20Overview%20Dashboard).", + "background_color": "green", + "font_size": "14", + "text_align": "center", + "vertical_align": "center", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true }, - "formulas": [{ - "cell_display_mode": "bar", - "alias": "Cost", - "formula": "query1" - }, { - "cell_display_mode": "bar", - "alias": "CPU % Utilization", - "formula": "query2" - }, { - "cell_display_mode": "bar", - "alias": "Memory Free", - "formula": "query3" - }, { - "cell_display_mode": "bar", - "alias": "Memory Total", - "formula": "query4" - }, { - "alias": "% Memory Allocation", - "formula": "1 - (query3 / query4)", - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "percent" + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 1 + } + }, + { + "id": 2449898261328074, + "definition": { + "title": "Cluster Cost and Observability Data", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "query_table", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:Databricks,$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query2", + "query": "avg:system.cpu.system{$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "avg:system.mem.free{$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", + "aggregator": "avg" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "avg:system.mem.total{$job_id,$workspace_id,$servicename,$charge_description,$cluster_id} by {cluster_id}", + "aggregator": "avg" + } + ], + "response_format": "scalar", + "sort": { + "count": 50, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "bar", + "alias": "Cost", + "formula": "query1" + }, + { + "cell_display_mode": "bar", + "alias": "CPU % Utilization", + "formula": "query2" + }, + { + "cell_display_mode": "bar", + "alias": "Memory Free", + "formula": "query3" + }, + { + "cell_display_mode": "bar", + "alias": "Memory Total", + "formula": "query4" + }, + { + "alias": "% Memory Allocation", + "formula": "1 - (query3 / query4)", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + } + } + ] } - } - }] - }], - "has_search_bar": "auto" - }, - "layout": { - "x": 0, - "y": 1, - "width": 12, - "height": 5 - } - }] + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 1, + "width": 12, + "height": 5 + } + } + ] + }, + "layout": { + "x": 0, + "y": 25, + "width": 12, + "height": 7 + } + } + ], + "template_variables": [ + { + "name": "servicename", + "prefix": "servicename", + "available_values": [], + "default": "*" + }, + { + "name": "charge_description", + "prefix": "charge_description", + "available_values": [], + "default": "*" + }, + { + "name": "job_id", + "prefix": "job_id", + "available_values": [], + "default": "*" + }, + { + "name": "cluster_id", + "prefix": "cluster_id", + "available_values": [], + "default": "*" }, - "layout": { - "x": 0, - "y": 21, - "width": 12, - "height": 7 + { + "name": "workspace_id", + "prefix": "workspace_id", + "available_values": [], + "default": "*" } - }], - "template_variables": [{ - "name": "servicename", - "prefix": "servicename", - "available_values": [], - "default": "*" - }, { - "name": "charge_description", - "prefix": "charge_description", - "available_values": [], - "default": "*" - }, { - "name": "job_id", - "prefix": "job_id", - "available_values": [], - "default": "*" - }, { - "name": "cluster_id", - "prefix": "cluster_id", - "available_values": [], - "default": "*" - }, { - "name": "workspace_id", - "prefix": "workspace_id", - "available_values": [], - "default": "*" - }], + ], "layout_type": "ordered", "notify_list": [], "reflow_type": "fixed" -} \ No newline at end of file +} diff --git a/datadog_checks_dev/changelog.d/21173.added b/datadog_checks_dev/changelog.d/21173.added new file mode 100644 index 0000000000000..ea06e9e0efd6b --- /dev/null +++ b/datadog_checks_dev/changelog.d/21173.added @@ -0,0 +1 @@ +Upgrade to psycopg3 diff --git a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/licenses.py b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/licenses.py index d5d766f8cfd6a..14b595088a9c8 100644 --- a/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/licenses.py +++ b/datadog_checks_dev/datadog_checks/dev/tooling/commands/validate/licenses.py @@ -53,9 +53,6 @@ 'mmh3': ['CC0-1.0'], # https://github.com/paramiko/paramiko/blob/master/LICENSE 'paramiko': ['LGPL-2.1-only'], - # https://github.com/psycopg/psycopg2/blob/master/LICENSE - # https://github.com/psycopg/psycopg2/blob/master/doc/COPYING.LESSER - 'psycopg2-binary': ['LGPL-3.0-only', 'BSD-3-Clause'], # https://github.com/psycopg/psycopg/blob/master/LICENSE.txt 'psycopg': ['LGPL-3.0-only'], # https://github.com/psycopg/psycopg/blob/master/psycopg_pool/LICENSE.txt diff --git a/klaviyo/CHANGELOG.md b/klaviyo/CHANGELOG.md index 8e97b8c43043c..5b6ae9a909955 100644 --- a/klaviyo/CHANGELOG.md +++ b/klaviyo/CHANGELOG.md @@ -1,6 +1,6 @@ # CHANGELOG - Klaviyo -## 1.0.0 / 2025-07-08 +## 1.0.0 / 2025-07-11 ***Added***: diff --git a/klaviyo/README.md b/klaviyo/README.md index 7040f8a727782..e78810ff977e7 100644 --- a/klaviyo/README.md +++ b/klaviyo/README.md @@ -16,8 +16,6 @@ Follow the instructions below to configure this integration for Klaviyo Marketin Within your Klaviyo account, first add the Datadog integration. The integration allows Datadog to see Klaviyo events and metrics via the Klaviyo API. -TODO: Insert screenshot of Datadog Integration Tile within Klaviyo - 1. Log in to your [Klaviyo account][2]. 2. In the left-side panel, navigate to **Integrations**. 3. Click **Add integrations**. diff --git a/klaviyo/assets/dashboards/klaviyo_ecommerce_overview.json b/klaviyo/assets/dashboards/klaviyo_ecommerce_overview.json new file mode 100644 index 0000000000000..a4cb64c859315 --- /dev/null +++ b/klaviyo/assets/dashboards/klaviyo_ecommerce_overview.json @@ -0,0 +1,1667 @@ +{ + "title": "Klaviyo - eCommerce Overview", + "description": "This Dashboard provides a comprehensive analysis of Klaviyo eCommerce events such as product purchases and returns.", + "widgets": [ + { + "id": 2074004104711888, + "definition": { + "type": "image", + "url": "/api/v2/images/abd3b6f9-af5e-4928-8f86-bc57f1905194", + "url_dark_theme": "/api/v2/images/68797a21-aec3-404c-9940-4110836143bc", + "sizing": "cover", + "has_background": true, + "has_border": true, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 1 + } + }, + { + "id": 1511984411456656, + "definition": { + "title": "Overview", + "background_color": "vivid_blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 7878012237089596, + "definition": { + "title": "Ordered Products", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Ordered Product\" $product" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count", + "metric": "@attributes.event_properties.Quantity" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 3414836774603344, + "definition": { + "title": "Placed Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Placed Order\" $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 715273201131506, + "definition": { + "title": "Placed Order Value", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + }, + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Placed Order\" $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "sum", + "metric": "@attributes.event_properties.$value" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 4, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 1383597922205404, + "definition": { + "title": "Active on Site", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Active on Site\"" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 2, + "width": 3, + "height": 2 + } + }, + { + "id": 3161686199415067, + "definition": { + "title": "Fulfilled Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Fulfilled Order\" $product" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 3, + "y": 2, + "width": 3, + "height": 2 + } + }, + { + "id": 5704262210117202, + "definition": { + "title": "Refunded Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Refunded Order\" $product" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 5, + "palette": "white_on_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 6, + "y": 2, + "width": 3, + "height": 1 + } + }, + { + "id": 3098870864966424, + "definition": { + "title": "Cancelled Orders", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Cancelled Order\" $product" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 10, + "palette": "white_on_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 6, + "y": 3, + "width": 3, + "height": 1 + } + }, + { + "id": 1725863741157462, + "definition": { + "title": "Monitor Summary", + "type": "manage_status", + "display_format": "list", + "color_preference": "text", + "hide_zero_counts": true, + "show_status": true, + "last_triggered_format": "relative", + "query": "tag:(source:klaviyo AND service:ecommerce-events)", + "sort": "status,asc", + "count": 50, + "start": 0, + "summary_type": "monitors", + "show_priority": false, + "show_last_triggered": false + }, + "layout": { + "x": 0, + "y": 4, + "width": 9, + "height": 3 + } + } + ] + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 8 + } + }, + { + "id": 8917290706883810, + "definition": { + "type": "note", + "content": "Gain better visibility into product purchase activity by monitoring Klaviyo eCommerce events with this dashboard.\n\nThe events fall into Klaviyo metric categories such as _Added to Cart_, _Viewed Product_, _Placed Order_, and more.\n\nAn _Ordered Product_ event from Klaviyo defines a single product which is part of a purchase, whereas a _Placed Order_ encompasses the set of products in an order.\n", + "background_color": "blue", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 1, + "width": 3, + "height": 7 + } + }, + { + "id": 17150730536389, + "definition": { + "title": "Top Products", + "background_color": "vivid_blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 6836248737505238, + "definition": { + "title": "Most Ordered Products", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Ordered Product\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.products", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "@attributes.event_properties.Quantity" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count", + "metric": "@attributes.event_properties.Quantity" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "text_formats": [], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "bar", + "formula": "query1" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 0, + "width": 5, + "height": 5 + } + }, + { + "id": 991970123942130, + "definition": { + "title": "Top Active on Site URLs", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Active on Site\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@http.url", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 5, + "y": 0, + "width": 7, + "height": 3 + } + }, + { + "id": 5154411203269646, + "definition": { + "title": "Most Viewed Products", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Viewed Product\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.products", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + } + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 5, + "y": 3, + "width": 7, + "height": 2 + } + }, + { + "id": 7502300914199598, + "definition": { + "title": "Top Ordered Product Types", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Placed Order\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.product_categories", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 5, + "width": 5, + "height": 4 + } + }, + { + "id": 6281401150190870, + "definition": { + "title": "Most Refunded Products", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Refunded Order\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.products", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + } + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_yellow" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 5, + "y": 5, + "width": 7, + "height": 2 + } + }, + { + "id": 8983025080998243, + "definition": { + "title": "Top Refunded Order Reasons", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Refunded Order\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.refund_reason", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + } + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_yellow" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 5, + "y": 7, + "width": 7, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 10 + } + }, + { + "id": 5618460877750433, + "definition": { + "title": "Placed Order Performance", + "background_color": "vivid_purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 3154476233147349, + "definition": { + "type": "note", + "content": "Compare product ordering timelines against email and sms campaigns. \n\nThe _Placed Order_ line represents all placed orders while the individual campaigns are each graphed according to total Email and SMS messages received.", + "background_color": "purple", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "right", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "id": 8329971943698914, + "definition": { + "title": "Placed Orders Compared to Campaign Email/SMS activity", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Campaign", + "style": { + "palette": "grey" + }, + "formula": "query1" + }, + { + "alias": "Placed Order", + "style": { + "palette": "dd20", + "palette_index": 13 + }, + "formula": "query2" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:(\"Received SMS\" OR \"Sent SMS\" OR \"Received Email\") $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Placed Order\" $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ], + "markers": [] + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 4 + } + }, + { + "id": 4559526856146998, + "definition": { + "title": "Average Placed Order Value", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query2", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + } + ], + "queries": [ + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Placed Order\" $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "avg", + "metric": "@attributes.event_properties.$value" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 4, + "width": 6, + "height": 2 + } + }, + { + "id": 8142366750902173, + "definition": { + "title": "Change in Placed Orders From last Month", + "title_size": "16", + "title_align": "left", + "type": "change", + "requests": [ + { + "increase_good": true, + "order_by": "change", + "change_type": "absolute", + "order_dir": "desc", + "response_format": "scalar", + "formulas": [ + { + "formula": "month_before(query1)" + }, + { + "formula": "query1", + "alias": "Placed Orders" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Placed Order\" $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ] + } + ] + }, + "layout": { + "x": 6, + "y": 4, + "width": 6, + "height": 2 + } + }, + { + "id": 1725057173420981, + "definition": { + "title": "Placed vs Fulfilled Order Value", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + }, + "alias": "Placed Orders", + "formula": "query2" + }, + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + }, + "alias": "Fulfilled Orders", + "formula": "query1" + } + ], + "queries": [ + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Placed Order\" $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "avg", + "metric": "@attributes.event_properties.$value" + }, + "storage": "hot" + }, + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Fulfilled Order\" $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "avg", + "metric": "@attributes.event_properties.$value" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "line" + } + ] + }, + "layout": { + "x": 0, + "y": 6, + "width": 12, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 10, + "is_column_break": true + } + }, + { + "id": 1592614727652525, + "definition": { + "title": "Order Details", + "background_color": "vivid_blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 7764853164683600, + "definition": { + "title": "Billing Addresses", + "title_size": "16", + "title_align": "left", + "type": "geomap", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.billing_region_code", + "limit": 250, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 250, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "palette": "hostmap_blues", + "palette_flip": false + }, + "view": { + "focus": "WORLD" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 1186112858039406, + "definition": { + "title": "Shipping Addresses", + "title_size": "16", + "title_align": "left", + "type": "geomap", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.shipping_region_code", + "limit": 250, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 250, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "palette": "hostmap_blues", + "palette_flip": false + }, + "view": { + "focus": "WORLD" + } + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 3 + } + }, + { + "id": 8819991829077414, + "definition": { + "title": "Top Billing Zip Codes", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Placed Order\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.billing_zip_code", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "alias": "Count", + "formula": "query1" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 3, + "width": 6, + "height": 2 + } + }, + { + "id": 1080847351359035, + "definition": { + "title": "Top Shipping Zip Codes", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:ecommerce-events @metric_name:\"Placed Order\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.shipping_zip_code", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "alias": "Count", + "formula": "query1" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 6, + "y": 3, + "width": 6, + "height": 2 + } + }, + { + "id": 897131023425208, + "definition": { + "title": "Order Details ", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_stream", + "query_string": "source:klaviyo service:ecommerce-events @metric_name:\"Ordered Product\" $product $billing_region_code $shipping_region_code $billing_zipcode $shipping_zipcode", + "indexes": [ + "*" + ], + "storage": "hot" + }, + "columns": [ + { + "field": "status_line", + "width": "auto" + }, + { + "field": "timestamp", + "width": "auto" + }, + { + "field": "@klaviyo.products", + "width": "auto" + }, + { + "field": "@attributes.event_properties.Quantity", + "width": "auto" + }, + { + "field": "@attributes.event_properties.$value", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 0, + "y": 5, + "width": 12, + "height": 4 + } + } + ] + }, + "layout": { + "x": 0, + "y": 10, + "width": 12, + "height": 10 + } + }, + { + "id": 1594770383480588, + "definition": { + "type": "note", + "content": "All Klaviyo events reference a metric name. The metric name falls into either the _marketing-events_, _ecommerce-events_, or _other-events_ service, or category.\n\nIf an accumulation of _other-events_ appears in this list, then Datadog can be contacted to consider defining a separate service type to more effectively display the event data.", + "background_color": "purple", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "right", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 6 + } + }, + { + "id": 3725343105146853, + "definition": { + "title": "All Events List", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_stream", + "query_string": "source:klaviyo service:ecommerce-events", + "indexes": [], + "storage": "hot" + }, + "columns": [ + { + "field": "status_line", + "width": "auto" + }, + { + "field": "timestamp", + "width": "auto" + }, + { + "field": "@metric_name", + "width": "auto" + }, + { + "field": "service", + "width": "auto" + }, + { + "field": "status", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 6 + } + } + ], + "template_variables": [ + { + "name": "product", + "prefix": "@klaviyo.products", + "available_values": [], + "default": "*" + }, + { + "name": "billing_region_code", + "prefix": "@klaviyo.billing_region_code", + "available_values": [], + "default": "*" + }, + { + "name": "shipping_region_code", + "prefix": "@klaviyo.shipping_region_code", + "available_values": [], + "default": "*" + }, + { + "name": "billing_zipcode", + "prefix": "@klaviyo.billing_zip_code", + "available_values": [], + "default": "*" + }, + { + "name": "shipping_zipcode", + "prefix": "@klaviyo.shipping_zip_code", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} diff --git a/klaviyo/assets/dashboards/klaviyo_marketing_overview.json b/klaviyo/assets/dashboards/klaviyo_marketing_overview.json new file mode 100644 index 0000000000000..12b87e75a3d72 --- /dev/null +++ b/klaviyo/assets/dashboards/klaviyo_marketing_overview.json @@ -0,0 +1,3170 @@ +{ + "title": "Klaviyo - Marketing Overview", + "description": "This Dashboard provides a comprehensive analysis of Klaviyo marketing events", + "widgets": [ + { + "id": 2074004104711888, + "definition": { + "type": "image", + "url": "/api/v2/images/abd3b6f9-af5e-4928-8f86-bc57f1905194", + "url_dark_theme": "/api/v2/images/68797a21-aec3-404c-9940-4110836143bc", + "sizing": "cover", + "has_background": true, + "has_border": true, + "vertical_align": "center", + "horizontal_align": "center" + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 1 + } + }, + { + "id": 1511984411456656, + "definition": { + "title": "Overview", + "background_color": "white", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 1383597922205404, + "definition": { + "title": "Received Email", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 0, + "width": 2, + "height": 1 + } + }, + { + "id": 7878012237089596, + "definition": { + "title": "Opened Email", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Opened Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 2, + "y": 0, + "width": 2, + "height": 1 + } + }, + { + "id": 5364741650165821, + "definition": { + "title": "Clicked Email", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 4, + "y": 0, + "width": 2, + "height": 1 + } + }, + { + "id": 5104249621174987, + "definition": { + "title": "Dropped/Bounced Email", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:(\"Dropped Email\" OR \"Bounced Email\") $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 1 + } + }, + { + "id": 3115102914114161, + "definition": { + "title": "Sent SMS", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Sent SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 1, + "width": 2, + "height": 1 + } + }, + { + "id": 6589006754909623, + "definition": { + "title": "Clicked SMS", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 2, + "y": 1, + "width": 2, + "height": 1 + } + }, + { + "id": 743460417967940, + "definition": { + "title": "Received SMS", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 4, + "y": 1, + "width": 2, + "height": 1 + } + }, + { + "id": 1872447279034631, + "definition": { + "title": "Failed to Deliver SMS", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Failed to Deliver SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 6, + "y": 1, + "width": 3, + "height": 1 + } + }, + { + "id": 7952409078053929, + "definition": { + "title": "Monitor Status", + "type": "manage_status", + "display_format": "list", + "color_preference": "text", + "hide_zero_counts": true, + "show_status": true, + "last_triggered_format": "relative", + "query": "tag:(source:klaviyo AND service:marketing-events)", + "sort": "status,asc", + "count": 50, + "start": 0, + "summary_type": "monitors", + "show_priority": false, + "show_last_triggered": false + }, + "layout": { + "x": 0, + "y": 2, + "width": 9, + "height": 3 + } + }, + { + "id": 6836248737505238, + "definition": { + "title": "Most Clicked URLs", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:(\"Clicked Email\" OR \"Clicked SMS\") $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + }, + { + "facet": "@http.url", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 100, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "count", + "formula": "query1" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 5, + "width": 9, + "height": 3 + } + } + ] + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 9 + } + }, + { + "id": 8917290706883810, + "definition": { + "type": "note", + "content": "Gain better visibility into your email marketing performance by monitoring Klaviyo Marketing events with this dashboard.\n\nThe events, which include the byproduct of flows, fall into Klaviyo metric categories such as Bounced Email, Clicked Email, Sent SMS, Failed to Deliver SMS, and more.\n\nUse the `campaign_name` and 'flow_name` template variables to restrict data to a subset of campaigns or flows.", + "background_color": "blue", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "top", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 1, + "width": 3, + "height": 8 + } + }, + { + "id": 7150718728972441, + "definition": { + "title": "Email", + "background_color": "vivid_blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 3109671020016291, + "definition": { + "type": "note", + "content": "Email events categorize the deliverability of the email as well as click activity within the opened email.\n\nSee the [description of email deliverability](https://help.klaviyo.com/hc/en-us/articles/115000201131).", + "background_color": "blue", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "right", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 3 + } + }, + { + "id": 8329971943698914, + "definition": { + "title": "Email Campaign Performance Summary", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Opened Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query3", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query4", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Bounced Email\" @klaviyo.bounce_type:Hard $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query5", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Bounced Email\" @klaviyo.bounce_type:Soft $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query6", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Marked Email as Spam\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query7", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked email to unsubscribe\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 70, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Received", + "formula": "query1" + }, + { + "cell_display_mode": "number", + "alias": "Opened", + "formula": "query2" + }, + { + "cell_display_mode": "number", + "alias": "Clicked", + "formula": "query3" + }, + { + "cell_display_mode": "number", + "alias": "Hard Bounced", + "formula": "query4" + }, + { + "cell_display_mode": "number", + "alias": "Soft Bounced", + "formula": "query5" + }, + { + "cell_display_mode": "number", + "alias": "Marked as Spam", + "formula": "query6" + }, + { + "cell_display_mode": "number", + "alias": "Unsubscribed", + "formula": "query7" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 3 + } + }, + { + "id": 1601260360695293, + "definition": { + "title": "Opened Email Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "min(100, (query1 / query2) * 100)" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Opened Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": "<", + "value": 10, + "palette": "white_on_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 8108409578166421, + "definition": { + "type": "note", + "content": "Opened and Clicked Email percentages are calculated by comparing the volume of Opened and Clicked Email compared to Received Email **within the same time span of the dashboard view**.", + "background_color": "blue", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "right", + "has_padding": true + }, + "layout": { + "x": 4, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 3381654448226522, + "definition": { + "title": "Clicked Email Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "min(100, (query1 / query2) * 100)" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": "<", + "value": 10, + "palette": "white_on_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 8, + "y": 3, + "width": 4, + "height": 3 + } + }, + { + "id": 897131023425208, + "definition": { + "title": "Email Open and Clicked Activity ", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + }, + { + "formula": "query2" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Opened Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "green", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ], + "yaxis": { + "include_zero": true, + "scale": "linear", + "min": "auto", + "max": "auto" + } + }, + "layout": { + "x": 0, + "y": 6, + "width": 6, + "height": 4 + } + }, + { + "id": 763900043629789, + "definition": { + "title": "Clicked Email Locations", + "title_size": "16", + "title_align": "left", + "type": "geomap", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@network.client.geoip.subdivision.iso_code", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "palette": "yellow_to_green", + "palette_flip": false + }, + "view": { + "focus": "WORLD" + } + }, + "layout": { + "x": 6, + "y": 6, + "width": 6, + "height": 4 + } + }, + { + "id": 1554629846571348, + "definition": { + "title": "Email Event Trends: Spam, Bounces, and Unsubscribes", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "style": { + "palette": "warm", + "palette_index": 3 + }, + "alias": "Hard Bounced", + "formula": "query1" + }, + { + "style": { + "palette": "orange", + "palette_index": 4 + }, + "alias": "Soft Bounced", + "formula": "query2" + }, + { + "style": { + "palette": "warm", + "palette_index": 0 + }, + "alias": "Marked as Spam", + "formula": "query3" + }, + { + "style": { + "palette": "warm", + "palette_index": 4 + }, + "alias": "Unsubscribed", + "formula": "query4" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Bounced Email\" @klaviyo.bounce_type:Hard $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Bounced Email\" @klaviyo.bounce_type:Soft $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query3", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Marked Email as Spam\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query4", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Unsubscribed from Email Marketing\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "area" + } + ] + }, + "layout": { + "x": 0, + "y": 10, + "width": 6, + "height": 3 + } + }, + { + "id": 2556161432574455, + "definition": { + "title": "Email Submission Breakdown", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:(\"Bounced Email\" OR \"Dropped Email\" OR \"Received Email\") $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@metric_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "style": { + "palette": "datadog16" + }, + "formulas": [ + { + "formula": "query2" + } + ], + "sort": { + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ], + "count": 10 + } + } + ], + "type": "sunburst", + "legend": { + "type": "automatic" + } + }, + "layout": { + "x": 6, + "y": 10, + "width": 6, + "height": 3 + } + }, + { + "id": 6281401150190870, + "definition": { + "title": "Top Campaigns for Soft Bounced Email Events", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Bounced Email\" @klaviyo.bounce_type:Soft $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_yellow" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 13, + "width": 6, + "height": 3 + } + }, + { + "id": 5154411203269646, + "definition": { + "title": "Most Opened Campaigns", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Opened Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 6, + "y": 13, + "width": 6, + "height": 3 + } + }, + { + "id": 1122235000103890, + "definition": { + "title": "Top Campaigns for Hard Bounced Email Events", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Bounced Email\" @klaviyo.bounce_type:Hard $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_red" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 16, + "width": 6, + "height": 3 + } + }, + { + "id": 1499029347730738, + "definition": { + "title": "Top Campaigns for Spam Email Events", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Marked Email as Spam\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_yellow" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 6, + "y": 16, + "width": 6, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 9, + "width": 12, + "height": 20 + } + }, + { + "id": 1155806806188451, + "definition": { + "title": "Subscription Updates", + "background_color": "vivid_blue", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 7183874539954419, + "definition": { + "title": "Subscribed To", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Subscribed to List\"" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Subscribed to Email Marketing\"" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query3", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Subscribed to SMS Marketing\"" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query4", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Subscribed to SMS Transactional\"" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ], + "cell_display_mode": "bar", + "alias": "List", + "formula": "query1" + }, + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ], + "cell_display_mode": "bar", + "alias": "Email Marketing", + "formula": "query2" + }, + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ], + "cell_display_mode": "bar", + "alias": "SMS Marketing", + "formula": "query3" + }, + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_green" + } + ], + "cell_display_mode": "bar", + "alias": "SMS Transactional", + "formula": "query4" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 0, + "width": 6, + "height": 2 + } + }, + { + "id": 8909761842733828, + "definition": { + "title": "Unsubscribed From", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "name": "query5", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Unsubscribed from Email Marketing\"" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query6", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Unsubscribed from SMS Marketing\"" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ], + "cell_display_mode": "bar", + "alias": "Email Marketing", + "formula": "query5" + }, + { + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "black_on_light_yellow" + } + ], + "cell_display_mode": "bar", + "alias": "SMS Marketing", + "formula": "query6" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 6, + "y": 0, + "width": 6, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 29, + "width": 12, + "height": 3 + } + }, + { + "id": 3657744283017699, + "definition": { + "title": "Flow Activity", + "background_color": "vivid_purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 1101349622865755, + "definition": { + "type": "note", + "content": "Shown here are marketing events associated with Klaviyo Flows. Flows are triggered by conditions such as eCommerce events and this display can aid in understanding the comparative volume among active flows.\n\nSee the [Klaviyo documentation](https://help.klaviyo.com/hc/en-us/articles/115002779351) describing per-flow analytics.", + "background_color": "pink", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "bottom", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 2 + } + }, + { + "id": 3279270655394946, + "definition": { + "title": "Events Per Flow", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "vertical", + "legend_columns": [ + "sum" + ], + "type": "timeseries", + "requests": [ + { + "response_format": "timeseries", + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @klaviyo.flow_name:* $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@metric_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + }, + { + "facet": "@klaviyo.flow_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "style": { + "palette": "dog_classic", + "order_by": "values", + "order_reverse": false, + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ], + "markers": [ + { + "value": "y = 0", + "display_type": "error dashed" + } + ] + }, + "layout": { + "x": 0, + "y": 2, + "width": 12, + "height": 6 + } + }, + { + "id": 2843939270953838, + "definition": { + "title": "Summary", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:(\"Received SMS\" OR \"Received Email\") $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.flow_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Opened Email\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.flow_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query3", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:(\"Clicked SMS\" OR \"Clicked Email\") $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.flow_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query4", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:(\"Dropped Email\" OR \"Bounced Email\" OR \"Failed to Deliver SMS\") $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.flow_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query6", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Marked Email as Spam\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.flow_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query7", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked email to unsubscribe\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.flow_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 60, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Received", + "formula": "query1" + }, + { + "cell_display_mode": "number", + "alias": "Opened", + "formula": "query2" + }, + { + "cell_display_mode": "number", + "alias": "Clicked", + "formula": "query3" + }, + { + "cell_display_mode": "number", + "alias": "Delivery Failed", + "formula": "query4" + }, + { + "cell_display_mode": "number", + "alias": "Marked as Spam", + "formula": "query6" + }, + { + "cell_display_mode": "number", + "alias": "Unsubscribed", + "formula": "query7" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 0, + "y": 8, + "width": 12, + "height": 4 + } + }, + { + "id": 1305627879238495, + "definition": { + "title": "Event Log", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_stream", + "query_string": "source:klaviyo service:marketing-events @klaviyo.flow_name:* $campaign_name $flow_name", + "indexes": [ + "*" + ], + "storage": "hot" + }, + "columns": [ + { + "field": "status_line", + "width": "auto" + }, + { + "field": "timestamp", + "width": "auto" + }, + { + "field": "@flow_name", + "width": "auto" + }, + { + "field": "content", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 0, + "y": 12, + "width": 12, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 16, + "is_column_break": true + } + }, + { + "id": 1363279763363657, + "definition": { + "title": "SMS", + "background_color": "vivid_purple", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 6131215568044537, + "definition": { + "type": "note", + "content": "SMS events are categorized by submission and receipt activity. Received SMS represents outbound SMS receipt while Sent SMS is when a user sends you an inbound message.\n\nSee the [SMS event descriptions](https://help.klaviyo.com/hc/en-us/articles/360035345572).", + "background_color": "pink", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "right", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 4 + } + }, + { + "id": 4091562417354812, + "definition": { + "title": "SMS Campaign Performance Summary", + "title_size": "16", + "title_align": "left", + "type": "query_table", + "requests": [ + { + "queries": [ + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query3", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query4", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Failed to Deliver SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query6", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received Automated Response SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + } + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query7", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Failed to Deliver Automated Response SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query5", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Sent SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "sort": { + "count": 60, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + }, + "formulas": [ + { + "cell_display_mode": "number", + "alias": "Received", + "formula": "query2" + }, + { + "cell_display_mode": "number", + "alias": "Clicked", + "formula": "query3" + }, + { + "cell_display_mode": "number", + "alias": "Failed to Deliver", + "formula": "query4" + }, + { + "cell_display_mode": "number", + "alias": "Auto Response", + "formula": "query6" + }, + { + "cell_display_mode": "number", + "alias": "Unsubscribed", + "formula": "query7" + }, + { + "cell_display_mode": "number", + "alias": "Sent", + "formula": "query5" + } + ] + } + ], + "has_search_bar": "auto" + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 4 + } + }, + { + "id": 8824704027595259, + "definition": { + "title": "Received and Clicked SMS Activity ", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + }, + { + "formula": "query2" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ], + "yaxis": { + "include_zero": true, + "scale": "linear", + "min": "auto", + "max": "auto" + } + }, + "layout": { + "x": 0, + "y": 4, + "width": 12, + "height": 3 + } + }, + { + "id": 4902837865430878, + "definition": { + "title": "Clicked SMS Percentage", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "(query1 / query2) * 100" + } + ], + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": "<", + "value": 10, + "palette": "white_on_yellow" + } + ] + } + ], + "autoscale": true, + "precision": 2, + "timeseries_background": { + "type": "area" + } + }, + "layout": { + "x": 0, + "y": 7, + "width": 12, + "height": 2 + } + }, + { + "id": 4864546405625946, + "definition": { + "title": "SMS Trends: Delivery Failures and Unsubscribes", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "horizontal", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "style": { + "palette": "warm", + "palette_index": 0 + }, + "alias": "Marked as Spam", + "formula": "query3" + }, + { + "style": { + "palette": "warm", + "palette_index": 4 + }, + "alias": "Unsubscribed", + "formula": "query4" + } + ], + "queries": [ + { + "name": "query3", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Failed to Deliver SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query4", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Unsubscribed from SMS Marketing\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "timeseries", + "style": { + "palette": "dog_classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "area" + } + ] + }, + "layout": { + "x": 0, + "y": 9, + "width": 6, + "height": 4 + } + }, + { + "id": 475741720783093, + "definition": { + "title": "Email Submission Breakdown", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "scalar", + "queries": [ + { + "name": "query2", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:(\"Failed to Deliver SMS\" OR \"Received SMS\") $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@metric_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "style": { + "palette": "datadog16" + }, + "formulas": [ + { + "formula": "query2" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "type": "sunburst", + "legend": { + "type": "automatic" + } + }, + "layout": { + "x": 6, + "y": 9, + "width": 6, + "height": 4 + } + }, + { + "id": 8344332108104141, + "definition": { + "title": "Campaign Message IDs", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_pattern_stream", + "query_string": "source:klaviyo service:marketing-events @metric_name:\"Received SMS\" $campaign_name $flow_name", + "indexes": [], + "clustering_pattern_field_path": "@klaviyo.campaign_name", + "group_by": [ + { + "facet": "@attributes.event_properties.extra.Message ID" + } + ], + "storage": "hot" + }, + "columns": [ + { + "field": "status_line", + "width": "auto" + }, + { + "field": "matches", + "width": "auto" + }, + { + "field": "@attributes.event_properties.extra.Message ID", + "width": "auto" + }, + { + "field": "@klaviyo.campaign_name", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 0, + "y": 13, + "width": 12, + "height": 2 + } + }, + { + "id": 6244152640067695, + "definition": { + "title": "Campaign Message Bodies", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_pattern_stream", + "query_string": "source:klaviyo service:marketing-events @metric_name:\"Received SMS\" $campaign_name $flow_name", + "indexes": [], + "clustering_pattern_field_path": "@klaviyo.campaign_name", + "group_by": [ + { + "facet": "@attributes.event_properties.extra.Message Body" + } + ], + "storage": "hot" + }, + "columns": [ + { + "field": "status_line", + "width": "auto" + }, + { + "field": "@attributes.event_properties.extra.Message Body", + "width": "full" + }, + { + "field": "@klaviyo.campaign_name", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 0, + "y": 15, + "width": 12, + "height": 2 + } + }, + { + "id": 2524442986239370, + "definition": { + "title": "Most Clicked Campaigns", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Clicked SMS\" $campaign_name $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 17, + "width": 6, + "height": 2 + } + }, + { + "id": 7034116953269158, + "definition": { + "title": "Most Unsubscribed Messages IDs", + "title_size": "16", + "title_align": "left", + "type": "toplist", + "requests": [ + { + "queries": [ + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Unsubscribed from SMS Marketing\" $flow_name" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@attributes.event_properties.extra.Message ID", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "response_format": "scalar", + "conditional_formats": [ + { + "comparator": ">", + "value": 0, + "palette": "white_on_red" + } + ], + "formulas": [ + { + "formula": "query1" + } + ], + "sort": { + "count": 10, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 6, + "y": 17, + "width": 6, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 16, + "width": 12, + "height": 20 + } + }, + { + "id": 1076574015997369, + "definition": { + "type": "note", + "content": "All Klaviyo events reference a metric name. The metric name falls into either the _marketing-events_, _ecommerce-events_, or _other-events_ service, or category.\n\nIf an accumulation of _other-events_ appears in this list, then Datadog can be contacted to consider defining a separate service type to more effectively display the event data.", + "background_color": "pink", + "font_size": "14", + "text_align": "left", + "vertical_align": "center", + "show_tick": true, + "tick_pos": "50%", + "tick_edge": "right", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 5 + } + }, + { + "id": 3725343105146853, + "definition": { + "title": "All Events List", + "title_size": "16", + "title_align": "left", + "requests": [ + { + "response_format": "event_list", + "query": { + "data_source": "logs_stream", + "query_string": "source:klaviyo service:*", + "indexes": [], + "storage": "hot" + }, + "columns": [ + { + "field": "status_line", + "width": "auto" + }, + { + "field": "timestamp", + "width": "auto" + }, + { + "field": "@metric_name", + "width": "auto" + }, + { + "field": "service", + "width": "auto" + }, + { + "field": "status", + "width": "auto" + } + ] + } + ], + "type": "list_stream" + }, + "layout": { + "x": 3, + "y": 0, + "width": 9, + "height": 5 + } + } + ], + "template_variables": [ + { + "name": "campaign_name", + "prefix": "@klaviyo.campaign_name", + "available_values": [], + "default": "*" + }, + { + "name": "flow_name", + "prefix": "@klaviyo.flow_name", + "available_values": [], + "default": "*" + } + ], + "layout_type": "ordered", + "notify_list": [], + "reflow_type": "fixed" +} diff --git a/klaviyo/assets/klaviyo.svg b/klaviyo/assets/klaviyo.svg new file mode 100644 index 0000000000000..cb1f8dfab5847 --- /dev/null +++ b/klaviyo/assets/klaviyo.svg @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/klaviyo/assets/logs/klaviyo.yaml b/klaviyo/assets/logs/klaviyo.yaml new file mode 100644 index 0000000000000..ff5b1922324f9 --- /dev/null +++ b/klaviyo/assets/logs/klaviyo.yaml @@ -0,0 +1,391 @@ +id: klaviyo +metric_id: klaviyo +backend_only: false +facets: + - groups: + - Web Access + name: URL Path + path: http.url + source: log + - groups: + - Geoip + name: City Name + path: network.client.geoip.city.name + source: log + - groups: + - Geoip + name: Continent Code + path: network.client.geoip.continent.code + source: log + - groups: + - Geoip + name: Continent Name + path: network.client.geoip.continent.name + source: log + - groups: + - Geoip + name: Country ISO Code + path: network.client.geoip.country.iso_code + source: log + - groups: + - Geoip + name: Country Name + path: network.client.geoip.country.name + source: log + - groups: + - Geoip + name: Subdivision ISO Code + path: network.client.geoip.subdivision.iso_code + source: log + - groups: + - Geoip + name: Subdivision Name + path: network.client.geoip.subdivision.name + source: log + - description: '' + facetType: list + groups: + - others + name: Campaign Name + path: klaviyo.campaign_name + source: log + type: string + - description: '' + facetType: list + groups: + - others + name: Flow Name + path: klaviyo.flow_name + source: log + type: string +pipeline: + type: pipeline + name: Klaviyo + enabled: true + filter: + query: source:klaviyo + processors: + - type: date-remapper + name: Define `attribute.datetime` as the official date of the log + enabled: true + sources: + - attributes.datetime + - name: Define `service` based on lookup of `metric_name` + enabled: true + source: metric_name + target: service + lookupTable: |- + Active on Site,ecommerce-events + Added to Cart,ecommerce-events + Cancelled Order,ecommerce-events + Fulfilled Order,ecommerce-events + Ordered Product,ecommerce-events + Placed Order,ecommerce-events + Refunded Order,ecommerce-events + Started Checkout,ecommerce-events + Checkout Started,ecommerce-events + Viewed Product,ecommerce-events + Bounced Email,marketing-events + Clicked email to unsubscribe,marketing-events + Clicked Email,marketing-events + Clicked SMS,marketing-events + Dropped Email,marketing-events + Failed to Deliver Automated Response SMS,marketing-events + Failed to Deliver SMS,marketing-events + Marked Email as Spam,marketing-events + Opened Email,marketing-events + Received Automated Response SMS,marketing-events + Received Email,marketing-events + Received SMS,marketing-events + Sent SMS,marketing-events + Subscribed to SMS Marketing,marketing-events + Subscribed to SMS Transactional,marketing-events + Subscribed to Email Marketing,marketing-events + Subscribed to List,marketing-events + Unsubscribed from Email Marketing,marketing-events + Unsubscribed from SMS Marketing,marketing-events + defaultLookup: other-events + type: lookup-processor + - type: attribute-remapper + name: "Remapper: Map `flow_name` to `klaviyo.flow_name`" + enabled: true + sources: + - flow_name + sourceType: attribute + target: klaviyo.flow_name + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: service-remapper + name: Define `service` as the official service of the log + enabled: true + sources: + - service + - type: pipeline + name: marketing events + enabled: true + filter: + query: source:klaviyo service:marketing-events + processors: + - type: attribute-remapper + name: 'Remapper: Map `attributes.event_properties."Campaign Name"` to + `klaviyo.campaign_name`' + enabled: true + sources: + - attributes.event_properties.Campaign Name + - attributes.event_properties.Message Name + sourceType: attribute + target: klaviyo.campaign_name + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: 'Remapper: Map `attributes.event_properties."Bounce Type"` to + `klaviyo.bounce_type`' + enabled: true + sources: + - attributes.event_properties.Bounce Type + sourceType: attribute + target: klaviyo.bounce_type + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: "Remapper: Map `attributes.event_properties.URL` to `http.url`" + enabled: true + sources: + - attributes.event_properties.URL + sourceType: attribute + target: http.url + targetType: attribute + preserveSource: true + overrideOnConflict: false + - name: Lookup for `metric_name` to `status` field + enabled: true + source: metric_name + target: status + lookupTable: |- + Bounced Email,warning + Clicked email to unsubscribe,warning + Clicked Email,info + Clicked SMS,info + Dropped Email,error + Failed to Deliver Automated Response SMS,error + Failed to Deliver SMS,error + Marked Email as Spam,warning + Opened Email,info + Received Automated Response SMS,info + Received Email,info + Received SMS,info + Sent SMS,info + Subscribed to SMS Marketing,info + Subscribed to SMS Transactional,info + Subscribed to Email Marketing,info + Subscribed to List,info + Unsubscribed from Email Marketing,warning + Unsubscribed from SMS Marketing,warning + defaultLookup: info + type: lookup-processor + - type: status-remapper + name: Define `status` as the official status of the log + enabled: true + sources: + - status + - type: geo-ip-parser + name: Define `attributes.event_properties._ip` as GeoIp source + enabled: true + sources: + - attributes.event_properties._ip + target: network.client.geoip + ip_processing_behavior: do-nothing + - type: pipeline + name: ecommerce events - Ordered/Viewed Product + enabled: true + filter: + query: source:klaviyo service:ecommerce-events @metric_name:("Ordered Product" + OR "Viewed Product") + processors: + - type: string-builder-processor + name: Define `klaviyo.products` equal to `ProductID` - 'Name`|`ProductName` + enabled: true + template: "%{attributes.event_properties.ProductID} - + %{attributes.event_properties.Name}%{attributes.event_properties.Pr\ + oductName}" + target: klaviyo.products + replaceMissing: true + - type: pipeline + name: ecommerce events - Klaviyo Placed/Refunded Order + enabled: true + filter: + query: source:klaviyo service:ecommerce-events + -@attributes.event_properties.$extra @metric_name:("Placed Order" OR + "Refunded Order") + processors: + - type: string-builder-processor + name: Define `klaviyo.shipping_region_code` equal to + ShippingAddress.CountryCode-ShippingAddress.RegionCode + enabled: true + template: "%{attributes.event_properties.ShippingAddress.CountryCode}-%{attribu\ + tes.event_properties.ShippingAddress.RegionCode}" + target: klaviyo.shipping_region_code + replaceMissing: true + - type: string-builder-processor + name: Define `klaviyo.billing_region_code` equal to + BillingAddress.CountryCode-BillingAddress.RegionCode + enabled: true + template: "%{attributes.event_properties.BillingAddress.CountryCode}-%{attribut\ + es.event_properties.BillingAddress.RegionCode}" + target: klaviyo.billing_region_code + replaceMissing: true + - type: attribute-remapper + name: "Remapper: Map `attributes.event_properties.ShippingAddress.Zip` to + `klaviyo.shipping_zip_code`" + enabled: true + sources: + - attributes.event_properties.ShippingAddress.Zip + sourceType: attribute + target: klaviyo.shipping_zip_code + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: "Remapper: Map `attributes.event_properties.BillingAddress.Zip` to + `klaviyo.billing_zip_code`" + enabled: true + sources: + - attributes.event_properties.BillingAddress.Zip + sourceType: attribute + target: klaviyo.billing_zip_code + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: string-builder-processor + name: Define `klaviyo.product_categories` equal to concatenated + `attributes.event_properties.Categories` + enabled: true + template: "%{attributes.event_properties.Categories}" + target: klaviyo.product_categories + replaceMissing: false + - type: string-builder-processor + name: Define `klaviyo.products` equal to concatenated `ProductID` - + `ProductName` + enabled: true + template: "%{attributes.event_properties.Items.ProductID} - + %{attributes.event_properties.Items.ProductName}" + target: klaviyo.products + replaceMissing: false + - type: string-builder-processor + name: Define `klaviyo.refund_reason` equal to + `attributes.event_properties.Reason` + enabled: true + template: "%{attributes.event_properties.Reason}" + target: klaviyo.refund_reason + replaceMissing: false + - type: pipeline + name: ecommerce events - Shopify Placed/Refunded Order + enabled: true + filter: + query: source:klaviyo service:ecommerce-events + @attributes.event_properties.$extra:* @metric_name:("Placed Order" OR + "Refunded Order") + processors: + - type: string-builder-processor + name: Define `klaviyo.shipping_region_code` equal to Shopify + shipping_address.country_code-shipping_address.province_code + enabled: true + template: "%{attributes.event_properties.$extra.shipping_address.country_code}-\ + %{attributes.event_properties.$extra.shipping_address.province_code\ + }" + target: klaviyo.shipping_region_code + replaceMissing: true + - type: string-builder-processor + name: Define `klaviyo.billing_region_code` equal to Shopify + billing_address.country_code-billing_address.province_code + enabled: true + template: "%{attributes.event_properties.$extra.billing_address.country_code}-%\ + {attributes.event_properties.$extra.billing_address.province_code}" + target: klaviyo.billing_region_code + replaceMissing: true + - type: attribute-remapper + name: "Remapper: Map `attributes.event_properties.$extra.billing_address.zip` to + `klaviyo.billing_zip_code`" + enabled: true + sources: + - attributes.event_properties.$extra.billing_address.zip + sourceType: attribute + target: klaviyo.billing_zip_code + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: attribute-remapper + name: "Remapper: Map `attributes.event_properties.$extra.shipping_address.zip` + to `klaviyo.shipping_zip_code`" + enabled: true + sources: + - attributes.event_properties.$extra.shipping_address.zip + sourceType: attribute + target: klaviyo.shipping_zip_code + targetType: attribute + preserveSource: true + overrideOnConflict: false + - type: string-builder-processor + name: Define `klaviyo.product_categories` equal to concatenated Shopify + `attributes.event_properties.$extra.line_items.product.product_type` + enabled: true + template: "%{attributes.event_properties.$extra.line_items.product.product_type\ + }" + target: klaviyo.product_categories + replaceMissing: false + - type: string-builder-processor + name: Define `klaviyo.products` equal to concatenated Shopify line_items + `product.id` - `product.title` + enabled: true + template: "%{attributes.event_properties.$extra.line_items.product.id} - + %{attributes.event_properties.$extra.line_items.product.title}" + target: klaviyo.products + replaceMissing: true + - type: string-builder-processor + name: Define `klaviyo.refund_reason` equal to concatenated Shopify refunds + `note` + enabled: true + template: "%{attributes.event_properties.$extra.refunds.note}" + target: klaviyo.refund_reason + replaceMissing: false + - type: pipeline + name: ecommerce events - common + enabled: true + filter: + query: source:klaviyo service:ecommerce-events + processors: + - type: attribute-remapper + name: "Remapper: Map `attributes.event_properties.page|URL` to `http.url`" + enabled: true + sources: + - attributes.event_properties.page + - attributes.event_properties.URL + sourceType: attribute + target: http.url + targetType: attribute + preserveSource: true + overrideOnConflict: false + - name: Lookup for `metric_name` to `status` field + enabled: true + source: metric_name + target: status + lookupTable: |- + Active on Site,info + Added to Cart,info + Cancelled Order,warning + Fulfilled Order,info + Ordered Product,info + Placed Order,info + Refunded Order,warning + Started Checkout,info + Checkout Started,info + Viewed Product,info + type: lookup-processor + - type: status-remapper + name: Define `status` as the official status of the log + enabled: true + sources: + - status diff --git a/klaviyo/assets/logs/klaviyo_tests.yaml b/klaviyo/assets/logs/klaviyo_tests.yaml new file mode 100644 index 0000000000000..d44d64c69c4d6 --- /dev/null +++ b/klaviyo/assets/logs/klaviyo_tests.yaml @@ -0,0 +1,1838 @@ +id: klaviyo +tests: + - sample: |- + { + "metric_name": "Received Email", + "profile_id": "01JXZDTRS5ZWET33T7930NVH2X", + "attributes": { + "datetime": "2025-07-10T20:41:13+00:00", + "uuid": "37d77a80-5dce-11f0-8001-1ce63ac70994", + "timestamp": 1752180073, + "event_properties": { + "_ip": "166.137.246.54", + "$event_id": "1752180073", + "Email Domain": "klaviyo-demo.com", + "Subject": "Internal Klaviyo - Subject Line", + "Campaign Name": "Internal Klaviyo - Test Campaign Name" + } + }, + "links": { + "self": "https://a.klaviyo.com/api/events/6jsMbujVKTe/" + }, + "id": "6jsMbujVKTe", + "type": "event" + } + result: + custom: + attributes: + datetime: "2025-07-10T20:41:13+00:00" + event_properties: + $event_id: "1752180073" + Campaign Name: "Internal Klaviyo - Test Campaign Name" + Email Domain: "klaviyo-demo.com" + Subject: "Internal Klaviyo - Subject Line" + _ip: "166.137.246.54" + timestamp: 1752180073 + uuid: "37d77a80-5dce-11f0-8001-1ce63ac70994" + id: "6jsMbujVKTe" + klaviyo: + campaign_name: "Internal Klaviyo - Test Campaign Name" + links: + self: "https://a.klaviyo.com/api/events/6jsMbujVKTe/" + metric_name: "Received Email" + network: + client: + geoip: { } + profile_id: "01JXZDTRS5ZWET33T7930NVH2X" + service: "marketing-events" + status: "info" + type: "event" + message: |- + { + "metric_name" : "Received Email", + "profile_id" : "01JXZDTRS5ZWET33T7930NVH2X", + "attributes" : { + "datetime" : "2025-07-10T20:41:13+00:00", + "uuid" : "37d77a80-5dce-11f0-8001-1ce63ac70994", + "timestamp" : 1752180073, + "event_properties" : { + "_ip" : "166.137.246.54", + "$event_id" : "1752180073", + "Email Domain" : "klaviyo-demo.com", + "Subject" : "Internal Klaviyo - Subject Line", + "Campaign Name" : "Internal Klaviyo - Test Campaign Name" + } + }, + "links" : { + "self" : "https://a.klaviyo.com/api/events/6jsMbujVKTe/" + }, + "id" : "6jsMbujVKTe", + "type" : "event" + } + service: "marketing-events" + status: "info" + tags: + - "source:LOGS_SOURCE" + timestamp: 1752180073000 + - sample: >- + { + "metric_name": "Reset Password", + "profile_id": "01JXFPJ6GZBSW06VGH5YC1BZ2R", + "attributes": { + "datetime": "2025-07-10T20:40:34+00:00", + "uuid": "20988d00-5dce-11f0-8001-abb877364eba", + "timestamp": 1752180034, + "event_properties": { + "$value": 0, + "$event_id": "1752180034", + "action": "Reset Password", + "PasswordResetLink": "https://www.website.com/reset/1234567890987654321" + } + }, + "links": { + "self": "https://a.klaviyo.com/api/events/6jsMdJ68urY/" + }, + "id": "6jsMdJ68urY", + "type": "event" + } + result: + custom: + attributes: + datetime: "2025-07-10T20:40:34+00:00" + event_properties: + $event_id: "1752180034" + $value: 0 + PasswordResetLink: "https://www.website.com/reset/1234567890987654321" + action: "Reset Password" + timestamp: 1752180034 + uuid: "20988d00-5dce-11f0-8001-abb877364eba" + id: "6jsMdJ68urY" + links: + self: "https://a.klaviyo.com/api/events/6jsMdJ68urY/" + metric_name: "Reset Password" + profile_id: "01JXFPJ6GZBSW06VGH5YC1BZ2R" + service: "other-events" + type: "event" + message: |- + { + "metric_name" : "Reset Password", + "profile_id" : "01JXFPJ6GZBSW06VGH5YC1BZ2R", + "attributes" : { + "datetime" : "2025-07-10T20:40:34+00:00", + "uuid" : "20988d00-5dce-11f0-8001-abb877364eba", + "timestamp" : 1752180034, + "event_properties" : { + "$value" : 0, + "$event_id" : "1752180034", + "action" : "Reset Password", + "PasswordResetLink" : "https://www.website.com/reset/1234567890987654321" + } + }, + "links" : { + "self" : "https://a.klaviyo.com/api/events/6jsMdJ68urY/" + }, + "id" : "6jsMdJ68urY", + "type" : "event" + } + service: "other-events" + tags: + - "source:LOGS_SOURCE" + timestamp: 1752180034000 + - sample: |- + { + "metric_name": "Placed Order", + "profile_id": "01JXZDTSG33JAJB9JRXRA0VGVV", + "attributes": { + "datetime": "2025-07-10T20:40:33+00:00", + "uuid": "1ffff680-5dce-11f0-8001-be92381409b3", + "timestamp": 1752180033, + "event_properties": { + "BillingAddress": { + "Zip": "02111", + "Company": "", + "RegionCode": "MA", + "Phone": "5005550006", + "FirstName": "Matt", + "Address2": "6th floor", + "Country": "USA", + "Region": "Massachusetts", + "LastName": "Kemp (Sample)", + "City": "Boston", + "Address1": "125 Summer St", + "CountryCode": "US" + }, + "Categories": [ + "Fiction", + "Classics", + "Children" + ], + "$value": 29.98, + "ShippingAddress": { + "Zip": "02111", + "Company": "", + "RegionCode": "MA", + "Phone": "5005550006", + "FirstName": "Matt", + "Address2": "6th floor", + "Country": "USA", + "Region": "Massachusetts", + "LastName": "Kemp (Sample)", + "City": "Boston", + "Address1": "125 Summer St", + "CountryCode": "US" + }, + "$event_id": "1752180033", + "Brands": [ + "Kids Books", + "Harcourt Classics" + ], + "Items": [ + { + "Brand": "Kids Books", + "RowTotal": 9.99, + "ProductName": "Winnie the Pooh", + "Categories": [ + "Fiction", + "Children" + ], + "ItemPrice": 9.99, + "Quantity": 1, + "ImageURL": "http://www.example.com/path/to/product/image.png", + "ProductID": "1111", + "SKU": "WINNIEPOOH", + "ProductURL": "http://www.example.com/path/to/product" + }, + { + "Brand": "Harcourt Classics", + "RowTotal": 19.99, + "ProductName": "A Tale of Two Cities", + "Categories": [ + "Fiction", + "Classics" + ], + "ItemPrice": 19.99, + "Quantity": 1, + "ImageURL": "http://www.example.com/path/to/product/image2.png", + "ProductID": "1112", + "SKU": "TALEOFTWO", + "ProductURL": "http://www.example.com/path/to/product2" + } + ], + "DiscountCode": "Free Shipping", + "DiscountValue": 5, + "OrderId": "1234", + "ItemNames": [ + "Winnie the Pooh", + "A Tale of Two Cities" + ] + } + }, + "links": { + "self": "https://a.klaviyo.com/api/events/6jsMbvMP9e5/" + }, + "id": "6jsMbvMP9e5", + "type": "event", + "status": "info" + } + result: + custom: + attributes: + datetime: "2025-07-10T20:40:33+00:00" + event_properties: + $event_id: "1752180033" + $value: 29.98 + BillingAddress: + Address1: "125 Summer St" + Address2: "6th floor" + City: "Boston" + Company: "" + Country: "USA" + CountryCode: "US" + FirstName: "Matt" + LastName: "Kemp (Sample)" + Phone: "5005550006" + Region: "Massachusetts" + RegionCode: "MA" + Zip: "02111" + Brands: + - "Kids Books" + - "Harcourt Classics" + Categories: + - "Fiction" + - "Classics" + - "Children" + DiscountCode: "Free Shipping" + DiscountValue: 5 + ItemNames: + - "Winnie the Pooh" + - "A Tale of Two Cities" + Items: + - Brand: "Kids Books" + RowTotal: 9.99 + ProductName: "Winnie the Pooh" + Categories: + - "Fiction" + - "Children" + ItemPrice: 9.99 + Quantity: 1 + ImageURL: "http://www.example.com/path/to/product/image.png" + ProductID: "1111" + SKU: "WINNIEPOOH" + ProductURL: "http://www.example.com/path/to/product" + - Brand: "Harcourt Classics" + RowTotal: 19.99 + ProductName: "A Tale of Two Cities" + Categories: + - "Fiction" + - "Classics" + ItemPrice: 19.99 + Quantity: 1 + ImageURL: "http://www.example.com/path/to/product/image2.png" + ProductID: "1112" + SKU: "TALEOFTWO" + ProductURL: "http://www.example.com/path/to/product2" + OrderId: "1234" + ShippingAddress: + Address1: "125 Summer St" + Address2: "6th floor" + City: "Boston" + Company: "" + Country: "USA" + CountryCode: "US" + FirstName: "Matt" + LastName: "Kemp (Sample)" + Phone: "5005550006" + Region: "Massachusetts" + RegionCode: "MA" + Zip: "02111" + timestamp: 1752180033 + uuid: "1ffff680-5dce-11f0-8001-be92381409b3" + id: "6jsMbvMP9e5" + klaviyo: + billing_region_code: "US-MA" + billing_zip_code: "02111" + product_categories: "Fiction,Classics,Children" + products: "1111,1112 - Winnie the Pooh,A Tale of Two Cities" + shipping_region_code: "US-MA" + shipping_zip_code: "02111" + links: + self: "https://a.klaviyo.com/api/events/6jsMbvMP9e5/" + metric_name: "Placed Order" + profile_id: "01JXZDTSG33JAJB9JRXRA0VGVV" + service: "ecommerce-events" + status: "info" + type: "event" + message: |- + { + "metric_name" : "Placed Order", + "profile_id" : "01JXZDTSG33JAJB9JRXRA0VGVV", + "attributes" : { + "datetime" : "2025-07-10T20:40:33+00:00", + "uuid" : "1ffff680-5dce-11f0-8001-be92381409b3", + "timestamp" : 1752180033, + "event_properties" : { + "BillingAddress" : { + "Zip" : "02111", + "Company" : "", + "RegionCode" : "MA", + "Phone" : "5005550006", + "FirstName" : "Matt", + "Address2" : "6th floor", + "Country" : "USA", + "Region" : "Massachusetts", + "LastName" : "Kemp (Sample)", + "City" : "Boston", + "Address1" : "125 Summer St", + "CountryCode" : "US" + }, + "Categories" : [ "Fiction", "Classics", "Children" ], + "$value" : 29.98, + "ShippingAddress" : { + "Zip" : "02111", + "Company" : "", + "RegionCode" : "MA", + "Phone" : "5005550006", + "FirstName" : "Matt", + "Address2" : "6th floor", + "Country" : "USA", + "Region" : "Massachusetts", + "LastName" : "Kemp (Sample)", + "City" : "Boston", + "Address1" : "125 Summer St", + "CountryCode" : "US" + }, + "$event_id" : "1752180033", + "Brands" : [ "Kids Books", "Harcourt Classics" ], + "Items" : [ { + "Brand" : "Kids Books", + "RowTotal" : 9.99, + "ProductName" : "Winnie the Pooh", + "Categories" : [ "Fiction", "Children" ], + "ItemPrice" : 9.99, + "Quantity" : 1, + "ImageURL" : "http://www.example.com/path/to/product/image.png", + "ProductID" : "1111", + "SKU" : "WINNIEPOOH", + "ProductURL" : "http://www.example.com/path/to/product" + }, { + "Brand" : "Harcourt Classics", + "RowTotal" : 19.99, + "ProductName" : "A Tale of Two Cities", + "Categories" : [ "Fiction", "Classics" ], + "ItemPrice" : 19.99, + "Quantity" : 1, + "ImageURL" : "http://www.example.com/path/to/product/image2.png", + "ProductID" : "1112", + "SKU" : "TALEOFTWO", + "ProductURL" : "http://www.example.com/path/to/product2" + } ], + "DiscountCode" : "Free Shipping", + "DiscountValue" : 5, + "OrderId" : "1234", + "ItemNames" : [ "Winnie the Pooh", "A Tale of Two Cities" ] + } + }, + "links" : { + "self" : "https://a.klaviyo.com/api/events/6jsMbvMP9e5/" + }, + "id" : "6jsMbvMP9e5", + "type" : "event", + "status" : "info" + } + service: "ecommerce-events" + status: "info" + tags: + - "source:LOGS_SOURCE" + timestamp: 1752180033000 + - sample: |- + { + "metric_name": "Opened Email", + "profile_id": "01K1WZ55QAJNB5PPNSG59X6PJ0", + "flow_name": "Customer Thank You - New vs. Returning", + "attributes": { + "datetime": "2025-08-06T13:39:56+00:00", + "uuid": "d6ba9e00-72ca-11f0-8001-62dc5e6734c0", + "timestamp": 1754487596, + "event_properties": { + "Inbox Provider": "Gsuite", + "$_cohort$message_send_cohort": "1754485652:XDj2Nt", + "Subject": "You're what makes us great", + "Email Domain": "mayalane.com", + "$message_interaction": "XDj2Nt", + "$ESP": 2, + "machine_open": true, + "$message": "XDj2Nt", + "$event_id": "XDj2Nt:01K1ZQWXCHYYF6HGNHYK9CC9KT:1754487596", + "$flow": "U4tfMv", + "Recipient Email Address": "brian+shop8@mayalane.com", + "$internal": { + "Handoff Time": 1754485652, + "Transmission ID": "01K1ZQWXCHYYF6HGNHYK9CC9KT", + "User Agent": "Mozilla/5.0", + "Friendly From Domain": "barefootcoders.com" + }, + "Campaign Name": "New Customer Thank You: Email #1" + } + }, + "links": { + "self": "https://a.klaviyo.com/api/events/6pmr8Cqxkap/" + }, + "id": "6pmr8Cqxkap", + "type": "event" + } + result: + custom: + attributes: + datetime: "2025-08-06T13:39:56+00:00" + event_properties: + $ESP: 2 + $_cohort$message_send_cohort: "1754485652:XDj2Nt" + $event_id: "XDj2Nt:01K1ZQWXCHYYF6HGNHYK9CC9KT:1754487596" + $flow: "U4tfMv" + $internal: + Friendly From Domain: "barefootcoders.com" + Handoff Time: 1754485652 + Transmission ID: "01K1ZQWXCHYYF6HGNHYK9CC9KT" + User Agent: "Mozilla/5.0" + $message: "XDj2Nt" + $message_interaction: "XDj2Nt" + Campaign Name: "New Customer Thank You: Email #1" + Email Domain: "mayalane.com" + Inbox Provider: "Gsuite" + Recipient Email Address: "brian+shop8@mayalane.com" + Subject: "You're what makes us great" + machine_open: true + timestamp: 1754487596 + uuid: "d6ba9e00-72ca-11f0-8001-62dc5e6734c0" + flow_name: "Customer Thank You - New vs. Returning" + id: "6pmr8Cqxkap" + klaviyo: + campaign_name: "New Customer Thank You: Email #1" + flow_name: "Customer Thank You - New vs. Returning" + links: + self: "https://a.klaviyo.com/api/events/6pmr8Cqxkap/" + metric_name: "Opened Email" + profile_id: "01K1WZ55QAJNB5PPNSG59X6PJ0" + service: "marketing-events" + status: "info" + type: "event" + message: |- + { + "metric_name" : "Opened Email", + "profile_id" : "01K1WZ55QAJNB5PPNSG59X6PJ0", + "flow_name" : "Customer Thank You - New vs. Returning", + "attributes" : { + "datetime" : "2025-08-06T13:39:56+00:00", + "uuid" : "d6ba9e00-72ca-11f0-8001-62dc5e6734c0", + "timestamp" : 1754487596, + "event_properties" : { + "Inbox Provider" : "Gsuite", + "$_cohort$message_send_cohort" : "1754485652:XDj2Nt", + "Subject" : "You're what makes us great", + "Email Domain" : "mayalane.com", + "$message_interaction" : "XDj2Nt", + "$ESP" : 2, + "machine_open" : true, + "$message" : "XDj2Nt", + "$event_id" : "XDj2Nt:01K1ZQWXCHYYF6HGNHYK9CC9KT:1754487596", + "$flow" : "U4tfMv", + "Recipient Email Address" : "brian+shop8@mayalane.com", + "$internal" : { + "Handoff Time" : 1754485652, + "Transmission ID" : "01K1ZQWXCHYYF6HGNHYK9CC9KT", + "User Agent" : "Mozilla/5.0", + "Friendly From Domain" : "barefootcoders.com" + }, + "Campaign Name" : "New Customer Thank You: Email #1" + } + }, + "links" : { + "self" : "https://a.klaviyo.com/api/events/6pmr8Cqxkap/" + }, + "id" : "6pmr8Cqxkap", + "type" : "event" + } + service: "marketing-events" + status: "info" + tags: + - "source:LOGS_SOURCE" + timestamp: 1754487596000 + - sample: |- + { + "metric_name": "Ordered Product", + "profile_id": "01K1XQCC9V79XP3CNKNMRBT6GT", + "attributes": { + "datetime": "2025-08-05T18:20:20+00:00", + "uuid": "d8338a00-7228-11f0-8001-d86a6b406da7", + "timestamp": 1754418020, + "event_properties": { + "Variant ID": 46305476280420, + "Variant Option: Title": "Default Title", + "Customer Locale": "en-US", + "Quantity": 1, + "ProductID": 8128980484196, + "Vendor": "My Store", + "Name": "Magic Carpet", + "Variant Name": "Default Title", + "$value": 2100, + "$event_id": "5777917050980:13904611934308:0", + "$extra": { + "translations": { + "presentment_title": "", + "presentment_variant_title": "" + }, + "price_set": { + "shop_money": { + "amount": 2100, + "currency": "USD" + }, + "presentment_money": { + "amount": 2100, + "currency": "USD" + } + } + }, + "SKU": "123SKU", + "Collections": [ + "Abstract Art" + ], + "Tags": [ + "shopify-tagged" + ] + } + }, + "links": { + "self": "https://a.klaviyo.com/api/events/6peNeyigRiY/" + }, + "id": "6peNeyigRiY", + "type": "event" + } + result: + custom: + attributes: + datetime: "2025-08-05T18:20:20+00:00" + event_properties: + $event_id: "5777917050980:13904611934308:0" + $extra: + price_set: + presentment_money: + amount: 2100 + currency: "USD" + shop_money: + amount: 2100 + currency: "USD" + translations: + presentment_title: "" + presentment_variant_title: "" + $value: 2100 + Collections: + - "Abstract Art" + Customer Locale: "en-US" + Name: "Magic Carpet" + ProductID: 8128980484196 + Quantity: 1 + SKU: "123SKU" + Tags: + - "shopify-tagged" + Variant ID: 46305476280420 + Variant Name: "Default Title" + 'Variant Option: Title': "Default Title" + Vendor: "My Store" + timestamp: 1754418020 + uuid: "d8338a00-7228-11f0-8001-d86a6b406da7" + id: "6peNeyigRiY" + klaviyo: + products: "8128980484196 - Magic Carpet" + links: + self: "https://a.klaviyo.com/api/events/6peNeyigRiY/" + metric_name: "Ordered Product" + profile_id: "01K1XQCC9V79XP3CNKNMRBT6GT" + service: "ecommerce-events" + status: "info" + type: "event" + message: |- + { + "metric_name" : "Ordered Product", + "profile_id" : "01K1XQCC9V79XP3CNKNMRBT6GT", + "attributes" : { + "datetime" : "2025-08-05T18:20:20+00:00", + "uuid" : "d8338a00-7228-11f0-8001-d86a6b406da7", + "timestamp" : 1754418020, + "event_properties" : { + "Variant ID" : 46305476280420, + "Variant Option: Title" : "Default Title", + "Customer Locale" : "en-US", + "Quantity" : 1, + "ProductID" : 8128980484196, + "Vendor" : "My Store", + "Name" : "Magic Carpet", + "Variant Name" : "Default Title", + "$value" : 2100, + "$event_id" : "5777917050980:13904611934308:0", + "$extra" : { + "translations" : { + "presentment_title" : "", + "presentment_variant_title" : "" + }, + "price_set" : { + "shop_money" : { + "amount" : 2100, + "currency" : "USD" + }, + "presentment_money" : { + "amount" : 2100, + "currency" : "USD" + } + } + }, + "SKU" : "123SKU", + "Collections" : [ "Abstract Art" ], + "Tags" : [ "shopify-tagged" ] + } + }, + "links" : { + "self" : "https://a.klaviyo.com/api/events/6peNeyigRiY/" + }, + "id" : "6peNeyigRiY", + "type" : "event" + } + service: "ecommerce-events" + status: "info" + tags: + - "source:LOGS_SOURCE" + timestamp: 1754418020000 + - sample: >- + { + "metric_name": "Placed Order", + "profile_id": "01K1XQCC9V79XP3CNKNMRBT6GT", + "attributes": { + "datetime": "2025-08-05T18:20:10+00:00", + "uuid": "d23da900-7228-11f0-8001-32bc1719f29c", + "timestamp": 1754418010, + "event_properties": { + "$currency_code": "USD", + "Item Count": 1, + "Source Name": "web", + "ShippingRate": "Economy", + "OptedInToSmsOrderUpdates": false, + "$value": 6300, + "Customer Locale": "en-US", + "$event_id": "5777917050980", + "Items": [ + "Magic Carpet" + ], + "$extra": { + "confirmation_number": "ARWRYXR0Z", + "total_cash_rounding_refund_adjustment_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "current_total_discounts_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "billing_address": { + "zip": "19975", + "country_code": "US", + "country": "United States", + "province": "Delaware", + "city": "Selbyville", + "address1": "31221 Americana Parkway", + "latitude": 38.4694439, + "name": "Larry Barry", + "last_name": "Barry", + "province_code": "DE", + "first_name": "Larry", + "longitude": -75.1137177 + }, + "line_items": [ + { + "total_discount": "0.00", + "gift_card": false, + "requires_shipping": true, + "total_discount_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "title": "Magic Carpet", + "product_exists": true, + "line_price_set": { + "shop_money": { + "amount": 6300, + "currency_code": "USD" + }, + "presentment_money": { + "amount": 6300, + "currency_code": "USD" + } + }, + "variant_id": 46305476280420, + "vendor": "My Store", + "translations": { + "presentment_title": "", + "presentment_variant_title": "" + }, + "price": 2100, + "product_id": 8128980484196, + "id": 13904611934308, + "grams": 0, + "sku": "123SKU", + "line_price": 6300, + "product": { + "body_html": "

3 x 4

", + "images": [ + { + "src": "https://cdn.shopify.com/s/files/1/0726/5288/2020/files/magiccarpet2025.png?v=1754047681", + "width": 1954, + "thumb_src": "https://cdn.shopify.com/s/files/1/0726/5288/2020/files/magiccarpet2025_x240.png?v=1754047681", + "height": 2649 + } + ], + "product_type": "Painting", + "variant_options": { + "Title": "Default Title" + }, + "vendor": "My Store", + "variant": { + "translations": { + "presentment_variant_title": "" + }, + "options": { + "Title": "Default Title" + }, + "id": 46305476280420, + "sku": "123SKU", + "title": "Default Title" + }, + "handle": "magic-carpet", + "id": 8128980484196, + "title": "Magic Carpet", + "tags": "shopify-tagged" + }, + "fulfillable_quantity": 3, + "quantity": 3, + "taxable": true, + "fulfillment_service": "manual", + "variant_inventory_management": "shopify", + "current_quantity": 3, + "admin_graphql_api_id": "gid://shopify/LineItem/13904611934308", + "name": "Magic Carpet", + "price_set": { + "shop_money": { + "amount": "2100.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "2100.00", + "currency_code": "USD" + } + } + } + ], + "total_discounts_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "presentment_currency": "USD", + "total_cash_rounding_payment_adjustment_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "landing_site": "/", + "merchant_business_entity_id": "37712494692", + "number": 11, + "checkout_id": 37131175133284, + "checkout_token": "196295873699a958546bd9284ce9e409", + "current_total_discounts": "0.00", + "customer_locale": "en-US", + "id": 5777917050980, + "app_id": 580111, + "subtotal_price": "6300.00", + "order_status_url": "https://qx0wtr-1y.myshopify.com/72652882020/orders/2e5c63a1dad1fd79265d096dfaa57915/authenticate?key=763c84f48cc52a64d8f08de84e85c4e9", + "current_total_price_set": { + "shop_money": { + "amount": "6300.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "6300.00", + "currency_code": "USD" + } + }, + "total_shipping_price_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "test": true, + "subtotal_price_set": { + "shop_money": { + "amount": "6300.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "6300.00", + "currency_code": "USD" + } + }, + "tax_exempt": false, + "payment_gateway_names": [ + "bogus" + ], + "total_tax": "0.00", + "tags": "", + "current_subtotal_price_set": { + "shop_money": { + "amount": "6300.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "6300.00", + "currency_code": "USD" + } + }, + "current_total_tax": "0.00", + "shipping_lines": [ + { + "code": "Economy", + "price": "0.00", + "is_removed": false, + "id": 4901692112996, + "source": "shopify", + "current_discounted_price_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "price_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "title": "Economy", + "discounted_price_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "discounted_price": "0.00" + } + ], + "name": "#1011", + "cart_token": "hWN1SQkj4QbLb1NKF6pwXgwM", + "total_tax_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "estimated_taxes": false, + "current_total_tax_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "current_subtotal_price": "6300.00", + "total_outstanding": "0.00", + "order_number": 1011, + "created_at": "2025-08-05T14:19:52-04:00", + "total_line_items_price_set": { + "shop_money": { + "amount": "6300.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "6300.00", + "currency_code": "USD" + } + }, + "taxes_included": false, + "buyer_accepts_marketing": true, + "confirmed": true, + "total_weight": 0, + "contact_email": "brian+shop12@mayalane.com", + "total_discounts": "0.00", + "client_details": { + "accept_language": "en-US", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15", + "browser_ip": "2601:140:9700:2530:242b:7ae2:7ed:d2a1" + }, + "updated_at": "2025-08-05T14:19:54-04:00", + "referring_site": "", + "processed_at": "2025-08-05T14:19:50-04:00", + "currency": "USD", + "shipping_address": { + "zip": "19975", + "country_code": "US", + "country": "United States", + "province": "Delaware", + "city": "Selbyville", + "address1": "31221 Americana Parkway", + "latitude": 38.4694439, + "name": "Larry Barry", + "last_name": "Barry", + "province_code": "DE", + "first_name": "Larry", + "longitude": -75.1137177 + }, + "source_name": "web", + "browser_ip": "2601:140:9700:2530:242b:7ae2:7ed:d2a1", + "current_shipping_price_set": { + "shop_money": { + "amount": "0.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "0.00", + "currency_code": "USD" + } + }, + "email": "brian+shop12@mayalane.com", + "total_price_set": { + "shop_money": { + "amount": "6300.00", + "currency_code": "USD" + }, + "presentment_money": { + "amount": "6300.00", + "currency_code": "USD" + } + }, + "total_price": "6300.00", + "full_landing_site": "http://qx0wtr-1y.myshopify.com/", + "total_line_items_price": "6300.00", + "duties_included": false, + "total_tip_received": "0.00", + "sms_order_updates_details": { + "created_at": "2025-08-05T14:19:52-04:00", + "order_id": 5777917050980, + "email": "brian+shop12@mayalane.com" + }, + "token": "2e5c63a1dad1fd79265d096dfaa57915", + "current_total_price": "6300.00", + "webhook_id": "dc48c824-ebb6-444f-b42d-281dde4e7851", + "admin_graphql_api_id": "gid://shopify/Order/5777917050980", + "financial_status": "paid", + "customer": { + "tax_exempt": false, + "email_marketing_consent": { + "consent_updated_at": "2025-08-05T14:19:53-04:00", + "state": "subscribed", + "opt_in_level": "single_opt_in" + }, + "created_at": "2025-08-05T14:19:51-04:00", + "last_name": "Barry", + "verified_email": true, + "tags": "", + "default_address": { + "zip": "19975", + "country": "United States", + "city": "Selbyville", + "address1": "31221 Americana Parkway", + "last_name": "Barry", + "province_code": "DE", + "country_code": "US", + "default": true, + "province": "Delaware", + "country_name": "United States", + "name": "Larry Barry", + "id": 9025902346340, + "customer_id": 7824081551460, + "first_name": "Larry" + }, + "updated_at": "2025-08-05T14:19:53-04:00", + "admin_graphql_api_id": "gid://shopify/Customer/7824081551460", + "currency": "USD", + "id": 7824081551460, + "state": "disabled", + "first_name": "Larry", + "email": "brian+shop12@mayalane.com" + } + }, + "Collections": [ + "Abstract Art" + ], + "Total Discounts": "0.00" + } + }, + "links": { + "self": "https://a.klaviyo.com/api/events/6peNgN4t4ss/" + }, + "id": "6peNgN4t4ss", + "type": "event" + } + result: + custom: + attributes: + datetime: "2025-08-05T18:20:10+00:00" + event_properties: + $currency_code: "USD" + $event_id: "5777917050980" + $extra: + admin_graphql_api_id: "gid://shopify/Order/5777917050980" + app_id: 580111 + billing_address: + address1: "31221 Americana Parkway" + city: "Selbyville" + country: "United States" + country_code: "US" + first_name: "Larry" + last_name: "Barry" + latitude: 38.4694439 + longitude: -75.1137177 + name: "Larry Barry" + province: "Delaware" + province_code: "DE" + zip: "19975" + browser_ip: "2601:140:9700:2530:242b:7ae2:7ed:d2a1" + buyer_accepts_marketing: true + cart_token: "hWN1SQkj4QbLb1NKF6pwXgwM" + checkout_id: 37131175133284 + checkout_token: "196295873699a958546bd9284ce9e409" + client_details: + accept_language: "en-US" + browser_ip: "2601:140:9700:2530:242b:7ae2:7ed:d2a1" + user_agent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15" + confirmation_number: "ARWRYXR0Z" + confirmed: true + contact_email: "brian+shop12@mayalane.com" + created_at: "2025-08-05T14:19:52-04:00" + currency: "USD" + current_shipping_price_set: + presentment_money: + amount: "0.00" + currency_code: "USD" + shop_money: + amount: "0.00" + currency_code: "USD" + current_subtotal_price: "6300.00" + current_subtotal_price_set: + presentment_money: + amount: "6300.00" + currency_code: "USD" + shop_money: + amount: "6300.00" + currency_code: "USD" + current_total_discounts: "0.00" + current_total_discounts_set: + presentment_money: + amount: "0.00" + currency_code: "USD" + shop_money: + amount: "0.00" + currency_code: "USD" + current_total_price: "6300.00" + current_total_price_set: + presentment_money: + amount: "6300.00" + currency_code: "USD" + shop_money: + amount: "6300.00" + currency_code: "USD" + current_total_tax: "0.00" + current_total_tax_set: + presentment_money: + amount: "0.00" + currency_code: "USD" + shop_money: + amount: "0.00" + currency_code: "USD" + customer: + admin_graphql_api_id: "gid://shopify/Customer/7824081551460" + created_at: "2025-08-05T14:19:51-04:00" + currency: "USD" + default_address: + address1: "31221 Americana Parkway" + city: "Selbyville" + country: "United States" + country_code: "US" + country_name: "United States" + customer_id: 7824081551460 + default: true + first_name: "Larry" + id: 9025902346340 + last_name: "Barry" + name: "Larry Barry" + province: "Delaware" + province_code: "DE" + zip: "19975" + email: "brian+shop12@mayalane.com" + email_marketing_consent: + consent_updated_at: "2025-08-05T14:19:53-04:00" + opt_in_level: "single_opt_in" + state: "subscribed" + first_name: "Larry" + id: 7824081551460 + last_name: "Barry" + state: "disabled" + tags: "" + tax_exempt: false + updated_at: "2025-08-05T14:19:53-04:00" + verified_email: true + customer_locale: "en-US" + duties_included: false + email: "brian+shop12@mayalane.com" + estimated_taxes: false + financial_status: "paid" + full_landing_site: "http://qx0wtr-1y.myshopify.com/" + id: 5777917050980 + landing_site: "/" + line_items: + - + total_discount: "0.00" + gift_card: false + requires_shipping: true + total_discount_set: + shop_money: + amount: "0.00" + currency_code: "USD" + presentment_money: + amount: "0.00" + currency_code: "USD" + title: "Magic Carpet" + product_exists: true + line_price_set: + shop_money: + amount: 6300 + currency_code: "USD" + presentment_money: + amount: 6300 + currency_code: "USD" + variant_id: 46305476280420 + vendor: "My Store" + translations: + presentment_title: "" + presentment_variant_title: "" + price: 2100 + product_id: 8128980484196 + id: 13904611934308 + grams: 0 + sku: "123SKU" + line_price: 6300 + product: + body_html: "

3 x 4

" + images: + - + src: "https://cdn.shopify.com/s/files/1/0726/5288/2020/files/magiccarpet2025.png?v=1754047681" + width: 1954 + thumb_src: "https://cdn.shopify.com/s/files/1/0726/5288/2020/files/magiccarpet2025_x240.png?v=1754047681" + height: 2649 + product_type: "Painting" + variant_options: + Title: "Default Title" + vendor: "My Store" + variant: + translations: + presentment_variant_title: "" + options: + Title: "Default Title" + id: 46305476280420 + sku: "123SKU" + title: "Default Title" + handle: "magic-carpet" + id: 8128980484196 + title: "Magic Carpet" + tags: "shopify-tagged" + fulfillable_quantity: 3 + quantity: 3 + taxable: true + fulfillment_service: "manual" + variant_inventory_management: "shopify" + current_quantity: 3 + admin_graphql_api_id: "gid://shopify/LineItem/13904611934308" + name: "Magic Carpet" + price_set: + shop_money: + amount: "2100.00" + currency_code: "USD" + presentment_money: + amount: "2100.00" + currency_code: "USD" + merchant_business_entity_id: "37712494692" + name: "#1011" + number: 11 + order_number: 1011 + order_status_url: "https://qx0wtr-1y.myshopify.com/72652882020/orders/2e5c63a1dad1fd79265d096dfaa57915/authenticate?key=763c84f48cc52a64d8f08de84e85c4e9" + payment_gateway_names: + - "bogus" + presentment_currency: "USD" + processed_at: "2025-08-05T14:19:50-04:00" + referring_site: "" + shipping_address: + address1: "31221 Americana Parkway" + city: "Selbyville" + country: "United States" + country_code: "US" + first_name: "Larry" + last_name: "Barry" + latitude: 38.4694439 + longitude: -75.1137177 + name: "Larry Barry" + province: "Delaware" + province_code: "DE" + zip: "19975" + shipping_lines: + - + code: "Economy" + price: "0.00" + is_removed: false + id: 4901692112996 + source: "shopify" + current_discounted_price_set: + shop_money: + amount: "0.00" + currency_code: "USD" + presentment_money: + amount: "0.00" + currency_code: "USD" + price_set: + shop_money: + amount: "0.00" + currency_code: "USD" + presentment_money: + amount: "0.00" + currency_code: "USD" + title: "Economy" + discounted_price_set: + shop_money: + amount: "0.00" + currency_code: "USD" + presentment_money: + amount: "0.00" + currency_code: "USD" + discounted_price: "0.00" + sms_order_updates_details: + created_at: "2025-08-05T14:19:52-04:00" + email: "brian+shop12@mayalane.com" + order_id: 5777917050980 + source_name: "web" + subtotal_price: "6300.00" + subtotal_price_set: + presentment_money: + amount: "6300.00" + currency_code: "USD" + shop_money: + amount: "6300.00" + currency_code: "USD" + tags: "" + tax_exempt: false + taxes_included: false + test: true + token: "2e5c63a1dad1fd79265d096dfaa57915" + total_cash_rounding_payment_adjustment_set: + presentment_money: + amount: "0.00" + currency_code: "USD" + shop_money: + amount: "0.00" + currency_code: "USD" + total_cash_rounding_refund_adjustment_set: + presentment_money: + amount: "0.00" + currency_code: "USD" + shop_money: + amount: "0.00" + currency_code: "USD" + total_discounts: "0.00" + total_discounts_set: + presentment_money: + amount: "0.00" + currency_code: "USD" + shop_money: + amount: "0.00" + currency_code: "USD" + total_line_items_price: "6300.00" + total_line_items_price_set: + presentment_money: + amount: "6300.00" + currency_code: "USD" + shop_money: + amount: "6300.00" + currency_code: "USD" + total_outstanding: "0.00" + total_price: "6300.00" + total_price_set: + presentment_money: + amount: "6300.00" + currency_code: "USD" + shop_money: + amount: "6300.00" + currency_code: "USD" + total_shipping_price_set: + presentment_money: + amount: "0.00" + currency_code: "USD" + shop_money: + amount: "0.00" + currency_code: "USD" + total_tax: "0.00" + total_tax_set: + presentment_money: + amount: "0.00" + currency_code: "USD" + shop_money: + amount: "0.00" + currency_code: "USD" + total_tip_received: "0.00" + total_weight: 0 + updated_at: "2025-08-05T14:19:54-04:00" + webhook_id: "dc48c824-ebb6-444f-b42d-281dde4e7851" + $value: 6300 + Collections: + - "Abstract Art" + Customer Locale: "en-US" + Item Count: 1 + Items: + - "Magic Carpet" + OptedInToSmsOrderUpdates: false + ShippingRate: "Economy" + Source Name: "web" + Total Discounts: "0.00" + timestamp: 1754418010 + uuid: "d23da900-7228-11f0-8001-32bc1719f29c" + id: "6peNgN4t4ss" + klaviyo: + billing_region_code: "US-DE" + billing_zip_code: "19975" + product_categories: "Painting" + products: "8128980484196 - Magic Carpet" + shipping_region_code: "US-DE" + shipping_zip_code: "19975" + links: + self: "https://a.klaviyo.com/api/events/6peNgN4t4ss/" + metric_name: "Placed Order" + profile_id: "01K1XQCC9V79XP3CNKNMRBT6GT" + service: "ecommerce-events" + status: "info" + type: "event" + message: |- + { + "metric_name" : "Placed Order", + "profile_id" : "01K1XQCC9V79XP3CNKNMRBT6GT", + "attributes" : { + "datetime" : "2025-08-05T18:20:10+00:00", + "uuid" : "d23da900-7228-11f0-8001-32bc1719f29c", + "timestamp" : 1754418010, + "event_properties" : { + "$currency_code" : "USD", + "Item Count" : 1, + "Source Name" : "web", + "ShippingRate" : "Economy", + "OptedInToSmsOrderUpdates" : false, + "$value" : 6300, + "Customer Locale" : "en-US", + "$event_id" : "5777917050980", + "Items" : [ "Magic Carpet" ], + "$extra" : { + "confirmation_number" : "ARWRYXR0Z", + "total_cash_rounding_refund_adjustment_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "current_total_discounts_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "billing_address" : { + "zip" : "19975", + "country_code" : "US", + "country" : "United States", + "province" : "Delaware", + "city" : "Selbyville", + "address1" : "31221 Americana Parkway", + "latitude" : 38.4694439, + "name" : "Larry Barry", + "last_name" : "Barry", + "province_code" : "DE", + "first_name" : "Larry", + "longitude" : -75.1137177 + }, + "line_items" : [ { + "total_discount" : "0.00", + "gift_card" : false, + "requires_shipping" : true, + "total_discount_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "title" : "Magic Carpet", + "product_exists" : true, + "line_price_set" : { + "shop_money" : { + "amount" : 6300, + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : 6300, + "currency_code" : "USD" + } + }, + "variant_id" : 46305476280420, + "vendor" : "My Store", + "translations" : { + "presentment_title" : "", + "presentment_variant_title" : "" + }, + "price" : 2100, + "product_id" : 8128980484196, + "id" : 13904611934308, + "grams" : 0, + "sku" : "123SKU", + "line_price" : 6300, + "product" : { + "body_html" : "

3 x 4

", + "images" : [ { + "src" : "https://cdn.shopify.com/s/files/1/0726/5288/2020/files/magiccarpet2025.png?v=1754047681", + "width" : 1954, + "thumb_src" : "https://cdn.shopify.com/s/files/1/0726/5288/2020/files/magiccarpet2025_x240.png?v=1754047681", + "height" : 2649 + } ], + "product_type" : "Painting", + "variant_options" : { + "Title" : "Default Title" + }, + "vendor" : "My Store", + "variant" : { + "translations" : { + "presentment_variant_title" : "" + }, + "options" : { + "Title" : "Default Title" + }, + "id" : 46305476280420, + "sku" : "123SKU", + "title" : "Default Title" + }, + "handle" : "magic-carpet", + "id" : 8128980484196, + "title" : "Magic Carpet", + "tags" : "shopify-tagged" + }, + "fulfillable_quantity" : 3, + "quantity" : 3, + "taxable" : true, + "fulfillment_service" : "manual", + "variant_inventory_management" : "shopify", + "current_quantity" : 3, + "admin_graphql_api_id" : "gid://shopify/LineItem/13904611934308", + "name" : "Magic Carpet", + "price_set" : { + "shop_money" : { + "amount" : "2100.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "2100.00", + "currency_code" : "USD" + } + } + } ], + "total_discounts_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "presentment_currency" : "USD", + "total_cash_rounding_payment_adjustment_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "landing_site" : "/", + "merchant_business_entity_id" : "37712494692", + "number" : 11, + "checkout_id" : 37131175133284, + "checkout_token" : "196295873699a958546bd9284ce9e409", + "current_total_discounts" : "0.00", + "customer_locale" : "en-US", + "id" : 5777917050980, + "app_id" : 580111, + "subtotal_price" : "6300.00", + "order_status_url" : "https://qx0wtr-1y.myshopify.com/72652882020/orders/2e5c63a1dad1fd79265d096dfaa57915/authenticate?key=763c84f48cc52a64d8f08de84e85c4e9", + "current_total_price_set" : { + "shop_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + } + }, + "total_shipping_price_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "test" : true, + "subtotal_price_set" : { + "shop_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + } + }, + "tax_exempt" : false, + "payment_gateway_names" : [ "bogus" ], + "total_tax" : "0.00", + "tags" : "", + "current_subtotal_price_set" : { + "shop_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + } + }, + "current_total_tax" : "0.00", + "shipping_lines" : [ { + "code" : "Economy", + "price" : "0.00", + "is_removed" : false, + "id" : 4901692112996, + "source" : "shopify", + "current_discounted_price_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "price_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "title" : "Economy", + "discounted_price_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "discounted_price" : "0.00" + } ], + "name" : "#1011", + "cart_token" : "hWN1SQkj4QbLb1NKF6pwXgwM", + "total_tax_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "estimated_taxes" : false, + "current_total_tax_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "current_subtotal_price" : "6300.00", + "total_outstanding" : "0.00", + "order_number" : 1011, + "created_at" : "2025-08-05T14:19:52-04:00", + "total_line_items_price_set" : { + "shop_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + } + }, + "taxes_included" : false, + "buyer_accepts_marketing" : true, + "confirmed" : true, + "total_weight" : 0, + "contact_email" : "brian+shop12@mayalane.com", + "total_discounts" : "0.00", + "client_details" : { + "accept_language" : "en-US", + "user_agent" : "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.5 Safari/605.1.15", + "browser_ip" : "2601:140:9700:2530:242b:7ae2:7ed:d2a1" + }, + "updated_at" : "2025-08-05T14:19:54-04:00", + "referring_site" : "", + "processed_at" : "2025-08-05T14:19:50-04:00", + "currency" : "USD", + "shipping_address" : { + "zip" : "19975", + "country_code" : "US", + "country" : "United States", + "province" : "Delaware", + "city" : "Selbyville", + "address1" : "31221 Americana Parkway", + "latitude" : 38.4694439, + "name" : "Larry Barry", + "last_name" : "Barry", + "province_code" : "DE", + "first_name" : "Larry", + "longitude" : -75.1137177 + }, + "source_name" : "web", + "browser_ip" : "2601:140:9700:2530:242b:7ae2:7ed:d2a1", + "current_shipping_price_set" : { + "shop_money" : { + "amount" : "0.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "0.00", + "currency_code" : "USD" + } + }, + "email" : "brian+shop12@mayalane.com", + "total_price_set" : { + "shop_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + }, + "presentment_money" : { + "amount" : "6300.00", + "currency_code" : "USD" + } + }, + "total_price" : "6300.00", + "full_landing_site" : "http://qx0wtr-1y.myshopify.com/", + "total_line_items_price" : "6300.00", + "duties_included" : false, + "total_tip_received" : "0.00", + "sms_order_updates_details" : { + "created_at" : "2025-08-05T14:19:52-04:00", + "order_id" : 5777917050980, + "email" : "brian+shop12@mayalane.com" + }, + "token" : "2e5c63a1dad1fd79265d096dfaa57915", + "current_total_price" : "6300.00", + "webhook_id" : "dc48c824-ebb6-444f-b42d-281dde4e7851", + "admin_graphql_api_id" : "gid://shopify/Order/5777917050980", + "financial_status" : "paid", + "customer" : { + "tax_exempt" : false, + "email_marketing_consent" : { + "consent_updated_at" : "2025-08-05T14:19:53-04:00", + "state" : "subscribed", + "opt_in_level" : "single_opt_in" + }, + "created_at" : "2025-08-05T14:19:51-04:00", + "last_name" : "Barry", + "verified_email" : true, + "tags" : "", + "default_address" : { + "zip" : "19975", + "country" : "United States", + "city" : "Selbyville", + "address1" : "31221 Americana Parkway", + "last_name" : "Barry", + "province_code" : "DE", + "country_code" : "US", + "default" : true, + "province" : "Delaware", + "country_name" : "United States", + "name" : "Larry Barry", + "id" : 9025902346340, + "customer_id" : 7824081551460, + "first_name" : "Larry" + }, + "updated_at" : "2025-08-05T14:19:53-04:00", + "admin_graphql_api_id" : "gid://shopify/Customer/7824081551460", + "currency" : "USD", + "id" : 7824081551460, + "state" : "disabled", + "first_name" : "Larry", + "email" : "brian+shop12@mayalane.com" + } + }, + "Collections" : [ "Abstract Art" ], + "Total Discounts" : "0.00" + } + }, + "links" : { + "self" : "https://a.klaviyo.com/api/events/6peNgN4t4ss/" + }, + "id" : "6peNgN4t4ss", + "type" : "event" + } + service: "ecommerce-events" + status: "info" + tags: + - "source:LOGS_SOURCE" + timestamp: 1754418010000 \ No newline at end of file diff --git a/klaviyo/assets/monitors/high_number_of_refunds.json b/klaviyo/assets/monitors/high_number_of_refunds.json new file mode 100644 index 0000000000000..2daa548e5ca70 --- /dev/null +++ b/klaviyo/assets/monitors/high_number_of_refunds.json @@ -0,0 +1,34 @@ +{ + "version": 2, + "created_at": "2025-07-28", + "last_updated_at": "2025-07-28", + "title": "High number of refunded orders", + "description": "Alert when the number of refunds within an hour exceeds threshold.", + "definition": { + "id": 177146470, + "name": "High number of refunded orders", + "type": "log alert", + "query": "logs(\"source:klaviyo service:ecommerce-events @metric_name:\\\"Refunded Order\\\"\").index(\"*\").rollup(\"count\").last(\"1h\") > 30", + "message": "The number of refunded orders has exceeded the {{#is_warning}} warning {{/is_warning}} {{#is_alert}} alert {{/is_alert}} threshold.", + "tags": [ + "source:klaviyo", + "service:ecommerce-events" + ], + "options": { + "thresholds": { + "critical": 30, + "warning": 20 + }, + "enable_logs_sample": false, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": false, + "new_host_delay": 300, + "groupby_simple_monitor": false + }, + "priority": null + }, + "tags": [ + "integration:klaviyo" + ] +} diff --git a/klaviyo/assets/monitors/high_percentage_of_email_failures.json b/klaviyo/assets/monitors/high_percentage_of_email_failures.json new file mode 100644 index 0000000000000..cabc1c95aeabc --- /dev/null +++ b/klaviyo/assets/monitors/high_percentage_of_email_failures.json @@ -0,0 +1,87 @@ +{ + "version": 2, + "created_at": "2025-07-28", + "last_updated_at": "2025-07-28", + "title": "High percentage of dropped or bounced email", + "description": "Alert when percentage of dropped or bounced email exceeds threshold.", + "definition": { + "id": 177147616, + "name": "High percentage of dropped or bounced email", + "type": "log alert", + "query": "formula(\"query / (query + query1)\").last(\"1h\") > 10", + "message": "The percentage of dropped or bounced email has exceeded the {{#is_warning}} warning {{/is_warning}} {{#is_alert}} alert {{/is_alert}} threshold.", + "tags": [ + "source:klaviyo", + "service:marketing-events" + ], + "options": { + "thresholds": { + "critical": 10, + "warning": 5 + }, + "enable_logs_sample": false, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": false, + "new_group_delay": 0, + "variables": [ + { + "name": "query", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:(\"Dropped Email\" OR \"Bounced Email\")" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + } + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received Email\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "groupby_simple_monitor": false + }, + "priority": null + }, + "tags": [ + "integration:klaviyo" + ] +} diff --git a/klaviyo/assets/monitors/high_percentage_of_email_marked_as_spam.json b/klaviyo/assets/monitors/high_percentage_of_email_marked_as_spam.json new file mode 100644 index 0000000000000..82d879a260ab6 --- /dev/null +++ b/klaviyo/assets/monitors/high_percentage_of_email_marked_as_spam.json @@ -0,0 +1,88 @@ +{ + "version": 2, + "created_at": "2025-07-28", + "last_updated_at": "2025-07-28", + "title": "High percentage of email marked as spam", + "description": "Alert when percentage of email marked as spam exceeds threshold.", + "definition": { + "id": 177258435, + "name": "High percentage of email marked as spam", + "type": "log alert", + "query": "formula(\"query / (query + query1)\").last(\"1d\") > 5", + "message": "The percentage of email marked as spam has exceeded the {{#is_warning}} warning {{/is_warning}} {{#is_alert}} alert {{/is_alert}} threshold.", + "tags": [ + "source:klaviyo", + "service:marketing-events" + ], + "options": { + "thresholds": { + "critical": 5, + "warning": 2 + }, + "enable_logs_sample": false, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": false, + "new_group_delay": 0, + "variables": [ + { + "name": "query", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Marked Email as Spam\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received Email\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "groupby_simple_monitor": false + }, + "priority": null + }, + "tags": [ + "integration:klaviyo" + ] +} diff --git a/klaviyo/assets/monitors/high_percentage_of_sms_failures.json b/klaviyo/assets/monitors/high_percentage_of_sms_failures.json new file mode 100644 index 0000000000000..578d5f1cfc1b4 --- /dev/null +++ b/klaviyo/assets/monitors/high_percentage_of_sms_failures.json @@ -0,0 +1,88 @@ +{ + "version": 2, + "created_at": "2025-07-28", + "last_updated_at": "2025-07-28", + "title": "High percentage of SMS delivery failures", + "description": "Alert when percentage of recent SMS failures exceeds threshold.", + "definition": { + "id": 177257688, + "name": "High percentage of SMS delivery failures", + "type": "log alert", + "query": "formula(\"query / (query + query1)\").last(\"1h\") > 10", + "message": "The percentage of SMS delivery failures has exceeded the {{#is_warning}} warning {{/is_warning}} {{#is_alert}} alert {{/is_alert}} threshold.", + "tags": [ + "source:klaviyo", + "service:marketing-events" + ], + "options": { + "thresholds": { + "critical": 10, + "warning": 5 + }, + "enable_logs_sample": false, + "notify_audit": false, + "on_missing_data": "default", + "include_tags": false, + "new_group_delay": 0, + "variables": [ + { + "name": "query", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Failed to Deliver SMS\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + }, + { + "name": "query1", + "data_source": "logs", + "search": { + "query": "source:klaviyo service:marketing-events @metric_name:\"Received SMS\"" + }, + "indexes": [ + "*" + ], + "group_by": [ + { + "facet": "@klaviyo.campaign_name", + "limit": 10, + "sort": { + "aggregation": "count", + "order": "desc", + "metric": "count" + }, + "should_exclude_missing": true + } + ], + "compute": { + "aggregation": "count" + }, + "storage": "hot" + } + ], + "groupby_simple_monitor": false + }, + "priority": null + }, + "tags": [ + "integration:klaviyo" + ] +} diff --git a/klaviyo/images/klaviyo_ecommerce_overview.png b/klaviyo/images/klaviyo_ecommerce_overview.png new file mode 100644 index 0000000000000..776b4648d649a Binary files /dev/null and b/klaviyo/images/klaviyo_ecommerce_overview.png differ diff --git a/klaviyo/images/klaviyo_marketing_overview_part1.png b/klaviyo/images/klaviyo_marketing_overview_part1.png new file mode 100644 index 0000000000000..d872cd1f18624 Binary files /dev/null and b/klaviyo/images/klaviyo_marketing_overview_part1.png differ diff --git a/klaviyo/images/klaviyo_marketing_overview_part2.png b/klaviyo/images/klaviyo_marketing_overview_part2.png new file mode 100644 index 0000000000000..0823606e52c93 Binary files /dev/null and b/klaviyo/images/klaviyo_marketing_overview_part2.png differ diff --git a/klaviyo/manifest.json b/klaviyo/manifest.json index 98d1c9de4be33..bc80d49c83d3f 100644 --- a/klaviyo/manifest.json +++ b/klaviyo/manifest.json @@ -2,7 +2,7 @@ "manifest_version": "2.0.0", "app_uuid": "bd010509-c47d-49ed-a9a7-d8c699c88f4c", "app_id": "klaviyo", - "display_on_public_website": false, + "display_on_public_website": true, "tile": { "overview": "README.md#Overview", "configuration": "README.md#Setup", @@ -10,7 +10,22 @@ "changelog": "CHANGELOG.md", "description": "Gain insights into Klaviyo marketing and eCommerce events.", "title": "Klaviyo", - "media": [], + "media": [ + { + "caption": "Klaviyo - Marketing Overview", + "image_url": "images/klaviyo_marketing_overview_part1.png", + "media_type": "image" + }, + { + "caption": "Klaviyo - Marketing Overview (Continued)", + "image_url": "images/klaviyo_marketing_overview_part2.png", + "media_type": "image" + }, + { + "caption": "Klaviyo - eCommerce Overview", + "image_url": "images/klaviyo_ecommerce_overview.png", + "media_type": "image" + }], "classifier_tags": [ "Category::Log Collection", "Submitted Data Type::Logs", @@ -28,6 +43,19 @@ "service_checks": { "metadata_path": "assets/service_checks.json" } + }, + "dashboards": { + "Klaviyo - Marketing Overview": "assets/dashboards/klaviyo_marketing_overview.json", + "Klaviyo - eCommerce Overview": "assets/dashboards/klaviyo_ecommerce_overview.json" + }, + "logs": { + "source": "klaviyo" + }, + "monitors": { + "High number of refunded orders" : "assets/monitors/high_number_of_refunds.json", + "High percentage of dropped or bounced email" : "assets/monitors/high_percentage_of_email_failures.json", + "High percentage of email marked as spam" : "assets/monitors/high_percentage_of_email_marked_as_spam.json", + "High percentage of SMS delivery failures" : "assets/monitors/high_percentage_of_sms_failures.json" } }, "author": { diff --git a/mysql/metadata.csv b/mysql/metadata.csv index b5485e9464819..a1190d85be2b6 100644 --- a/mysql/metadata.csv +++ b/mysql/metadata.csv @@ -176,7 +176,8 @@ mysql.performance.open_tables,gauge,,table,,The number of of tables that are ope mysql.performance.opened_tables,gauge,,table,second,"The number of tables that have been opened. If `opened_tables` is big, your `table_open_cache` value is probably too small.",0,mysql,mysql performance opened_tables, mysql.performance.performance_schema_digest_lost,gauge,,,,The number of digest instances that could not be instrumented in the events_statements_summary_by_digest table. This can be nonzero if the value of performance_schema_digests_size is too small.,0,mysql,mysql performance performance schema digest lost, mysql.performance.prepared_stmt_count,gauge,,query,second,The current number of prepared statements.,0,mysql,current prepared statements, -mysql.performance.qcache.utilization,gauge,,fraction,,Fraction of the query cache memory currently being used.,0,mysql,mysql performance qcache utilization, +mysql.performance.qcache.utilization,gauge,,percent,,Percentage of queries served from query cache since server start.,0,mysql,mysql performance qcache utilization, +mysql.performance.qcache.utilization.instant,gauge,,percent,,Percentage of queries served from query cache since last check.,0,mysql,mysql performance qcache utilization delta, mysql.performance.qcache_free_blocks,gauge,,block,,The number of free memory blocks in the query cache.,0,mysql,mysql performance qcache_free_blocks, mysql.performance.qcache_free_memory,gauge,,byte,,The amount of free memory for the query cache.,0,mysql,mysql performance qcache_free_memory, mysql.performance.qcache_hits,gauge,,hit,second,The rate of query cache hits.,0,mysql,query cache hits, diff --git a/openai/assets/dashboards/cost_overview_dashboard.json b/openai/assets/dashboards/cost_overview_dashboard.json index 42818440b72a1..c86d35ed5e918 100644 --- a/openai/assets/dashboards/cost_overview_dashboard.json +++ b/openai/assets/dashboards/cost_overview_dashboard.json @@ -1,1048 +1,1606 @@ { "title": "OpenAI Cost Overview", "description": "This dashboard provides insights into various OpenAI costs and their sources of attribution.", - "widgets": [{ - "id": 3636494906945158, - "definition": { - "type": "note", - "content": "The 3 sections in this dashboard all require Cloud Cost Management for OpenAI to be set up in order to see data.\n\n1. **OpenAI Cost Management**: [Set up](https://docs.datadoghq.com/cloud_cost_management/saas_costs/?tab=openai#setup) Cloud Cost Management and then the OpenAI integration\n2. **OpenAI Integration (API)**: [Set up](https://docs.datadoghq.com/integrations/openai/?tab=apikey) the **free** API set-up portion\n3. **LLM Observability**: [Set up](https://docs.datadoghq.com/llm_observability/setup/?tab=decorators) LLM Observability \n\nYou can clone and customize this dashboard for your specific tags", - "background_color": "blue", - "font_size": "16", - "text_align": "left", - "vertical_align": "center", - "show_tick": false, - "tick_pos": "50%", - "tick_edge": "left", - "has_padding": true - }, - "layout": { - "x": 0, - "y": 0, - "width": 12, - "height": 3 - } - }, { - "id": 8347049005274980, - "definition": { - "title": "OpenAI Cost Management", - "background_color": "vivid_green", - "show_title": true, - "type": "group", - "layout_type": "ordered", - "widgets": [{ - "id": 8416974066900506, - "definition": { - "type": "note", - "content": "Set up the Cloud Cost Management [OpenAI Cost Integration](https://docs.datadoghq.com/cloud_cost_management/saas_costs/?tab=openai#setup) to see data in this section. ", - "background_color": "green", - "font_size": "14", - "text_align": "center", - "vertical_align": "center", - "show_tick": false, - "tick_pos": "50%", - "tick_edge": "left", - "has_padding": true + "widgets": [ + { + "id": 151461691119560, + "definition": { + "title": "Total OpenAI Spend", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true }, - "layout": { - "x": 0, - "y": 0, - "width": 12, - "height": 1 - } - }, { - "id": 409821317360376, - "definition": { - "type": "image", - "url": "/static/images/logos/openai_large.svg", - "url_dark_theme": "/static/images/logos/openai_reversed_large.svg", - "sizing": "cover", - "has_background": true, - "has_border": true, - "vertical_align": "center", - "horizontal_align": "center" + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:custom.cost.amortized{providername:OpenAI,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + } + ], + "response_format": "scalar" + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 0, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 5042132219805083, + "definition": { + "title": "Total Costs (past 30d)", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "month", + "value": 1, + "hide_incomplete_cost_data": true }, - "layout": { - "x": 0, - "y": 1, - "width": 3, - "height": 2 - } - }, { - "id": 1457644073160806, - "definition": { - "title": "Total Costs (past 30d)", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true - }, - "type": "query_value", - "requests": [{ - "formulas": [{ - "formula": "query1" - }], - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:custom.cost.amortized{providername:OpenAI,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "aggregator": "sum" - }], + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:custom.cost.amortized{provider_name:OpenAI,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + } + ], "response_format": "scalar" - }], - "autoscale": true, - "precision": 2 + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 3, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 3253421450589468, + "definition": { + "title": "Total Costs (30d prior)", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "month", + "value": 1, + "hide_incomplete_cost_data": true }, - "layout": { - "x": 3, - "y": 1, - "width": 3, - "height": 2 - } - }, { - "id": 5140360304170010, - "definition": { - "title": "Total Costs (30d prior)", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true - }, - "type": "query_value", - "requests": [{ - "formulas": [{ - "formula": "query1" - }], - "queries": [{ - "query": "sum:custom.cost.amortized{providername:OpenAI,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "data_source": "cloud_cost", - "name": "query1", - "aggregator": "sum" - }], + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "timeshift(query1, -2592000)" + } + ], + "queries": [ + { + "query": "sum:custom.cost.amortized{provider_name:OpenAI,$api_key,$ml_app,$service,$model,$project}", + "data_source": "cloud_cost", + "name": "query1", + "aggregator": "sum" + } + ], "response_format": "scalar" - }], - "autoscale": true, - "precision": 2 + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 6, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 1285180881880373, + "definition": { + "title": "MoM Change", + "title_size": "16", + "title_align": "left", + "time": { + "type": "live", + "unit": "month", + "value": 1, + "hide_incomplete_cost_data": true }, - "layout": { - "x": 6, - "y": 1, - "width": 3, - "height": 2 - } - }, { - "id": 8170994894343814, - "definition": { - "title": "% change (past 30d vs 30d prior)", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true - }, - "type": "query_value", - "requests": [{ - "formulas": [{ - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "percent" - } + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "percent" + } + }, + "formula": "(query1 - timeshift(query2, -2592000)) / timeshift(query2, -2592000) * 100" + } + ], + "queries": [ + { + "query": "sum:custom.cost.amortized{provider_name:OpenAI,$api_key,$ml_app,$service,$model,$project}", + "data_source": "cloud_cost", + "name": "query1", + "aggregator": "sum" }, - "formula": "(query1 - calendar_shift(query2, '-1mo', 'Europe/Paris')) / calendar_shift(query2, '-1mo', 'Europe/Paris') * 100" - }], - "queries": [{ - "query": "sum:custom.cost.amortized{providername:OpenAI,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "data_source": "cloud_cost", - "name": "query1", - "aggregator": "sum" - }, { - "query": "sum:custom.cost.amortized{providername:OpenAI,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "data_source": "cloud_cost", - "name": "query2", - "aggregator": "sum" - }], + { + "query": "sum:custom.cost.amortized{provider_name:OpenAI,$api_key,$ml_app,$service,$model,$project}", + "data_source": "cloud_cost", + "name": "query2", + "aggregator": "sum" + } + ], "response_format": "scalar", - "conditional_formats": [{ - "comparator": "<", - "value": 5, - "palette": "black_on_light_green" - }, { - "comparator": "<", - "value": 10, - "palette": "black_on_light_yellow" - }, { - "comparator": ">", - "value": 10, - "palette": "black_on_light_red" - }] - }], - "autoscale": true, - "precision": 2 - }, - "layout": { - "x": 9, - "y": 1, - "width": 3, - "height": 2 - } - }, { - "id": 3631138712976072, - "definition": { - "type": "note", - "content": "OpenAI models have different capabilities and price points, and are priced by the **number of input and output tokens**.\n\nIn Cloud Cost Management, costs are broken down by the **input and output costs per model**. ", - "background_color": "green", - "font_size": "14", - "text_align": "center", - "vertical_align": "center", - "show_tick": false, - "tick_pos": "50%", - "tick_edge": "left", - "has_padding": true + "conditional_formats": [ + { + "comparator": "<", + "value": 5, + "palette": "black_on_light_green" + }, + { + "comparator": "<", + "value": 10, + "palette": "black_on_light_yellow" + }, + { + "comparator": "<", + "value": 100, + "palette": "black_on_light_red" + }, + { + "comparator": ">=", + "value": 100, + "palette": "white_on_red" + } + ] + } + ], + "autoscale": true, + "precision": 2 + }, + "layout": { + "x": 9, + "y": 0, + "width": 3, + "height": 2 + } + }, + { + "id": 4923995266806901, + "definition": { + "type": "note", + "content": "OpenAI models have different capabilities and price points, and are priced by the **number of input and output tokens**.\n\nIn Cloud Cost Management, costs are broken down by the **input and output costs per model**. ", + "background_color": "green", + "font_size": "14", + "text_align": "center", + "vertical_align": "center", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 2, + "width": 3, + "height": 4 + } + }, + { + "id": 6176679445548644, + "definition": { + "title": "Cost per ServiceName", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true }, - "layout": { - "x": 0, - "y": 3, - "width": 3, - "height": 4 - } - }, { - "id": 8525206763937204, - "definition": { - "title": "Cost per ServiceName", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true - }, - "requests": [{ + "requests": [ + { "response_format": "scalar", - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:OpenAI,$api_key,$ml_app,$service,$model,$project} by {servicename}", - "aggregator": "sum" - }], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:OpenAI,$api_key,$ml_app,$service,$model,$project} by {servicename}", + "aggregator": "sum" + } + ], "style": { - "palette": "datadog16" + "palette": "classic" }, - "formulas": [{ - "formula": "query1" - }], + "formulas": [ + { + "formula": "query1" + } + ], "sort": { "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] } - }], - "type": "sunburst", - "legend": { - "type": "table" } - }, - "layout": { - "x": 3, - "y": 3, - "width": 9, - "height": 4 + ], + "type": "sunburst", + "legend": { + "type": "table" } - }, { - "id": 4968678189694062, - "definition": { - "title": "Cost Changes Over Time by ServiceName", - "title_size": "16", - "title_align": "left", - "show_legend": false, - "legend_layout": "auto", - "legend_columns": ["avg", "min", "max", "value", "sum"], - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true - }, - "type": "timeseries", - "requests": [{ - "formulas": [{ - "formula": "query1" - }], - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:OpenAI,$api_key,$ml_app,$service,$model,$project} by {servicename}.rollup(sum, daily)" - }], + }, + "layout": { + "x": 3, + "y": 2, + "width": 9, + "height": 4 + } + }, + { + "id": 5809017898757948, + "definition": { + "title": "Cost Changes Over Time by ServiceName", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "hide_incomplete_cost_data": true + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:OpenAI,$api_key,$ml_app,$service,$model,$project} by {servicename}.rollup(sum, daily)" + } + ], "response_format": "timeseries", "style": { - "palette": "datadog16", + "palette": "classic", "order_by": "values", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" - }] + } + ] + }, + "layout": { + "x": 0, + "y": 6, + "width": 12, + "height": 3 + } + }, + { + "id": 132965784094236, + "definition": { + "title": "Cost by Organization and Project", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true }, - "layout": { - "x": 0, - "y": 7, - "width": 12, - "height": 3 - } - }, { - "id": 7084921580361980, - "definition": { - "title": "Cost by Organization and Project", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true - }, - "type": "toplist", - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:OpenAI,$api_key,$ml_app,$service,$model,$project} by {organization_name,project_name}", - "aggregator": "sum" - }], + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:OpenAI,$api_key,$ml_app,$service,$model,$project} by {organization_name,project_name}", + "aggregator": "sum" + } + ], "response_format": "scalar", - "formulas": [{ - "formula": "query1" - }], + "formulas": [ + { + "formula": "query1" + } + ], "sort": { "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] } - }], - "style": { - "display": { - "type": "stacked", - "legend": "automatic" - }, - "palette": "datadog16" } - }, - "layout": { - "x": 0, - "y": 10, - "width": 5, - "height": 3 - } - }, { - "id": 2890013906934278, - "definition": { - "title": "Project Costs Over Time", - "title_size": "16", - "title_align": "left", - "show_legend": true, - "legend_layout": "auto", - "legend_columns": ["avg", "min", "max", "value", "sum"], - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" }, - "type": "timeseries", - "requests": [{ - "formulas": [{ - "alias": "Total Cost", - "formula": "query1" - }], - "queries": [{ - "data_source": "cloud_cost", - "name": "query1", - "query": "sum:all.cost{providername:OpenAI,$api_key,$ml_app,$service,$model,$project} by {project_name}.rollup(sum, daily)" - }], + "palette": "dog_classic" + } + }, + "layout": { + "x": 0, + "y": 9, + "width": 5, + "height": 3 + } + }, + { + "id": 3178117944554849, + "definition": { + "title": "Project Costs Over Time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "hide_incomplete_cost_data": true + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "alias": "Total Cost", + "formula": "query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query1", + "query": "sum:all.cost{providername:OpenAI,$api_key,$ml_app,$service,$model,$project} by {project_name}.rollup(sum, daily)" + } + ], "response_format": "timeseries", "style": { - "palette": "dog_classic", + "palette": "classic", "order_by": "values", "line_type": "solid", "line_width": "normal" }, "display_type": "bars" - }] - }, - "layout": { - "x": 5, - "y": 10, - "width": 7, - "height": 3 - } - }] + } + ] + }, + "layout": { + "x": 5, + "y": 9, + "width": 7, + "height": 3 + } }, - "layout": { - "x": 0, - "y": 3, - "width": 12, - "height": 14 - } - }, { - "id": 2162071155995882, - "definition": { - "title": "OpenAI Integration (API)", - "background_color": "vivid_blue", - "show_title": true, - "type": "group", - "layout_type": "ordered", - "widgets": [{ - "id": 6603322518845350, - "definition": { - "type": "note", - "content": "Set up the **free** [API part](https://app.datadoghq.com/integrations/openai?search=OpenAI) of the [OpenAI general integration](https://docs.datadoghq.com/integrations/openai/?tab=python) to see data in this section. Dive deeper in [OpenAI Usage Overview](https://app.datadoghq.com/dash/integration/31040/openai-usage-overview). Widgets are focused on GPT costs.", - "background_color": "blue", - "font_size": "14", - "text_align": "center", - "vertical_align": "center", - "show_tick": false, - "tick_pos": "50%", - "tick_edge": "left", - "has_padding": true - }, - "layout": { - "x": 0, - "y": 0, - "width": 12, - "height": 1 - } - }, { - "id": 7450240655684922, - "definition": { - "title": "Cost per Model", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true + { + "id": 2162071155995882, + "definition": { + "title": "OpenAI Integration (API)", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 6603322518845350, + "definition": { + "type": "note", + "content": "Set up the **free** [API part](https://app.datadoghq.com/integrations/openai?search=OpenAI) of the [OpenAI general integration](https://docs.datadoghq.com/integrations/openai/?tab=python) to see data in this section. Dive deeper in [OpenAI Usage Overview](https://app.datadoghq.com/dash/integration/31040/openai-usage-overview).", + "background_color": "green", + "font_size": "14", + "text_align": "center", + "vertical_align": "center", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 1 + } }, - "type": "toplist", - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query2", - "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*gpt*,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query3", - "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project} by {model}", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query4", - "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project} by {model}", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query1", - "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project}", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query5", - "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project}", - "aggregator": "sum" - }], - "response_format": "scalar", - "formulas": [{ - "formula": "query2 * (query3 + query4) / (query1 + query5)" - }], - "sort": { - "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + { + "id": 7450240655684922, + "definition": { + "title": "Cost per Model", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*GPT*,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project} by {model}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project} by {model}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query2 * (query3 + query4) / (query1 + query5)" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + } + } + }, + "layout": { + "x": 0, + "y": 1, + "width": 6, + "height": 3 } - }], - "style": { - "display": { - "type": "stacked", - "legend": "automatic" + }, + { + "id": 2146480474569892, + "definition": { + "title": "Cost per Operation", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{provider_name:OpenAI,servicename:*GPT*,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project} by {operation}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project} by {operation}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "style": { + "palette": "classic" + }, + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + }, + "formula": "query2 * (query3 + query4) / (query1 + query5)" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "type": "sunburst", + "legend": { + "type": "automatic" + } + }, + "layout": { + "x": 6, + "y": 1, + "width": 6, + "height": 3 } - } - }, - "layout": { - "x": 0, - "y": 1, - "width": 6, - "height": 3 - } - }, { - "id": 2146480474569892, - "definition": { - "title": "Cost per Operation", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true }, - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query2", - "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*gpt*,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query3", - "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project} by {operation}", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query4", - "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project} by {operation}", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query1", - "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project}", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query5", - "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project}", - "aggregator": "sum" - }], - "response_format": "scalar", - "style": { - "palette": "datadog16" + { + "id": 92163252088124, + "definition": { + "title": "Cost per 1 Billion Tokens", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + }, + "unit_scale": { + "type": "canonical_unit", + "unit_name": "dollar" + } + }, + "formula": "(query2 / (query1 + query5)) * 1000000000" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI AND (servicename:*GPT*)}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.api.usage.n_context_tokens_total{*}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "sum:openai.api.usage.n_generated_tokens_total{*}", + "aggregator": "avg" + } + ], + "response_format": "scalar" + } + ], + "autoscale": false, + "precision": 2 }, - "formulas": [{ - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "dollar" + "layout": { + "x": 0, + "y": 4, + "width": 6, + "height": 2 + } + }, + { + "id": 2871269133729220, + "definition": { + "title": "# Input Tokens", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query5" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query5", + "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project}", + "aggregator": "avg" + } + ], + "response_format": "scalar" } - }, - "formula": "query2 * (query3 + query4) / (query1 + query5)" - }], - "sort": { - "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + ], + "autoscale": true, + "precision": 2, + "time": { + "hide_incomplete_cost_data": true + } + }, + "layout": { + "x": 6, + "y": 4, + "width": 3, + "height": 2 } - }], - "type": "sunburst", - "legend": { - "type": "automatic" - } - }, - "layout": { - "x": 6, - "y": 1, - "width": 6, - "height": 3 - } - }, { - "id": 92163252088124, - "definition": { - "title": "Cost per Token ", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true }, - "type": "query_value", - "requests": [{ - "formulas": [{ - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "dollar" - }, - "unit_scale": { - "type": "canonical_unit", - "unit_name": "dollar" + { + "id": 5960580251979426, + "definition": { + "title": "# Output Tokens", + "title_size": "16", + "title_align": "left", + "type": "query_value", + "requests": [ + { + "formulas": [ + { + "formula": "query5" + } + ], + "queries": [ + { + "data_source": "metrics", + "name": "query5", + "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project}", + "aggregator": "avg" + } + ], + "response_format": "scalar" } - }, - "formula": "(query2 / (query1 + query5))" - }], - "queries": [{ - "data_source": "cloud_cost", - "name": "query2", - "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*gpt*,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query1", - "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project}", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query5", - "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project}", - "aggregator": "avg" - }], - "response_format": "scalar" - }], - "autoscale": false, - "precision": 2 - }, - "layout": { - "x": 0, - "y": 4, - "width": 6, - "height": 2 - } - }, { - "id": 2871269133729220, - "definition": { - "title": "# Input Tokens", - "title_size": "16", - "title_align": "left", - "type": "query_value", - "requests": [{ - "formulas": [{ - "formula": "query5" - }], - "queries": [{ - "data_source": "metrics", - "name": "query5", - "query": "sum:openai.api.usage.n_context_tokens_total{$api_key,$ml_app,$service,$model,$project}", - "aggregator": "avg" - }], - "response_format": "scalar" - }], - "autoscale": true, - "precision": 2 - }, - "layout": { - "x": 6, - "y": 4, - "width": 3, - "height": 2 - } - }, { - "id": 5960580251979426, - "definition": { - "title": "# Output Tokens", - "title_size": "16", - "title_align": "left", - "type": "query_value", - "requests": [{ - "formulas": [{ - "formula": "query5" - }], - "queries": [{ - "data_source": "metrics", - "name": "query5", - "query": "sum:openai.api.usage.n_generated_tokens_total{$api_key,$ml_app,$service,$model,$project}", - "aggregator": "avg" - }], - "response_format": "scalar" - }], - "autoscale": true, - "precision": 2 - }, - "layout": { - "x": 9, - "y": 4, - "width": 3, - "height": 2 - } - }] + ], + "autoscale": true, + "precision": 2, + "time": { + "hide_incomplete_cost_data": true + } + }, + "layout": { + "x": 9, + "y": 4, + "width": 3, + "height": 2 + } + } + ] + }, + "layout": { + "x": 0, + "y": 12, + "width": 12, + "height": 7 + } }, - "layout": { - "x": 0, - "y": 17, - "width": 12, - "height": 7 - } - }, { - "id": 8181163626805416, - "definition": { - "title": "LLM Observability ", - "background_color": "vivid_purple", - "show_title": true, - "type": "group", - "layout_type": "ordered", - "widgets": [{ - "id": 7136039305405288, - "definition": { - "type": "image", - "url": "/static/images/integration_dashboard/llm-observability_hero-1.jpeg", - "sizing": "cover", - "has_background": true, - "has_border": true, - "vertical_align": "center", - "horizontal_align": "center" - }, - "layout": { - "x": 0, - "y": 0, - "width": 4, - "height": 1 - } - }, { - "id": 3520544867910278, - "definition": { - "type": "note", - "content": "Please [set up](https://docs.datadoghq.com/llm_observability/) LLM Observability to see data in this section. With [LLM Observability](https://app.datadoghq.com/llm/traces), you can monitor, troubleshoot, and evaluate your LLM-powered applications. ", - "background_color": "purple", - "font_size": "14", - "text_align": "center", - "vertical_align": "center", - "show_tick": false, - "tick_pos": "50%", - "tick_edge": "left", - "has_padding": true - }, - "layout": { - "x": 4, - "y": 0, - "width": 8, - "height": 1 - } - }, { - "id": 2767009516392116, - "definition": { - "title": "Cost per ML App", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true - }, - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query2", - "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*gpt*,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query3", - "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project} by {ml_app}.as_count()", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query4", - "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project} by {ml_app}.as_count()", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query1", - "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query5", - "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", - "aggregator": "sum" - }], - "response_format": "scalar", - "style": { - "palette": "datadog16" + { + "id": 3812785000101592, + "definition": { + "title": "OpenAI Integration (APM)", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 1797831271623678, + "definition": { + "type": "note", + "content": "Set up the [APM](https://app.datadoghq.com/integrations/openai?search=OpenAI) part of [OpenAI general integration](https://docs.datadoghq.com/integrations/openai/?tab=python) to see data in this section. You must be an APM customer to set up. Dive deeper in [OpenAI Overview Dashboard](https://app.datadoghq.com/dash/integration/30978/openai-overview-dashboard). ", + "background_color": "green", + "font_size": "14", + "text_align": "center", + "vertical_align": "center", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true }, - "formulas": [{ - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "dollar" + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 1 + } + }, + { + "id": 6305672979624266, + "definition": { + "title": "Cost per Service", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project} by {service}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + }, + "formula": "query2 * query3 / query1" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 0, + "y": 1, + "width": 4, + "height": 3 + } + }, + { + "id": 205108745443972, + "definition": { + "title": "Cost per Service Over Time", + "title_size": "16", + "title_align": "left", + "show_legend": false, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "hide_incomplete_cost_data": true }, - "formula": "query2 * (query3 + query4) / (query1 + query5)" - }], - "sort": { - "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + }, + "formula": "query2 * query3 / query1" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project} by {service}.as_count()" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project}.as_count()" + } + ], + "response_format": "timeseries", + "style": { + "palette": "classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ] + }, + "layout": { + "x": 4, + "y": 1, + "width": 8, + "height": 3 } - }], - "type": "sunburst", - "legend": { - "type": "automatic" - } - }, - "layout": { - "x": 0, - "y": 1, - "width": 4, - "height": 3 - } - }, { - "id": 650790161525400, - "definition": { - "title": "Cost per ML App Over Time", - "title_size": "16", - "title_align": "left", - "show_legend": true, - "legend_layout": "auto", - "legend_columns": ["avg", "min", "max", "value", "sum"], - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true }, - "type": "timeseries", - "requests": [{ - "formulas": [{ - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "dollar" + { + "id": 2997929981443078, + "definition": { + "title": "Cost per Org", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{provider_name:OpenAI,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project} by {openai.organization.name}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query2 * query3 / query1", + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + } + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 0, + "y": 4, + "width": 4, + "height": 3 + } + }, + { + "id": 1477288913573598, + "definition": { + "title": "Cost per Env", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true }, - "formula": "query2 * (query3 + query4) / (query1 + query5)" - }], - "queries": [{ - "data_source": "cloud_cost", - "name": "query2", - "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*gpt*,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)" - }, { - "data_source": "metrics", - "name": "query3", - "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project} by {ml_app}.as_count()" - }, { - "data_source": "metrics", - "name": "query4", - "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project} by {ml_app}.as_count()" - }, { - "data_source": "metrics", - "name": "query1", - "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()" - }, { - "data_source": "metrics", - "name": "query5", - "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()" - }], - "response_format": "timeseries", - "style": { - "palette": "datadog16", - "order_by": "values", - "line_type": "solid", - "line_width": "normal" + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project} by {env}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + }, + "formula": "query2 * query3 / query1" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } }, - "display_type": "bars" - }] - }, - "layout": { - "x": 4, - "y": 1, - "width": 8, - "height": 3 - } - }, { - "id": 1463818413200384, - "definition": { - "title": "Cost per Service", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true + "layout": { + "x": 4, + "y": 4, + "width": 4, + "height": 3 + } }, - "type": "toplist", - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query2", - "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*gpt*,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query3", - "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project} by {service}.as_count()", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query4", - "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project} by {service}.as_count()", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query1", - "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query5", - "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", - "aggregator": "sum" - }], - "response_format": "scalar", - "formulas": [{ - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "dollar" + { + "id": 5959445400656100, + "definition": { + "title": "Cost per API key", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project} by {openai.user.api_key}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:openai.tokens.total{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "number_format": { + "unit": { + "type": "canonical_unit", + "unit_name": "dollar" + } + }, + "formula": "query2 * query3 / query1" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 8, + "y": 4, + "width": 4, + "height": 3 + } + } + ] + }, + "layout": { + "x": 0, + "y": 19, + "width": 12, + "height": 8, + "is_column_break": true + } + }, + { + "id": 8181163626805416, + "definition": { + "title": "LLM Observability ", + "background_color": "vivid_green", + "show_title": true, + "type": "group", + "layout_type": "ordered", + "widgets": [ + { + "id": 3520544867910278, + "definition": { + "type": "note", + "content": "Set up [LLM Observability](https://docs.datadoghq.com/llm_observability/) to see data in this section. With LLM Observability [Traces](https://app.datadoghq.com/llm/traces), you can monitor, troubleshoot, and evaluate your LLM-powered applications. ", + "background_color": "green", + "font_size": "14", + "text_align": "center", + "vertical_align": "center", + "show_tick": false, + "tick_pos": "50%", + "tick_edge": "left", + "has_padding": true + }, + "layout": { + "x": 0, + "y": 0, + "width": 12, + "height": 1 + } + }, + { + "id": 2767009516392116, + "definition": { + "title": "Cost per ML App", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true }, - "formula": "query2 * (query3 + query4) / (query1 + query5)" - }], - "sort": { - "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*GPT*,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project} by {ml_app}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project} by {ml_app}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "style": { + "palette": "classic" + }, + "formulas": [ + { + "formula": "query2 * (query3 + query4) / (query1 + query5)" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "type": "sunburst", + "legend": { + "type": "automatic" + } + }, + "layout": { + "x": 0, + "y": 1, + "width": 4, + "height": 3 } - }], - "style": { - "display": { - "type": "stacked", - "legend": "automatic" + }, + { + "id": 650790161525400, + "definition": { + "title": "Cost per ML App Over Time", + "title_size": "16", + "title_align": "left", + "show_legend": true, + "legend_layout": "auto", + "legend_columns": [ + "avg", + "min", + "max", + "value", + "sum" + ], + "time": { + "hide_incomplete_cost_data": true + }, + "type": "timeseries", + "requests": [ + { + "formulas": [ + { + "formula": "query2 * (query3 + query4) / (query1 + query5)" + } + ], + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*GPT*,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project} by {ml_app}.as_count()" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project} by {ml_app}.as_count()" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()" + } + ], + "response_format": "timeseries", + "style": { + "palette": "classic", + "order_by": "values", + "line_type": "solid", + "line_width": "normal" + }, + "display_type": "bars" + } + ] }, - "palette": "datadog16" - } - }, - "layout": { - "x": 0, - "y": 4, - "width": 6, - "height": 3 - } - }, { - "id": 1324979932092638, - "definition": { - "title": "Cost per Env", - "title_size": "16", - "title_align": "left", - "time": { - "type": "live", - "unit": "month", - "value": 1, - "hide_incomplete_cost_data": true + "layout": { + "x": 4, + "y": 1, + "width": 8, + "height": 3 + } }, - "type": "toplist", - "requests": [{ - "queries": [{ - "data_source": "cloud_cost", - "name": "query2", - "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*GPT*,$api_key,$ml_app,$service,$model,$project}.rollup(sum, daily)", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query3", - "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project} by {env}.as_count()", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query4", - "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project} by {env}.as_count()", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query1", - "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", - "aggregator": "sum" - }, { - "data_source": "metrics", - "name": "query5", - "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", - "aggregator": "sum" - }], - "response_format": "scalar", - "formulas": [{ - "number_format": { - "unit": { - "type": "canonical_unit", - "unit_name": "dollar" + { + "id": 1463818413200384, + "definition": { + "title": "Cost per Service", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*GPT*,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project} by {service}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project} by {service}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query2 * (query3 + query4) / (query1 + query5)" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 0, + "y": 4, + "width": 4, + "height": 3 + } + }, + { + "id": 1324979932092638, + "definition": { + "title": "Cost per Env", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true }, - "formula": "query2 * (query3 + query4) / (query1 + query5)" - }], - "sort": { - "count": 500, - "order_by": [{ - "type": "formula", - "index": 0, - "order": "desc" - }] + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{providername:OpenAI,servicename:*GPT*,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project} by {env}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project} by {env}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query2 * (query3 + query4) / (query1 + query5)" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } + }, + "layout": { + "x": 4, + "y": 4, + "width": 4, + "height": 3 } - }], - "style": { - "display": { - "type": "stacked", - "legend": "automatic" + }, + { + "id": 6992909168651373, + "definition": { + "title": "Cost per Model Name", + "title_size": "16", + "title_align": "left", + "time": { + "hide_incomplete_cost_data": true + }, + "type": "toplist", + "requests": [ + { + "queries": [ + { + "data_source": "cloud_cost", + "name": "query2", + "query": "sum:custom.cost.amortized{provider_name:OpenAI,servicename:*GPT*,$api_key,$ml_app,$service,$model,$project}", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query3", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project} by {model_name}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query4", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project} by {model_name}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query1", + "query": "sum:ml_obs.span.llm.input.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + }, + { + "data_source": "metrics", + "name": "query5", + "query": "sum:ml_obs.span.llm.output.tokens{$api_key,$ml_app,$service,$model,$project}.as_count()", + "aggregator": "sum" + } + ], + "response_format": "scalar", + "formulas": [ + { + "formula": "query2 * (query3 + query4) / (query1 + query5)" + } + ], + "sort": { + "count": 500, + "order_by": [ + { + "type": "formula", + "index": 0, + "order": "desc" + } + ] + } + } + ], + "style": { + "display": { + "type": "stacked", + "legend": "automatic" + }, + "palette": "dog_classic" + } }, - "palette": "datadog16" + "layout": { + "x": 8, + "y": 4, + "width": 4, + "height": 3 + } } - }, - "layout": { - "x": 6, - "y": 4, - "width": 6, - "height": 3 - } - }] + ] + }, + "layout": { + "x": 0, + "y": 27, + "width": 12, + "height": 8 + } + } + ], + "template_variables": [ + { + "name": "project", + "prefix": "project", + "available_values": [], + "default": "*" + }, + { + "name": "model", + "prefix": "model", + "available_values": [], + "default": "*" + }, + { + "name": "service", + "prefix": "service", + "available_values": [], + "default": "*" + }, + { + "name": "ml_app", + "prefix": "ml_app", + "available_values": [], + "default": "*" }, - "layout": { - "x": 0, - "y": 24, - "width": 12, - "height": 8 + { + "name": "api_key", + "prefix": "api_key", + "available_values": [], + "default": "*" } - }], - "template_variables": [{ - "name": "project", - "prefix": "project", - "available_values": [], - "default": "*" - }, { - "name": "model", - "prefix": "model", - "available_values": [], - "default": "*" - }, { - "name": "service", - "prefix": "service", - "available_values": [], - "default": "*" - }, { - "name": "ml_app", - "prefix": "ml_app", - "available_values": [], - "default": "*" - }, { - "name": "api_key", - "prefix": "api_key", - "available_values": [], - "default": "*" - }], + ], "layout_type": "ordered", "notify_list": [], "reflow_type": "fixed" -} \ No newline at end of file +} diff --git a/pgbouncer/changelog.d/21173.added b/pgbouncer/changelog.d/21173.added new file mode 100644 index 0000000000000..ea06e9e0efd6b --- /dev/null +++ b/pgbouncer/changelog.d/21173.added @@ -0,0 +1 @@ +Upgrade to psycopg3 diff --git a/pgbouncer/datadog_checks/pgbouncer/pgbouncer.py b/pgbouncer/datadog_checks/pgbouncer/pgbouncer.py index 15c9b691f57a8..b4653a2074136 100644 --- a/pgbouncer/datadog_checks/pgbouncer/pgbouncer.py +++ b/pgbouncer/datadog_checks/pgbouncer/pgbouncer.py @@ -5,8 +5,9 @@ import time from urllib.parse import urlparse -import psycopg2 as pg -from psycopg2 import extras as pgextras +import psycopg as pg +from psycopg import ClientCursor +from psycopg.rows import dict_row from datadog_checks.base import AgentCheck, ConfigurationError, is_affirmative from datadog_checks.pgbouncer.metrics import ( @@ -73,51 +74,50 @@ def _collect_stats(self, db): metric_scope.append(SERVERS_METRICS) try: - with db.cursor(cursor_factory=pgextras.DictCursor) as cursor: - for scope in metric_scope: - descriptors = scope['descriptors'] - metrics = scope['metrics'] - query = scope['query'] - - try: - self.log.debug("Running query: %s", query) - cursor.execute(query) - rows = self.iter_rows(cursor) - - except Exception as e: - self.log.exception("Not all metrics may be available: %s", str(e)) - - else: - for row in rows: - if 'key' in row: # We are processing "config metrics" - # Make a copy of the row to allow mutation - # (a `psycopg2.lib.extras.DictRow` object doesn't accept a new key) - row = row.copy() - # We flip/rotate the row: row value becomes the column name - row[row['key']] = row['value'] - # Skip the "pgbouncer" database - elif row.get('database') == self.DB_NAME: - continue - - tags = list(self.tags) - tags += ["%s:%s" % (tag, row[column]) for (column, tag) in descriptors if column in row] - for column, (name, reporter) in metrics: - if column in row: - value = row[column] - if column in ['connect_time', 'request_time']: - self.log.debug("Parsing timestamp; original value: %s", value) - # First get rid of any UTC suffix. - value = re.findall(r'^[^ ]+ [^ ]+', value)[0] - value = time.strptime(value, '%Y-%m-%d %H:%M:%S') - value = time.mktime(value) - reporter(self, name, value, tags) - - if not rows: - self.log.warning("No results were found for query: %s", query) - - except pg.Error: - self.log.exception("Connection error") + for scope in metric_scope: + descriptors = scope['descriptors'] + metrics = scope['metrics'] + query = scope['query'] + try: + cursor = db.cursor(row_factory=dict_row) + self.log.debug("Running query: %s", query) + cursor.execute(query) + rows = self.iter_rows(cursor) + + except (pg.InterfaceError, pg.OperationalError) as e: + self.log.error("Not all metrics may be available: %s", e) + raise ShouldReconnectException + + else: + for row in rows: + if 'key' in row: # We are processing "config metrics" + # Make a copy of the row to allow mutation + row = row.copy() + # We flip/rotate the row: row value becomes the column name + row[row['key']] = row['value'] + # Skip the "pgbouncer" database + elif row.get('database') == self.DB_NAME: + continue + + tags = list(self.tags) + tags += ["%s:%s" % (tag, row[column]) for (column, tag) in descriptors if column in row] + for column, (name, reporter) in metrics: + if column in row: + value = row[column] + if column in ['connect_time', 'request_time']: + self.log.debug("Parsing timestamp; original value: %s", value) + # First get rid of any UTC suffix. + value = re.findall(r'^[^ ]+ [^ ]+', value)[0] + value = time.strptime(value, '%Y-%m-%d %H:%M:%S') + value = time.mktime(value) + reporter(self, name, value, tags) + + if not rows: + self.log.warning("No results were found for query: %s", query) + + except pg.Error as e: + self.log.error("Not all metrics may be available: %s", e) raise ShouldReconnectException def iter_rows(self, cursor): @@ -138,21 +138,25 @@ def iter_rows(self, cursor): def _get_connect_kwargs(self): """ - Get the params to pass to psycopg2.connect() based on passed-in vals + Get the params to pass to psycopg.connect() based on passed-in vals from yaml settings file """ + # It's important to set the client_encoding to utf-8 + # PGBouncer defaults to an encoding of 'UNICODE`, which will cause psycopg to error out if self.database_url: - return {'dsn': self.database_url} + return {'conninfo': self.database_url, 'client_encoding': 'utf-8'} if self.host in ('localhost', '127.0.0.1') and self.password == '': # Use ident method - return {'dsn': "user={} dbname={}".format(self.user, self.DB_NAME)} + return {'conninfo': "user={} dbname={} client_encoding=utf-8".format(self.user, self.DB_NAME)} args = { 'host': self.host, 'user': self.user, 'password': self.password, - 'database': self.DB_NAME, + 'dbname': self.DB_NAME, + 'cursor_factory': ClientCursor, + 'client_encoding': 'utf-8', } if self.port: args['port'] = self.port @@ -164,8 +168,9 @@ def _new_connection(self): connection = None try: connect_kwargs = self._get_connect_kwargs() - connection = pg.connect(**connect_kwargs) - connection.set_isolation_level(pg.extensions.ISOLATION_LEVEL_AUTOCOMMIT) + # Somewhat counterintuitively, we need to set autocommit to True to avoid a BEGIN/COMMIT block + # https://www.psycopg.org/psycopg3/docs/basic/transactions.html#autocommit-transactions + connection = pg.connect(**connect_kwargs, autocommit=True) return connection except Exception: if connection: @@ -196,6 +201,7 @@ def _get_redacted_dsn(self): return self.database_url def _close_connection(self): + """Close the connection to PgBouncer""" if self.connection: try: self.connection.close() @@ -238,18 +244,31 @@ def _collect_metadata(self, db): self.set_metadata('version', pgbouncer_version) def get_version(self, db): + """ + Get the version of PgBouncer. + """ if not db: self.log.warning("Cannot get version: no active connection") return None regex = r'\d+\.\d+\.\d+' - with db.cursor(cursor_factory=pgextras.DictCursor) as cursor: - cursor.execute('SHOW VERSION;') - if db.notices: - data = db.notices[0] - else: - data = cursor.fetchone()[0] - res = re.findall(regex, data) - if res: - return res[0] - self.log.debug("Couldn't detect version from %s", data) + try: + with db.cursor() as cursor: + # This command was added in pgbouncer 1.12 + cursor.execute('SHOW VERSION;') + result = cursor.fetchone() + if result: + # Result looks like: ['PgBouncer 1.2.3 ...'] + version_string = result[0] + return re.findall(regex, version_string)[0] + else: + self.log.debug("No version found in result: %s", result) + return None + except pg.ProgrammingError as e: + # This is expected on versions of pgbouncer < 1.12 + self.log.debug("Cannot retrieve pgbouncer version using `SHOW VERSION;`: %s", e) + return None + except Exception as e: + self.log.error("An unexpected error occurred when retrieving pgbouncer version: %s", e) + return None + return None diff --git a/pgbouncer/pyproject.toml b/pgbouncer/pyproject.toml index ef7d3945dd460..1fdd0222f338c 100644 --- a/pgbouncer/pyproject.toml +++ b/pgbouncer/pyproject.toml @@ -37,7 +37,7 @@ license = "BSD-3-Clause" [project.optional-dependencies] deps = [ - "psycopg2-binary==2.9.9", + "psycopg[binary,pool]==3.2.7", ] [project.urls] diff --git a/pgbouncer/tests/conftest.py b/pgbouncer/tests/conftest.py index 2e23e87f0b2b8..709fab8ffab41 100644 --- a/pgbouncer/tests/conftest.py +++ b/pgbouncer/tests/conftest.py @@ -5,7 +5,7 @@ import os from copy import deepcopy -import psycopg2 +import psycopg import pytest from packaging import version @@ -20,8 +20,8 @@ def container_up(service_name, port): """ Try to connect to postgres/pgbouncer """ - psycopg2.connect( - host=common.HOST, port=port, user=common.USER, password=common.PASS, database=common.DB, connect_timeout=2 + psycopg.connect( + host=common.HOST, port=port, user=common.USER, password=common.PASS, dbname=common.DB, connect_timeout=2 ) diff --git a/pgbouncer/tests/test_pgbouncer_integration_e2e.py b/pgbouncer/tests/test_pgbouncer_integration_e2e.py index 18b60ca6330bc..503fda1c783a2 100644 --- a/pgbouncer/tests/test_pgbouncer_integration_e2e.py +++ b/pgbouncer/tests/test_pgbouncer_integration_e2e.py @@ -2,7 +2,7 @@ # All rights reserved # Licensed under Simplified BSD License (see LICENSE) -import psycopg2 +import psycopg import pytest from packaging import version @@ -13,51 +13,40 @@ @pytest.mark.integration @pytest.mark.usefixtures("dd_environment") -def test_check(instance, aggregator, datadog_agent, dd_run_check): - # add some stats - connection = psycopg2.connect( - host=common.HOST, - port=common.PORT, - user=common.USER, - password=common.PASS, - database=common.DB, - connect_timeout=1, - ) - connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) - cur = connection.cursor() - cur.execute('SELECT * FROM persons;') - - # run the check +def test_check(aggregator, instance, datadog_agent, dd_run_check): check = PgBouncer('pgbouncer', {}, [instance]) - check.check_id = 'test:123' dd_run_check(check) env_version = common.get_version_from_env() assert_metric_coverage(env_version, aggregator) - version_metadata = { - 'version.raw': str(env_version), - 'version.scheme': 'semver', - 'version.major': str(env_version.major), - 'version.minor': str(env_version.minor), - 'version.patch': str(env_version.micro), - } - datadog_agent.assert_metadata('test:123', version_metadata) + # SHOW VERSION; is only available on pgbouncer 1.12+ + if env_version >= version.parse('1.12.0'): + version_metadata = { + 'version.raw': str(env_version), + 'version.scheme': 'semver', + 'version.major': str(env_version.major), + 'version.minor': str(env_version.minor), + 'version.patch': str(env_version.micro), + } + datadog_agent.assert_metadata(check.check_id, version_metadata) + else: + # No version metadata expected for older versions + datadog_agent.assert_metadata(check.check_id, {}) @pytest.mark.integration @pytest.mark.usefixtures("dd_environment") def test_check_with_clients(instance, aggregator, datadog_agent, dd_run_check): # add some stats - connection = psycopg2.connect( + connection = psycopg.connect( host=common.HOST, port=common.PORT, user=common.USER, password=common.PASS, - database=common.DB, + dbname=common.DB, connect_timeout=1, ) - connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) cur = connection.cursor() cur.execute('SELECT * FROM persons;') @@ -80,15 +69,14 @@ def test_check_with_clients(instance, aggregator, datadog_agent, dd_run_check): @pytest.mark.usefixtures("dd_environment") def test_check_with_servers(instance, aggregator, datadog_agent, dd_run_check): # add some stats - connection = psycopg2.connect( + connection = psycopg.connect( host=common.HOST, port=common.PORT, user=common.USER, password=common.PASS, - database=common.DB, + dbname=common.DB, connect_timeout=1, ) - connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT) cur = connection.cursor() cur.execute('SELECT * FROM persons;') @@ -127,14 +115,17 @@ def test_check_with_url(instance_with_url, aggregator, datadog_agent, dd_run_che env_version = common.get_version_from_env() assert_metric_coverage(env_version, aggregator) - version_metadata = { - 'version.raw': str(env_version), - 'version.scheme': 'semver', - 'version.major': str(env_version.major), - 'version.minor': str(env_version.minor), - 'version.patch': str(env_version.micro), - } - datadog_agent.assert_metadata('test:123', version_metadata) + if env_version >= version.parse('1.12.0'): + version_metadata = { + 'version.raw': str(env_version), + 'version.scheme': 'semver', + 'version.major': str(env_version.major), + 'version.minor': str(env_version.minor), + 'version.patch': str(env_version.micro), + } + datadog_agent.assert_metadata(check.check_id, version_metadata) + else: + datadog_agent.assert_metadata(check.check_id, {}) @pytest.mark.e2e diff --git a/pgbouncer/tests/test_pgbouncer_unit.py b/pgbouncer/tests/test_pgbouncer_unit.py index 52ccde097e19f..33e545f7108a4 100644 --- a/pgbouncer/tests/test_pgbouncer_unit.py +++ b/pgbouncer/tests/test_pgbouncer_unit.py @@ -4,7 +4,7 @@ from unittest.mock import ANY, MagicMock, patch -import psycopg2 as pg +import psycopg as pg import pytest from datadog_checks.base import AgentCheck, ConfigurationError @@ -32,7 +32,7 @@ def test_connection_cleanup_on_error(instance, use_cached): This test ensures that connection resources are properly cleaned up when a connection fails to establish. """ instance['use_cached'] = use_cached - with patch('psycopg2.connect', side_effect=Exception("Connection failed")): + with patch('psycopg.connect', side_effect=Exception("Connection failed")): check = PgBouncer('pgbouncer', {}, [instance]) with pytest.raises(Exception): check._get_connection() @@ -52,7 +52,7 @@ def test_connection_lifecycle_with_caching(instance): mock_connection.notices = [] mock_connection.cursor.return_value.__enter__.return_value.fetchone.return_value = ['1.2.3'] - with patch('psycopg2.connect', return_value=mock_connection) as mock_connect: + with patch('psycopg.connect', return_value=mock_connection) as mock_connect: instance['use_cached'] = True check = PgBouncer('pgbouncer', {}, [instance]) @@ -75,7 +75,7 @@ def test_connection_lifecycle(instance, use_cached): mock_connection.notices = [] mock_connection.cursor.return_value.__enter__.return_value.fetchone.return_value = ['1.2.3'] instance['use_cached'] = use_cached - with patch('psycopg2.connect', return_value=mock_connection) as mock_connect: + with patch('psycopg.connect', return_value=mock_connection) as mock_connect: check = PgBouncer('pgbouncer', {}, [instance]) # Run the check @@ -94,16 +94,23 @@ def test_connection_lifecycle(instance, use_cached): @pytest.mark.unit def test_metadata_collection(instance): """ - This test verifies that metadata collection works correctly. + This test verifies that metadata collection works correctly by checking + that `set_metadata` is called with the correct version. """ mock_connection = MagicMock() - mock_connection.notices = [] - mock_connection.cursor.return_value.__enter__.return_value.fetchone.return_value = ['PgBouncer 1.2.3'] + # The version query `SHOW VERSION;` returns a full string like 'PgBouncer 1.2.3 ...' + # The `get_version` method is responsible for parsing this. + mock_connection.cursor.return_value.__enter__.return_value.fetchone.return_value = [ + 'PgBouncer 1.2.3 some other info' + ] - with patch('psycopg2.connect', return_value=mock_connection): + with patch('datadog_checks.pgbouncer.pgbouncer.pg.connect', return_value=mock_connection): check = PgBouncer('pgbouncer', {}, [instance]) + # We patch `set_metadata` on the check instance to intercept the final call with patch.object(check, 'set_metadata') as mock_set_metadata: check.check(instance) + + # Assert that the version was correctly parsed and submitted mock_set_metadata.assert_called_once_with('version', '1.2.3') @@ -120,26 +127,6 @@ def test_metadata_collection_without_connection(instance): mock_set_metadata.assert_not_called() -@pytest.mark.unit -@pytest.mark.parametrize('use_cached', [True, False]) -def test_connection_cleanup_on_isolation_level_error(instance, use_cached): - """ - This test ensures that connection resources are properly cleaned up when setting the isolation level fails. - """ - instance['use_cached'] = use_cached - mock_connection = MagicMock() - mock_connection.set_isolation_level.side_effect = Exception("Failed to set isolation level") - - with patch('psycopg2.connect', return_value=mock_connection): - check = PgBouncer('pgbouncer', {}, [instance]) - with pytest.raises(Exception): - check.check(instance) - - # Verify connection was closed and not stored - assert mock_connection.close.call_count == 1 - assert check.connection is None - - @pytest.mark.unit @pytest.mark.parametrize('use_cached', [True, False]) def test_connection_lifecycle_pg_error_once(instance, use_cached): @@ -163,7 +150,7 @@ def test_connection_lifecycle_pg_error_once(instance, use_cached): mock_conn2.cursor.return_value = fake_cursor # pg.connect will return mock_conn1 first, then mock_conn2. - with patch('psycopg2.connect', side_effect=[mock_conn1, mock_conn2]) as mock_connect: + with patch('psycopg.connect', side_effect=[mock_conn1, mock_conn2]) as mock_connect: check = PgBouncer('pgbouncer', {}, [instance]) # Patch _collect_metadata to avoid its side effects with patch.object(check, '_collect_metadata', return_value=None): @@ -203,7 +190,7 @@ def test_connection_lifecycle_pg_error_twice(instance, use_cached): mock_conn2 = MagicMock() mock_conn2.cursor.side_effect = pg.Error("Simulated pg.Error on second connection") - with patch('psycopg2.connect', side_effect=[mock_conn1, mock_conn2]) as mock_connect: + with patch('psycopg.connect', side_effect=[mock_conn1, mock_conn2]) as mock_connect: check = PgBouncer('pgbouncer', {}, [instance]) with patch.object(check, '_collect_metadata', return_value=None): with patch.object(check, 'service_check') as service_check_patch: @@ -235,7 +222,7 @@ def test_no_new_connection_when_cached_exists(instance): mock_existing_connection.notices = [] mock_existing_connection.cursor.return_value.__enter__.return_value.fetchone.return_value = ['1.2.3'] - with patch('psycopg2.connect') as mock_connect: + with patch('psycopg.connect') as mock_connect: check = PgBouncer('pgbouncer', {}, [instance]) # Set the existing connection diff --git a/postgres/CHANGELOG.md b/postgres/CHANGELOG.md index d49ae06c42117..fcbfdb9576824 100644 --- a/postgres/CHANGELOG.md +++ b/postgres/CHANGELOG.md @@ -34,7 +34,7 @@ * PG: Add recovery prefetch metrics ([#20464](https://github.com/DataDog/integrations-core/pull/20464)) * Update dependencies ([#20561](https://github.com/DataDog/integrations-core/pull/20561)) -* PG: Add wait events counts from `pg_stat_activity` ([#20588](https://github.com/DataDog/integrations-core/pull/20588)) +* PG: Add wait events counts from pg_stat_activity ([#20588](https://github.com/DataDog/integrations-core/pull/20588)) * Upgrade psycopg to version 3 for Postgres integration ([#20617](https://github.com/DataDog/integrations-core/pull/20617)) ***Fixed***: diff --git a/postgres/changelog.d/21173.added b/postgres/changelog.d/21173.added new file mode 100644 index 0000000000000..ea06e9e0efd6b --- /dev/null +++ b/postgres/changelog.d/21173.added @@ -0,0 +1 @@ +Upgrade to psycopg3 diff --git a/postgres/datadog_checks/postgres/connection_pool.py b/postgres/datadog_checks/postgres/connection_pool.py new file mode 100644 index 0000000000000..c3cac770c65dc --- /dev/null +++ b/postgres/datadog_checks/postgres/connection_pool.py @@ -0,0 +1,247 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import threading +import time +from collections import OrderedDict +from dataclasses import dataclass +from typing import Any, Dict, Optional, Tuple, Union + +from psycopg import Connection +from psycopg_pool import ConnectionPool + +from .cursor import CommenterCursor, SQLASCIITextLoader + + +@dataclass(frozen=True) +class PostgresConnectionArgs: + """ + Immutable PostgreSQL connection arguments. + """ + + application_name: str + user: str + host: Optional[str] = None + port: Optional[int] = None + password: Optional[str] = None + ssl_mode: Optional[str] = "allow" + ssl_cert: Optional[str] = None + ssl_root_cert: Optional[str] = None + ssl_key: Optional[str] = None + ssl_password: Optional[str] = None + + def as_kwargs(self, dbname: str) -> Dict[str, Union[str, int]]: + """ + Return a dictionary of connection arguments for psycopg. + + Args: + dbname (str): The database name to connect to. + + Returns: + Dict[str, Union[str, int]]: Connection arguments dictionary with string and integer values. + """ + kwargs = { + "application_name": self.application_name, + "user": self.user, + "dbname": dbname, + "sslmode": self.ssl_mode, + } + if self.host: + kwargs["host"] = self.host + if self.password: + kwargs["password"] = self.password + if self.port: + kwargs["port"] = self.port + if self.ssl_cert: + kwargs["sslcert"] = self.ssl_cert + if self.ssl_root_cert: + kwargs["sslrootcert"] = self.ssl_root_cert + if self.ssl_key: + kwargs["sslkey"] = self.ssl_key + if self.ssl_password: + kwargs["sslpassword"] = self.ssl_password + return kwargs + + +class LRUConnectionPoolManager: + """ + Manages a fixed-size set of psycopg3 ConnectionPools, one per database name (dbname), + evicting the least recently used pool when the limit is exceeded. + + Each dbname is assigned its own psycopg_pool.ConnectionPool instance. Only one pool + is maintained per dbname, and the total number of active pools is capped by `max_db`. + + Pools are reused across calls, and usage is tracked to enforce LRU eviction. When a + new dbname is accessed beyond the pool limit, the least recently used pool is closed + and removed to make room. + + Optionally supports runtime inspection of pool stats and last-used times. + """ + + def __init__( + self, + max_db: int, + base_conn_args: PostgresConnectionArgs, + pool_config: Optional[Dict[str, Any]] = None, + statement_timeout: Optional[int] = None, # milliseconds + sqlascii_encodings: Optional[list[str]] = None, + ) -> None: + """ + Initialize the pool manager. + + Args: + max_db (int): Maximum number of unique dbname pools to maintain. + base_conn_args (PostgresConnectionArgs): Common connection parameters. + pool_config (dict, optional): Additional ConnectionPool settings (min_size, max_size, etc). + statement_timeout (int, optional): Statement timeout in milliseconds. + sqlascii_encodings (list[str], optional): List of encodings to handle for SQLASCII text. + """ + self.max_db = max_db + self.base_conn_args = base_conn_args + self.statement_timeout = statement_timeout + self.sqlascii_encodings = sqlascii_encodings + + self.pool_config = { + **(pool_config or {}), + "min_size": 0, + "max_size": 2, + "open": True, + } + self.lock = threading.Lock() + self.pools: OrderedDict[str, Tuple[ConnectionPool, float, bool]] = OrderedDict() + self._closed = False + + def _configure_connection(self, conn: Connection) -> None: + conn.autocommit = True + + if conn.info.encoding.lower() in ['ascii', 'sqlascii', 'sql_ascii']: + text_loader = SQLASCIITextLoader + text_loader.encodings = self.sqlascii_encodings + for typ in ["text", "varchar", "name", "regclass"]: + conn.adapters.register_loader(typ, text_loader) + + conn.cursor_factory = CommenterCursor + + with conn.cursor() as cur: + if self.statement_timeout is not None: + cur.execute("SET statement_timeout = %s", (self.statement_timeout,)) + + def _create_pool(self, dbname: str) -> ConnectionPool: + """ + Create a new ConnectionPool for a given dbname using a kwargs. + + Args: + dbname (str): The target database name. + + Returns: + ConnectionPool: A new pool instance configured for the dbname. + """ + kwargs = self.base_conn_args.as_kwargs(dbname=dbname) + + return ConnectionPool(kwargs=kwargs, configure=self._configure_connection, **self.pool_config) + + def get_connection(self, dbname: str, persistent: bool = False): + """ + Context-managed access to a single connection from the pool associated with the given dbname. + + Ensures that the connection is returned to its pool after use. Returns the context manager + from psycopg_pool.ConnectionPool.connection(). + + Usage: + with manager.get_connection("mydb") as conn: + with conn.cursor() as cur: + cur.execute(...) + + Args: + dbname (str): The database name to get a connection for. + persistent (bool): Whether the underlying pool should be marked as persistent. + + Returns: + Context manager yielding a psycopg.Connection. + + Raises: + RuntimeError: If the pool manager has been closed. + """ + with self.lock: + if self._closed: + raise RuntimeError("Pool manager is closed and cannot get connection") + + now = time.monotonic() + + # Get or create pool + if dbname in self.pools: + pool, _, was_persistent = self.pools.pop(dbname) + self.pools[dbname] = (pool, now, was_persistent or persistent) + else: + # Create new pool, potentially evicting old ones + if len(self.pools) >= self.max_db: + # Try to evict a non-persistent pool first + for evict_dbname in list(self.pools.keys()): + _, _, is_persistent = self.pools[evict_dbname] + if not is_persistent: + old_pool, _, _ = self.pools.pop(evict_dbname) + old_pool.close() + break + else: + # All remaining are persistent, evict true LRU + evict_dbname, (old_pool, _, _) = self.pools.popitem(last=False) + old_pool.close() + + pool = self._create_pool(dbname) + self.pools[dbname] = (pool, now, persistent) + + # Return the pool's context manager directly + return pool.connection() + + def get_pool_stats(self, dbname: str) -> Optional[Dict[str, Any]]: + """ + Retrieve runtime statistics and metadata for the ConnectionPool associated with the given dbname. + + Includes: + - used: Connections currently in use + - available: Idle connections ready for use + - total: Total connections managed by the pool + - waiters: Threads waiting for a connection + - last_used: Monotonic timestamp of last access + - persistent: Whether the pool is marked as persistent + + Args: + dbname (str): The database name to fetch stats for. + + Returns: + Optional[Dict[str, Any]]: Dictionary of pool stats and metadata, or None if the pool does not exist. + """ + with self.lock: + entry = self.pools.get(dbname) + if not entry: + return None + pool, last_used, persistent = entry + stats = pool.get_stats() + return { + **stats, + "last_used": last_used, + "persistent": persistent, + } + + def close_all(self) -> None: + """ + Gracefully close all active pools including persistent ones and release all underlying connections. + + This clears the pool manager state and prevents new pools from being created. + Should be called during shutdown or cleanup. + """ + with self.lock: + self._closed = True + for pool, _, _ in self.pools.values(): + pool.close() + self.pools.clear() + + def is_closed(self) -> bool: + """ + Check if the pool manager has been closed. + + Returns: + bool: True if the pool manager is closed, False otherwise. + """ + return self._closed diff --git a/postgres/datadog_checks/postgres/connections.py b/postgres/datadog_checks/postgres/connections.py deleted file mode 100644 index 5c78e5d716042..0000000000000 --- a/postgres/datadog_checks/postgres/connections.py +++ /dev/null @@ -1,213 +0,0 @@ -# (C) Datadog, Inc. 2023-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -import contextlib -import datetime -import inspect -import threading -import time -from typing import Callable, Dict - -import psycopg2 - - -class ConnectionPoolFullError(Exception): - def __init__(self, size, timeout): - self.size = size - self.timeout = timeout - - def __str__(self): - return "Could not insert connection in pool size {} within {} seconds".format(self.size, self.timeout) - - -class ConnectionInfo: - def __init__( - self, - connection: psycopg2.extensions.connection, - deadline: int, - active: bool, - last_accessed: int, - thread: threading.Thread, - persistent: bool, - ): - self.connection = connection - self.deadline = deadline - self.active = active - self.last_accessed = last_accessed - self.thread = thread - self.persistent = persistent - - -class MultiDatabaseConnectionPool(object): - """ - Manages a connection pool across many logical databases with a maximum of 1 conn per - database. Traditional connection pools manage a set of connections to a single database, - however the usage patterns of the Agent application should aim to have minimal footprint - and reuse a single connection as much as possible. - - Even when limited to a single connection per database, an instance with hundreds of - databases still present a connection overhead risk. This class provides a mechanism - to prune connections to a database which were not used in the time specified by their - TTL. - - If max_conns is specified, the connection pool will limit concurrent connections. - """ - - class Stats(object): - def __init__(self): - self.connection_opened = 0 - self.connection_pruned = 0 - self.connection_closed = 0 - self.connection_closed_failed = 0 - - def __repr__(self): - return str(self.__dict__) - - def reset(self): - self.__init__() - - def __init__(self, connect_fn: Callable[[str], None], max_conns: int = None): - self.max_conns: int = max_conns - self._stats = self.Stats() - self._mu = threading.RLock() - self._conns: Dict[str, ConnectionInfo] = {} - - if hasattr(inspect, 'signature'): - connect_sig = inspect.signature(connect_fn) - if len(connect_sig.parameters) != 1: - raise ValueError( - "Invalid signature for the connection function. " - "A single parameter for dbname is expected, got signature: {}".format(connect_sig) - ) - self.connect_fn = connect_fn - - def _get_connection_raw( - self, - dbname: str, - ttl_ms: int, - timeout: int = None, - startup_fn: Callable[[psycopg2.extensions.connection], None] = None, - persistent: bool = False, - ) -> psycopg2.extensions.connection: - """ - Return a connection from the pool. - Pass a function to startup_func if there is an action needed with the connection - when re-establishing it. - """ - start = datetime.datetime.now() - self.prune_connections() - with self._mu: - conn = self._conns.pop(dbname, None) - db = conn.connection if conn else None - if db is None or db.closed: - if self.max_conns is not None: - # try to free space until we succeed - while len(self._conns) >= self.max_conns: - self.prune_connections() - self.evict_lru() - if timeout is not None and (datetime.datetime.now() - start).total_seconds() > timeout: - raise ConnectionPoolFullError(self.max_conns, timeout) - time.sleep(0.01) - continue - self._stats.connection_opened += 1 - db = self.connect_fn(dbname) - if startup_fn: - startup_fn(db) - else: - # if already in pool, retain persistence status - persistent = conn.persistent - - if db.status != psycopg2.extensions.STATUS_READY: - # Some transaction went wrong and the connection is in an unhealthy state. Let's fix that - db.rollback() - - deadline = datetime.datetime.now() + datetime.timedelta(milliseconds=ttl_ms) - self._conns[dbname] = ConnectionInfo( - connection=db, - deadline=deadline, - active=True, - last_accessed=datetime.datetime.now(), - thread=threading.current_thread(), - persistent=persistent, - ) - return db - - @contextlib.contextmanager - def get_connection( - self, - dbname: str, - ttl_ms: int, - timeout: int = None, - startup_fn: Callable[[psycopg2.extensions.connection], None] = None, - persistent: bool = False, - ): - """ - Grab a connection from the pool if the database is already connected. - If max_conns is specified, and the database isn't already connected, - make a new connection if the max_conn limit hasn't been reached. - Blocks until a connection can be added to the pool, - and optionally takes a timeout in seconds. - Note that leaving a connection context here does NOT close the connection in psycopg2; - connections must be manually closed by `close_all_connections()`. - """ - try: - with self._mu: - db = self._get_connection_raw(dbname, ttl_ms, timeout, startup_fn, persistent) - yield db - finally: - with self._mu: - try: - self._conns[dbname].active = False - except KeyError: - # if self._get_connection_raw hit an exception, self._conns[dbname] didn't get populated - pass - - def prune_connections(self): - """ - This function should be called periodically to prune all connections which have not been - accessed since their TTL. This means that connections which are actually active on the - server can still be closed with this function. For instance, if a connection is opened with - ttl 1000ms, but the query it's running takes 5000ms, this function will still try to close - the connection mid-query. - """ - with self._mu: - now = datetime.datetime.now() - for dbname, conn in list(self._conns.items()): - if conn.deadline < now: - self._stats.connection_pruned += 1 - self._terminate_connection_unsafe(dbname) - - def close_all_connections(self): - success = True - with self._mu: - while self._conns: - dbname = next(iter(self._conns)) - if not self._terminate_connection_unsafe(dbname): - success = False - return success - - def evict_lru(self) -> str: - """ - Evict and close the inactive connection which was least recently used. - Return the dbname connection that was evicted or None if we couldn't evict a connection. - """ - with self._mu: - sorted_conns = sorted(self._conns.items(), key=lambda i: i[1].last_accessed) - for name, conn_info in sorted_conns: - if not conn_info.active and not conn_info.persistent: - self._terminate_connection_unsafe(name) - return name - - # Could not evict a candidate; return None - return None - - def _terminate_connection_unsafe(self, dbname: str): - db = self._conns.pop(dbname, ConnectionInfo(None, None, None, None, None, None)).connection - if db is not None: - try: - self._stats.connection_closed += 1 - db.close() - except Exception: - self._stats.connection_closed_failed += 1 - return False - return True diff --git a/postgres/datadog_checks/postgres/cursor.py b/postgres/datadog_checks/postgres/cursor.py index a122790231b6d..5bff49a1c085e 100644 --- a/postgres/datadog_checks/postgres/cursor.py +++ b/postgres/datadog_checks/postgres/cursor.py @@ -2,10 +2,10 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import psycopg2.extensions -import psycopg2.extras +import psycopg from datadog_checks.base.utils.db.sql_commenter import add_sql_comment +from datadog_checks.postgres.encoding import decode_with_encodings DD_QUERY_ATTRIBUTES = { 'service': 'datadog-agent', @@ -17,7 +17,7 @@ def __init__(self, *args, **kwargs): self.__attributes = DD_QUERY_ATTRIBUTES super().__init__(*args, **kwargs) - def execute(self, query, vars=None, ignore_query_metric=False): + def execute(self, query, params=None, ignore_query_metric=False, binary=False, prepare=None): ''' When ignore is True, a /* DDIGNORE */ comment will be added to the query. This comment indicates that the query should be ignored in query metrics. @@ -25,12 +25,31 @@ def execute(self, query, vars=None, ignore_query_metric=False): query = add_sql_comment(query, prepand=True, **self.__attributes) if ignore_query_metric: query = '{} {}'.format('/* DDIGNORE */', query) - return super().execute(query, vars) + return super().execute(query, params, binary=binary, prepare=prepare) -class CommenterCursor(BaseCommenterCursor, psycopg2.extensions.cursor): +class CommenterCursor(BaseCommenterCursor, psycopg.ClientCursor): pass -class CommenterDictCursor(BaseCommenterCursor, psycopg2.extras.DictCursor): - pass +class SQLASCIITextLoader(psycopg.adapt.Loader): + """ + Custom loader for SQLASCII encoding. + """ + + encodings = ['utf-8'] + format = psycopg.pq.Format.TEXT + + def load(self, data): + if data is None: + return data + if isinstance(data, memoryview): + # Convert memoryview to bytes + data = data.tobytes() + if not isinstance(data, bytes): + return data + try: + return decode_with_encodings(data, self.encodings) + except Exception: + # Fallback to utf8 with replacement + return data.decode('utf-8', errors='backslashreplace') diff --git a/postgres/datadog_checks/postgres/discovery.py b/postgres/datadog_checks/postgres/discovery.py index 89434dea3eee5..0d0b4d60ff815 100644 --- a/postgres/datadog_checks/postgres/discovery.py +++ b/postgres/datadog_checks/postgres/discovery.py @@ -6,7 +6,6 @@ from datadog_checks.base import AgentCheck from datadog_checks.base.utils.discovery import Discovery -from datadog_checks.postgres.cursor import CommenterCursor from datadog_checks.postgres.util import DatabaseConfigurationError, warning_with_tags AUTODISCOVERY_QUERY: str = """select datname from pg_catalog.pg_database where datistemplate = false;""" @@ -71,8 +70,8 @@ def get_items(self) -> List[str]: return items_parsed def _get_databases(self) -> List[str]: - with self.db_pool.get_connection(self._db, self._default_ttl) as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + with self.db_pool.get_connection(self._db) as conn: + with conn.cursor() as cursor: cursor.execute(AUTODISCOVERY_QUERY) databases = list(cursor.fetchall()) databases = [ diff --git a/postgres/datadog_checks/postgres/explain_parameterized_queries.py b/postgres/datadog_checks/postgres/explain_parameterized_queries.py index 0eea44d762aca..48767f629fd61 100644 --- a/postgres/datadog_checks/postgres/explain_parameterized_queries.py +++ b/postgres/datadog_checks/postgres/explain_parameterized_queries.py @@ -5,10 +5,9 @@ import logging import re -import psycopg2 +import psycopg from datadog_checks.base.utils.tracking import tracked_method -from datadog_checks.postgres.cursor import CommenterDictCursor from .util import DBExplainError from .version_utils import V12 @@ -76,50 +75,50 @@ def explain_statement(self, dbname, statement, obfuscated_statement, query_signa if self._check.version < V12: # if pg version < 12, skip explaining parameterized queries because # plan_cache_mode is not supported - e = psycopg2.errors.UndefinedParameter("Unable to explain parameterized query") + e = psycopg.errors.UndefinedParameter("Unable to explain parameterized query") logger.debug( "Unable to explain parameterized query. Postgres version %s does not support plan_cache_mode", self._check.version, ) return None, DBExplainError.parameterized_query, '{}'.format(type(e)) - self._set_plan_cache_mode(dbname) - - try: - self._create_prepared_statement(dbname, statement, obfuscated_statement, query_signature) - except psycopg2.errors.IndeterminateDatatype as e: - return None, DBExplainError.indeterminate_datatype, '{}'.format(type(e)) - except psycopg2.errors.UndefinedFunction as e: - return None, DBExplainError.undefined_function, '{}'.format(type(e)) - except Exception as e: - # if we fail to create a prepared statement, we cannot explain the query - return None, DBExplainError.failed_to_explain_with_prepared_statement, '{}'.format(type(e)) - - try: - result = self._explain_prepared_statement(dbname, statement, obfuscated_statement, query_signature) - if result: - plan = result[0][0][0] - return plan, DBExplainError.explained_with_prepared_statement, None - else: - # the explain function was executed but no plan was returned - logger.debug( - "Unable to explain parameterized query. " - "The explain function %s was executed but no plan was returned", - self._explain_function, - ) - return None, DBExplainError.no_plan_returned_with_prepared_statement, None - except Exception as e: - return None, DBExplainError.failed_to_explain_with_prepared_statement, '{}'.format(type(e)) - finally: - self._deallocate_prepared_statement(dbname, query_signature) - - def _set_plan_cache_mode(self, dbname): - self._execute_query(dbname, "SET plan_cache_mode = force_generic_plan") + with self._check.db_pool.get_connection(dbname) as conn: + try: + self._set_plan_cache_mode(conn) + self._create_prepared_statement(conn, statement, obfuscated_statement, query_signature) + except psycopg.errors.IndeterminateDatatype as e: + return None, DBExplainError.indeterminate_datatype, '{}'.format(type(e)) + except psycopg.errors.UndefinedFunction as e: + return None, DBExplainError.undefined_function, '{}'.format(type(e)) + except Exception as e: + # if we fail to create a prepared statement, we cannot explain the query + return None, DBExplainError.failed_to_explain_with_prepared_statement, '{}'.format(type(e)) + + try: + result = self._explain_prepared_statement(conn, statement, obfuscated_statement, query_signature) + if result: + plan = result[0][0][0] + return plan, DBExplainError.explained_with_prepared_statement, None + else: + # the explain function was executed but no plan was returned + logger.debug( + "Unable to explain parameterized query. " + "The explain function %s was executed but no plan was returned", + self._explain_function, + ) + return None, DBExplainError.no_plan_returned_with_prepared_statement, None + except Exception as e: + return None, DBExplainError.failed_to_explain_with_prepared_statement, '{}'.format(type(e)) + finally: + self._deallocate_prepared_statement(conn, query_signature) + + def _set_plan_cache_mode(self, conn): + self._execute_query(conn, "SET plan_cache_mode = force_generic_plan") @tracked_method(agent_check_getter=agent_check_getter) - def _create_prepared_statement(self, dbname, statement, obfuscated_statement, query_signature): + def _create_prepared_statement(self, conn, statement, obfuscated_statement, query_signature): try: self._execute_query( - dbname, + conn, PREPARE_STATEMENT_QUERY.format(query_signature=query_signature, statement=statement), ) except Exception as e: @@ -135,16 +134,14 @@ def _create_prepared_statement(self, dbname, statement, obfuscated_statement, qu raise @tracked_method(agent_check_getter=agent_check_getter) - def _get_number_of_parameters_for_prepared_statement(self, dbname, query_signature): - rows = self._execute_query_and_fetch_rows( - dbname, PARAM_TYPES_COUNT_QUERY.format(query_signature=query_signature) - ) + def _get_number_of_parameters_for_prepared_statement(self, conn, query_signature): + rows = self._execute_query_and_fetch_rows(conn, PARAM_TYPES_COUNT_QUERY.format(query_signature=query_signature)) return rows[0][0] if rows else 0 @tracked_method(agent_check_getter=agent_check_getter) - def _generate_prepared_statement_query(self, dbname: str, query_signature: str) -> str: + def _generate_prepared_statement_query(self, conn, query_signature: str) -> str: parameters = "" - num_params = self._get_number_of_parameters_for_prepared_statement(dbname, query_signature) + num_params = self._get_number_of_parameters_for_prepared_statement(conn, query_signature) if num_params > 0: null_parameters = ','.join('null' for _ in range(num_params)) @@ -153,11 +150,11 @@ def _generate_prepared_statement_query(self, dbname: str, query_signature: str) return EXECUTE_PREPARED_STATEMENT_QUERY.format(prepared_statement=query_signature, parameters=parameters) @tracked_method(agent_check_getter=agent_check_getter) - def _explain_prepared_statement(self, dbname, statement, obfuscated_statement, query_signature): - prepared_statement_query = self._generate_prepared_statement_query(dbname, query_signature) + def _explain_prepared_statement(self, conn, statement, obfuscated_statement, query_signature): try: + prepared_statement_query = self._generate_prepared_statement_query(conn, query_signature) return self._execute_query_and_fetch_rows( - dbname, + conn, EXPLAIN_QUERY.format( explain_function=self._explain_function, statement=prepared_statement_query, @@ -175,11 +172,9 @@ def _explain_prepared_statement(self, dbname, statement, obfuscated_statement, q ) raise - def _deallocate_prepared_statement(self, dbname, query_signature): + def _deallocate_prepared_statement(self, conn, query_signature): try: - self._execute_query( - dbname, "DEALLOCATE PREPARE dd_{query_signature}".format(query_signature=query_signature) - ) + self._execute_query(conn, "DEALLOCATE PREPARE dd_{query_signature}".format(query_signature=query_signature)) except Exception as e: logger.debug( 'Failed to deallocate prepared statement query_signature=[%s] | err=[%s]', @@ -187,19 +182,15 @@ def _deallocate_prepared_statement(self, dbname, query_signature): e, ) - def _execute_query(self, dbname, query): - # Psycopg2 connections do not get closed when context ends; - # leaving context will just mark the connection as inactive in MultiDatabaseConnectionPool - with self._check.db_pool.get_connection(dbname, self._check._config.idle_connection_timeout) as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: - logger.debug('Executing query=[%s]', query) - cursor.execute(query, ignore_query_metric=True) - - def _execute_query_and_fetch_rows(self, dbname, query): - with self._check.db_pool.get_connection(dbname, self._check._config.idle_connection_timeout) as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: - cursor.execute(query, ignore_query_metric=True) - return cursor.fetchall() + def _execute_query(self, conn, query): + with conn.cursor() as cursor: + logger.debug('Executing query=[%s]', query) + cursor.execute(query, ignore_query_metric=True) + + def _execute_query_and_fetch_rows(self, conn, query): + with conn.cursor() as cursor: + cursor.execute(query, ignore_query_metric=True) + return cursor.fetchall() def _is_parameterized_query(self, statement: str) -> bool: # Use regex to match $1 to determine if a query is parameterized diff --git a/postgres/datadog_checks/postgres/metadata.py b/postgres/datadog_checks/postgres/metadata.py index 441ea948d9363..b18cb3ae38cc5 100644 --- a/postgres/datadog_checks/postgres/metadata.py +++ b/postgres/datadog_checks/postgres/metadata.py @@ -6,9 +6,8 @@ import time from typing import Dict, List, Optional, Tuple, Union # noqa: F401 -import psycopg2 - -from datadog_checks.postgres.cursor import CommenterDictCursor +import psycopg +from psycopg.rows import dict_row try: import datadog_agent @@ -61,7 +60,8 @@ """ PG_EXTENSIONS_QUERY = """ -SELECT extname, nspname schemaname FROM pg_extension left join pg_namespace on extnamespace = pg_namespace.oid; +SELECT extname, nspname schemaname +FROM pg_extension left join pg_namespace on extnamespace = pg_namespace.oid; """ PG_EXTENSION_LOADER_QUERY = { @@ -224,7 +224,7 @@ class PostgresMetadata(DBMAsyncJob): 2. collection of pg_settings """ - def __init__(self, check, config, shutdown_callback): + def __init__(self, check, config): self.pg_settings_ignored_patterns = config.settings_metadata_config.get( "ignored_settings_patterns", DEFAULT_SETTINGS_IGNORED_PATTERNS ) @@ -256,9 +256,8 @@ def __init__(self, check, config, shutdown_callback): enabled=is_affirmative(config.resources_metadata_config.get("enabled", True)), dbms="postgres", min_collection_interval=config.min_collection_interval, - expected_db_exceptions=(psycopg2.errors.DatabaseError,), + expected_db_exceptions=(psycopg.errors.DatabaseError,), job_name="database-metadata", - shutdown_callback=shutdown_callback, ) self._check = check self._config = config @@ -293,7 +292,6 @@ def run_job(self): self._tags_no_db = [t for t in self.tags if not t.startswith("db:")] self.report_postgres_metadata() self.report_postgres_extensions() - self._check.db_pool.prune_connections() @tracked_method(agent_check_getter=agent_check_getter) def report_postgres_extensions(self): @@ -318,8 +316,10 @@ def report_postgres_extensions(self): @tracked_method(agent_check_getter=agent_check_getter) def _collect_postgres_extensions(self): + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: + with conn.cursor(row_factory=dict_row) as cursor: self._time_since_last_extension_query = time.time() # Get loaded extensions @@ -393,8 +393,8 @@ def _collect_postgres_schemas(self): if not self._should_collect_metadata(dbname, "database"): continue - with self.db_pool.get_connection(dbname, self._config.idle_connection_timeout) as conn: - with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor: + with self.db_pool.get_connection(dbname) as conn: + with conn.cursor(row_factory=dict_row) as cursor: for schema in database["schemas"]: if not self._should_collect_metadata(schema["name"], "schema"): continue @@ -413,7 +413,7 @@ def _collect_postgres_schemas(self): tables_buffer = [] for tables in table_chunks: - table_info = self._query_table_information(cursor, schema['name'], tables) + table_info = self._query_table_information(cursor, dbname, tables) tables_buffer = [*tables_buffer, *table_info] for t in table_info: @@ -510,9 +510,7 @@ def _collect_schema_info(self): self._last_schemas_query_time = time.time() return metadata - def _query_database_information( - self, cursor: psycopg2.extensions.cursor, dbname: str - ) -> Dict[str, Union[str, int]]: + def _query_database_information(self, cursor: psycopg.Cursor, dbname: str) -> Dict[str, Union[str, int]]: """ Collect database info. Returns description: str @@ -521,11 +519,13 @@ def _query_database_information( encoding: str owner: str """ + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") cursor.execute(DATABASE_INFORMATION_QUERY.format(dbname=dbname)) row = cursor.fetchone() return row - def _query_schema_information(self, cursor: psycopg2.extensions.cursor, dbname: str) -> Dict[str, str]: + def _query_schema_information(self, cursor: psycopg.Cursor, dbname: str) -> Dict[str, str]: """ Collect user schemas. Returns id: str @@ -542,6 +542,8 @@ def _query_schema_information(self, cursor: psycopg2.extensions.cursor, dbname: else: schema_query_ = schema_query_.format("") + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") cursor.execute(schema_query_) rows = cursor.fetchall() schemas = [] @@ -551,7 +553,7 @@ def _query_schema_information(self, cursor: psycopg2.extensions.cursor, dbname: self._log.debug("Schemas found for database '{database}': [{schemas}]".format(database=dbname, schemas=rows)) return schemas - def _get_table_info(self, cursor, dbname, schema_id): + def _get_table_info(self, cursor: psycopg.Cursor, dbname, schema_id): """ Tables will be sorted by the number of total accesses (index_rel_scans + seq_scans) and truncated to the max_tables limit. @@ -563,6 +565,7 @@ def _get_table_info(self, cursor, dbname, schema_id): cursor.execute(PG_TABLES_QUERY_V9.format(schema_oid=schema_id, filter=filter)) else: cursor.execute(PG_TABLES_QUERY_V10_PLUS.format(schema_oid=schema_id, filter=filter)) + rows = cursor.fetchall() table_info = [dict(row) for row in rows] @@ -613,7 +616,7 @@ def _get_tables_filter(self): return sql def _sort_and_limit_table_info( - self, cursor, dbname, table_info: List[Dict[str, Union[str, bool]]], limit: int + self, cursor: psycopg.Cursor, dbname, table_info: List[Dict[str, Union[str, bool]]], limit: int ) -> List[Dict[str, Union[str, bool]]]: def sort_tables(info): cache = self._check.metrics_cache.table_activity_metrics @@ -632,6 +635,8 @@ def sort_tables(info): return table_data.get("index_scans", 0) + table_data.get("seq_scans", 0) else: # get activity + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") cursor.execute(PARTITION_ACTIVITY_QUERY.format(parent_oid=info["id"])) row = cursor.fetchone() return row.get("total_activity", 0) if row is not None else 0 @@ -641,7 +646,7 @@ def sort_tables(info): return table_info[:limit] def _query_tables_for_schema( - self, cursor: psycopg2.extensions.cursor, schema_id: str, dbname: str + self, cursor: psycopg.Cursor, schema_id: str, dbname: str ) -> List[Dict[str, Union[str, Dict]]]: """ Collect list of tables for a schema. Returns a list of dictionaries @@ -672,7 +677,7 @@ def _query_tables_for_schema( return table_payloads def _query_table_information( - self, cursor: psycopg2.extensions.cursor, schema_name: str, table_info: List[Dict[str, Union[str, bool]]] + self, cursor: psycopg.Cursor, dbname: str, table_info: List[Dict[str, Union[str, bool]]] ) -> List[Dict[str, Union[str, Dict]]]: """ Collect table information . Returns a dictionary @@ -710,7 +715,10 @@ def _query_table_information( table_ids = ",".join(["'{}'".format(t.get("id")) for t in table_info]) # Get indexes + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") cursor.execute(PG_INDEXES_QUERY.format(table_ids=table_ids)) + rows = cursor.fetchall() for row in rows: # Partition indexes in some versions of Postgres have appended digits for each partition @@ -751,8 +759,8 @@ def _query_table_information( def _collect_metadata_for_database(self, dbname): metadata = {} - with self.db_pool.get_connection(dbname, self._config.idle_connection_timeout) as conn: - with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor: + with self.db_pool.get_connection(dbname) as conn: + with conn.cursor(row_factory=dict_row) as cursor: database_info = self._query_database_information(cursor, dbname) metadata.update( { @@ -772,8 +780,10 @@ def _collect_metadata_for_database(self, dbname): @tracked_method(agent_check_getter=agent_check_getter) def _collect_postgres_settings(self): + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: + with conn.cursor(row_factory=dict_row) as cursor: # Get loaded extensions cursor.execute(PG_EXTENSIONS_QUERY) rows = cursor.fetchall() @@ -785,7 +795,9 @@ def _collect_postgres_settings(self): query = PG_EXTENSION_LOADER_QUERY[extension] + "\n" + query else: self._log.warning( - "unable to collect settings for extension %s in schema %s", extension, row['schemaname'] + "unable to collect settings for extension %s in schema %s", + extension, + row['schemaname'], ) else: self._log.warning("unable to collect settings for unknown extension %s", extension) @@ -800,6 +812,14 @@ def _collect_postgres_settings(self): ) self._time_since_last_settings_query = time.time() cursor.execute(query, (self.pg_settings_ignored_patterns,)) - rows = cursor.fetchall() + # pg3 returns a set of results for each statement in the multiple statement query + # We want to retrieve the last one that actually has the settings results + rows = [] + has_more_results = True + while has_more_results: + if cursor.pgresult.status == psycopg.pq.ExecStatus.TUPLES_OK: + rows = cursor.fetchall() + has_more_results = cursor.nextset() + self._log.debug("Loaded %s rows from pg_settings", rows) self._log.debug("Loaded %s rows from pg_settings", len(rows)) - return [dict(row) for row in rows] + return rows diff --git a/postgres/datadog_checks/postgres/postgres.py b/postgres/datadog_checks/postgres/postgres.py index b6cec1521823b..eb1df80368ad1 100644 --- a/postgres/datadog_checks/postgres/postgres.py +++ b/postgres/datadog_checks/postgres/postgres.py @@ -8,7 +8,7 @@ from string import Template from time import time -import psycopg2 +import psycopg from cachetools import TTLCache from datadog_checks.base import AgentCheck @@ -21,8 +21,7 @@ from datadog_checks.base.utils.db.utils import resolve_db_host as agent_host_resolver from datadog_checks.base.utils.serialization import json from datadog_checks.postgres import aws, azure -from datadog_checks.postgres.connections import MultiDatabaseConnectionPool -from datadog_checks.postgres.cursor import CommenterCursor, CommenterDictCursor +from datadog_checks.postgres.connection_pool import LRUConnectionPoolManager, PostgresConnectionArgs from datadog_checks.postgres.discovery import PostgresAutodiscovery from datadog_checks.postgres.metadata import PostgresMetadata from datadog_checks.postgres.metrics_cache import PostgresMetricsCache @@ -136,11 +135,16 @@ def __init__(self, name, init_config, instances): self.set_resource_tags() self.pg_settings = {} self._warnings_by_code = {} - self.db_pool = MultiDatabaseConnectionPool(self._new_connection, self._config.max_connections) + self.db_pool = LRUConnectionPoolManager( + max_db=self._config.max_connections, + base_conn_args=self.build_connection_args(), + statement_timeout=self._config.query_timeout, + sqlascii_encodings=self._config.query_encodings, + ) self.metrics_cache = PostgresMetricsCache(self._config) - self.statement_metrics = PostgresStatementMetrics(self, self._config, shutdown_callback=self._close_db_pool) - self.statement_samples = PostgresStatementSamples(self, self._config, shutdown_callback=self._close_db_pool) - self.metadata_samples = PostgresMetadata(self, self._config, shutdown_callback=self._close_db_pool) + self.statement_metrics = PostgresStatementMetrics(self, self._config) + self.statement_samples = PostgresStatementSamples(self, self._config) + self.metadata_samples = PostgresMetadata(self, self._config) self._relations_manager = RelationsManager(self._config.relations, self._config.max_relations) self._clean_state() self._query_manager = QueryManager(self, lambda _: None, queries=[]) # query executor is set later @@ -235,7 +239,7 @@ def _new_query_executor(self, queries, db): def execute_query_raw(self, query, db): with db() as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + with conn.cursor() as cursor: cursor.execute(query) rows = cursor.fetchall() return rows @@ -250,11 +254,11 @@ def db(self): self._db = self._new_connection(self._config.dbname) # once the connection is reinitialized, we need to reload the pg_settings self._load_pg_settings(self._db) - if self._db.status != psycopg2.extensions.STATUS_READY: + if self._db.info.status != psycopg.pq.ConnStatus.OK: self._db.rollback() try: yield self._db - except (psycopg2.InterfaceError, InterruptedError): + except (psycopg.InterfaceError, InterruptedError): # if we get an interface error or an interrupted error, # we gracefully close the connection self.log.warning( @@ -275,10 +279,11 @@ def _connection_health_check(self, conn): try: # run a simple query to check if the connection is healthy # health check should run after a connection is established - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + with conn.cursor() as cursor: cursor.execute("SELECT 1") cursor.fetchall() - except psycopg2.OperationalError as e: + self.log.debug("Connection health check passed for database %s", conn.info.dbname) + except psycopg.Error as e: err_msg = f"Database {self._config.dbname} connection health check failed: {str(e)}" self.log.error(err_msg) raise DatabaseHealthCheckError(err_msg) @@ -403,9 +408,13 @@ def cancel(self): self.statement_samples.cancel() self.statement_metrics.cancel() self.metadata_samples.cancel() + if self.statement_metrics._job_loop_future: + self.statement_metrics._job_loop_future.result() + if self.statement_samples._job_loop_future: + self.statement_samples._job_loop_future.result() + if self.metadata_samples._job_loop_future: + self.metadata_samples._job_loop_future.result() self._close_db_pool() - if self._db: - self._db.close() def _clean_state(self): self.log.debug("Cleaning state") @@ -417,7 +426,7 @@ def _get_debug_tags(self): def _get_replication_role(self): with self.db() as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + with conn.cursor() as cursor: cursor.execute('SELECT pg_is_in_recovery();') role = cursor.fetchone()[0] # value fetched for role is of @@ -468,13 +477,13 @@ def _get_local_wal_file_age(self): def load_system_identifier(self): with self.db() as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + with conn.cursor() as cursor: cursor.execute('SELECT system_identifier FROM pg_control_system();') self.system_identifier = cursor.fetchone()[0] def load_cluster_name(self): with self.db() as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + with conn.cursor() as cursor: cursor.execute('SHOW cluster_name;') self.cluster_name = cursor.fetchone()[0] @@ -490,7 +499,7 @@ def initialize_is_aurora(self): def _get_wal_level(self): with self.db() as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + with conn.cursor() as cursor: cursor.execute('SHOW wal_level;') wal_level = cursor.fetchone()[0] return wal_level @@ -560,7 +569,7 @@ def database_hostname(self): def resolve_db_host(self): return agent_host_resolver(self._config.host) - def _run_query_scope(self, cursor, scope, is_custom_metrics, cols, descriptors): + def _run_query_scope(self, scope, is_custom_metrics, cols, descriptors, dbname=None): if scope is None: return None if scope == REPLICATION_METRICS or not self.version >= V9: @@ -571,56 +580,68 @@ def _run_query_scope(self, cursor, scope, is_custom_metrics, cols, descriptors): results = None is_relations = scope.get('relation') and self._relations_manager.has_relations try: - query = fmt.format(scope['query'], metrics_columns=", ".join(cols)) - with tracked_query(check=self, operation='custom_metrics' if is_custom_metrics else scope['name']): - # if this is a relation-specific query, we need to list all relations last - if is_relations: - schema_field = get_schema_field(descriptors) - formatted_query = self._relations_manager.filter_relation_query(query, schema_field) - cursor.execute(formatted_query) - else: - self.log.debug("Running query: %s", str(query)) - cursor.execute(query.replace(r'%', r'%%')) - - results = cursor.fetchall() - except psycopg2.errors.FeatureNotSupported as e: + with self.db() if dbname is None else self.db_pool.get_connection(dbname) as conn: + with conn.cursor() as cursor: + query = fmt.format(scope['query'], metrics_columns=", ".join(cols)) + with tracked_query(check=self, operation='custom_metrics' if is_custom_metrics else scope['name']): + # if this is a relation-specific query, we need to list all relations last + if is_relations: + schema_field = get_schema_field(descriptors) + formatted_query = self._relations_manager.filter_relation_query(query, schema_field) + cursor.execute(formatted_query) + else: + self.log.debug("Running query: %s", str(query)) + cursor.execute(query.replace(r'%', r'%%')) + + results = cursor.fetchall() + if not results: + return None + + if is_custom_metrics and len(results) > MAX_CUSTOM_RESULTS: + self.log.debug( + "Query: %s returned more than %s results (%s). Truncating", + query, + MAX_CUSTOM_RESULTS, + len(results), + ) + results = results[:MAX_CUSTOM_RESULTS] + + if is_relations and len(results) > self._config.max_relations: + self.log.debug( + "Query: %s returned more than %s results (%s). " + "Truncating. You can edit this limit by setting the `max_relations` config option", + query, + self._config.max_relations, + len(results), + ) + results = results[: self._config.max_relations] + + return results + + except psycopg.errors.FeatureNotSupported as e: # This happens for example when trying to get replication metrics from readers in Aurora. Let's ignore it. log_func(e) self.log.debug("Disabling replication metrics") self.is_aurora = False self.metrics_cache.replication_metrics = {} - except psycopg2.errors.UndefinedFunction as e: + except psycopg.errors.UndefinedFunction as e: log_func(e) log_func( "It seems the PG version has been incorrectly identified as %s. " "A reattempt to identify the right version will happen on next agent run." % self.version ) self._clean_state() - except (psycopg2.ProgrammingError, psycopg2.errors.QueryCanceled) as e: + except (psycopg.ProgrammingError, psycopg.errors.QueryCanceled) as e: log_func("Not all metrics may be available: %s" % str(e)) - - if not results: - return None - - if is_custom_metrics and len(results) > MAX_CUSTOM_RESULTS: - self.log.debug( - "Query: %s returned more than %s results (%s). Truncating", query, MAX_CUSTOM_RESULTS, len(results) - ) - results = results[:MAX_CUSTOM_RESULTS] - - if is_relations and len(results) > self._config.max_relations: - self.log.debug( - "Query: %s returned more than %s results (%s). " - "Truncating. You can edit this limit by setting the `max_relations` config option", - query, - self._config.max_relations, - len(results), + except psycopg.Error as e: + log_func( + "Error while executing query: %s. ", + e, ) - results = results[: self._config.max_relations] - return results + return None - def _query_scope(self, cursor, scope, instance_tags, is_custom_metrics, dbname=None): + def _query_scope(self, scope, instance_tags, is_custom_metrics, dbname=None): if scope is None: return None # build query @@ -630,7 +651,7 @@ def _query_scope(self, cursor, scope, instance_tags, is_custom_metrics, dbname=N # A descriptor is the association of a Postgres column name (e.g. 'schemaname') # to a tag name (e.g. 'schema'). descriptors = scope['descriptors'] - results = self._run_query_scope(cursor, scope, is_custom_metrics, cols, descriptors) + results = self._run_query_scope(scope, is_custom_metrics, cols, descriptors, dbname=dbname) if not results: return None @@ -715,10 +736,8 @@ def _collect_metric_autodiscovery(self, instance_tags, scopes, scope_type): databases = self.autodiscovery.get_items() for db in databases: try: - with self.db_pool.get_connection(db, self._config.idle_connection_timeout) as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: - for scope in scopes: - self._query_scope(cursor, scope, instance_tags, False, db) + for scope in scopes: + self._query_scope(scope, instance_tags, False, dbname=db) except Exception as e: self.log.error("Error collecting metrics for database %s %s", db, str(e)) elapsed_ms = (time() - start_time) * 1000 @@ -750,9 +769,7 @@ def _collect_dynamic_queries_autodiscovery(self, queries): databases = self.autodiscovery.get_items() for dbname in databases: - db = functools.partial( - self.db_pool.get_connection, dbname=dbname, ttl_ms=self._config.idle_connection_timeout - ) + db = functools.partial(self.db_pool.get_connection, dbname=dbname) self._dynamic_queries.append(self._new_query_executor(queries, db=db)) def _emit_running_metric(self): @@ -804,25 +821,22 @@ def _collect_stats(self, instance_tags): replication_metrics_query['metrics'] = replication_metrics metric_scope.append(replication_metrics_query) - with self.db() as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: - results_len = self._query_scope(cursor, db_instance_metrics, instance_tags, False) - if results_len is not None: - self.gauge( - "db.count", - results_len, - tags=self.tags_without_db, - hostname=self.reported_hostname, - ) + results_len = self._query_scope(db_instance_metrics, instance_tags, False) + if results_len is not None: + self.gauge( + "db.count", + results_len, + tags=self.tags_without_db, + hostname=self.reported_hostname, + ) - with conn.cursor(cursor_factory=CommenterCursor) as cursor: - self._query_scope(cursor, bgw_instance_metrics, instance_tags, False) - with conn.cursor(cursor_factory=CommenterCursor) as cursor: - self._query_scope(cursor, archiver_instance_metrics, instance_tags, False) + self._query_scope(bgw_instance_metrics, instance_tags, False) + self._query_scope(archiver_instance_metrics, instance_tags, False) - if self._config.collect_checksum_metrics and self.version >= V12: - # SHOW queries need manual cursor execution so can't be bundled with the metrics - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + if self._config.collect_checksum_metrics and self.version >= V12: + # SHOW queries need manual cursor execution so can't be bundled with the metrics + with self.db() as conn: + with conn.cursor() as cursor: cursor.execute("SHOW data_checksums;") enabled = cursor.fetchone()[0] self.count( @@ -831,44 +845,38 @@ def _collect_stats(self, instance_tags): tags=self.tags_without_db + ["enabled:" + "true" if enabled == "on" else "false"], hostname=self.reported_hostname, ) - if self._config.collect_activity_metrics: - activity_metrics = self.metrics_cache.get_activity_metrics(self.version) - with conn.cursor(cursor_factory=CommenterCursor) as cursor: - self._query_scope(cursor, activity_metrics, instance_tags, False) - - if per_database_metric_scope: - # if autodiscovery is enabled, get per-database metrics from all databases found - if self.autodiscovery: - self._collect_metric_autodiscovery( - instance_tags, - scopes=per_database_metric_scope, - scope_type='_collect_stat_autodiscovery', - ) - else: - # otherwise, continue just with dbname - metric_scope.extend(per_database_metric_scope) + if self._config.collect_activity_metrics: + activity_metrics = self.metrics_cache.get_activity_metrics(self.version) + self._query_scope(activity_metrics, instance_tags, False) + + if per_database_metric_scope: + # if autodiscovery is enabled, get per-database metrics from all databases found + if self.autodiscovery: + self._collect_metric_autodiscovery( + instance_tags, + scopes=per_database_metric_scope, + scope_type='_collect_stat_autodiscovery', + ) + else: + # otherwise, continue just with dbname + metric_scope.extend(per_database_metric_scope) - for scope in list(metric_scope): - with conn.cursor(cursor_factory=CommenterCursor) as cursor: - self._query_scope(cursor, scope, instance_tags, False) + for scope in list(metric_scope): + self._query_scope(scope, instance_tags, False) - for scope in self._config.custom_metrics: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: - self._query_scope(cursor, scope, instance_tags, True) + for scope in self._config.custom_metrics: + self._query_scope(scope, instance_tags, True) if self.dynamic_queries: for dynamic_query in self.dynamic_queries: dynamic_query.execute() - def _new_connection(self, dbname): + def build_connection_args(self) -> PostgresConnectionArgs: if self._config.host == 'localhost' and self._config.password == '': - # Use ident method - connection_string = "user=%s dbname=%s application_name=%s" % ( - self._config.user, - dbname, - self._config.application_name, + return PostgresConnectionArgs( + application_name=self._config.application_name, + user=self._config.user, ) - conn = psycopg2.connect(connection_string) else: password = self._config.password if 'aws' in self.cloud_metadata and 'managed_authentication' in self.cloud_metadata['aws']: @@ -896,32 +904,25 @@ def _new_connection(self, dbname): self._config.host, "password" if password == self._config.password else "token", ) + return PostgresConnectionArgs( + application_name=self._config.application_name, + user=self._config.user, + host=self._config.host, + port=self._config.port, + password=password, + ssl_mode=self._config.ssl_mode, + ssl_cert=self._config.ssl_cert, + ssl_root_cert=self._config.ssl_root_cert, + ssl_key=self._config.ssl_key, + ssl_password=self._config.ssl_password, + ) - args = { - 'host': self._config.host, - 'user': self._config.user, - 'password': password, - 'database': dbname, - 'sslmode': self._config.ssl_mode, - 'application_name': self._config.application_name, - } - if self._config.port: - args['port'] = self._config.port - if self._config.ssl_cert: - args['sslcert'] = self._config.ssl_cert - if self._config.ssl_root_cert: - args['sslrootcert'] = self._config.ssl_root_cert - if self._config.ssl_key: - args['sslkey'] = self._config.ssl_key - if self._config.ssl_password: - args['sslpassword'] = self._config.ssl_password - conn = psycopg2.connect(**args) - # Autocommit is enabled by default for safety for all new connections (to prevent long-lived transactions). - conn.set_session(autocommit=True, readonly=True) - if self._config.query_timeout: - # Set the statement_timeout for the session - with conn.cursor() as cursor: - cursor.execute("SET statement_timeout TO %d" % self._config.query_timeout) + def _new_connection(self, dbname): + # TODO: Keeping this main connection outside of the pool for now to keep existing behavior. + # We should move this to the pool in the future. + conn_args = self.build_connection_args() + conn = psycopg.connect(**conn_args.as_kwargs(dbname=dbname)) + self.db_pool._configure_connection(conn) return conn def _connect(self): @@ -936,7 +937,7 @@ def _connect(self): # Reload pg_settings on a new connection to the main db def _load_pg_settings(self, db): try: - with db.cursor(cursor_factory=CommenterDictCursor) as cursor: + with db.cursor() as cursor: self.log.debug("Running query [%s]", PG_SETTINGS_QUERY) cursor.execute( PG_SETTINGS_QUERY, @@ -947,7 +948,7 @@ def _load_pg_settings(self, db): for setting in rows: name, val = setting self.pg_settings[name] = val - except (psycopg2.DatabaseError, psycopg2.OperationalError) as err: + except psycopg.Error as err: self.log.warning("Failed to query for pg_settings: %s", repr(err)) self.count( "dd.postgres.error", @@ -959,20 +960,20 @@ def _load_pg_settings(self, db): def _get_main_db(self): """ - Returns a memoized, persistent psycopg2 connection to `self.dbname`. + Returns a memoized, persistent psycopg connection to `self.dbname`. Threadsafe as long as no transactions are used - :return: a psycopg2 connection + :return: a psycopg connection """ # reload settings for the main DB only once every time the connection is reestablished - return self.db_pool.get_connection( + conn = self.db_pool.get_connection( self._config.dbname, - self._config.idle_connection_timeout, - startup_fn=self._load_pg_settings, persistent=True, ) + return conn + def _close_db_pool(self): - self.db_pool.close_all_connections() + self.db_pool.close_all() def record_warning(self, code, message): # type: (DatabaseConfigurationError, str) -> None diff --git a/postgres/datadog_checks/postgres/statement_samples.py b/postgres/datadog_checks/postgres/statement_samples.py index 343d56ba941e3..19cc622e1ae2f 100644 --- a/postgres/datadog_checks/postgres/statement_samples.py +++ b/postgres/datadog_checks/postgres/statement_samples.py @@ -8,10 +8,9 @@ from enum import Enum from typing import Dict, Optional, Tuple # noqa: F401 -import psycopg2 +import psycopg from cachetools import TTLCache - -from datadog_checks.postgres.cursor import CommenterCursor, CommenterDictCursor +from psycopg.rows import dict_row try: import datadog_agent @@ -30,7 +29,6 @@ from datadog_checks.base.utils.serialization import json from datadog_checks.base.utils.time import get_timestamp from datadog_checks.base.utils.tracking import tracked_method -from datadog_checks.postgres.encoding import decode_with_encodings from datadog_checks.postgres.explain_parameterized_queries import ExplainParameterizedQueries from .util import DatabaseConfigurationError, DBExplainError, trim_leading_set_stmts, warning_with_tags @@ -142,7 +140,7 @@ class PostgresStatementSamples(DBMAsyncJob): Collects statement samples and execution plans. """ - def __init__(self, check, config, shutdown_callback): + def __init__(self, check, config): collection_interval = float( config.statement_samples_config.get('collection_interval', DEFAULT_COLLECTION_INTERVAL) ) @@ -167,9 +165,8 @@ def __init__(self, check, config, shutdown_callback): ), dbms="postgres", min_collection_interval=config.min_collection_interval, - expected_db_exceptions=(psycopg2.errors.DatabaseError,), + expected_db_exceptions=(psycopg.errors.DatabaseError,), job_name="query-samples", - shutdown_callback=shutdown_callback, ) self._check = check self._config = config @@ -244,8 +241,10 @@ def _get_active_connections(self): query = PG_ACTIVE_CONNECTIONS_QUERY.format( pg_stat_activity_view=self._config.pg_stat_activity_view, extra_filters=extra_filters ) + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: + with conn.cursor(row_factory=dict_row) as cursor: self._log.debug("Running query [%s] %s", query, params) cursor.execute(query, params) rows = cursor.fetchall() @@ -277,14 +276,12 @@ def _get_new_pg_stat_activity(self, available_activity_columns, activity_columns pg_stat_activity_view=self._config.pg_stat_activity_view, extra_filters=extra_filters, ) + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: + with conn.cursor(row_factory=dict_row) as cursor: self._log.debug("Running query [%s] %s", query, params) - if conn.encoding == "SQLASCII": - # SQLASCII can truncate encodings across bytes, e.g. UTF8 multi-byte characters - # so we need to read in the data as bytes and then decode as best we can - psycopg2.extensions.register_type(psycopg2.extensions.BYTES, cursor) cursor.execute(query, params) rows = cursor.fetchall() @@ -301,8 +298,10 @@ def _get_pg_stat_activity_cols_cached(self, expected_cols): @tracked_method(agent_check_getter=agent_check_getter, track_result_length=True) def _get_available_activity_columns(self, all_expected_columns): + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: + with conn.cursor(row_factory=dict_row) as cursor: try: cursor.execute( "select * from {pg_stat_activity_view} LIMIT 0".format( @@ -317,12 +316,12 @@ def _get_available_activity_columns(self, all_expected_columns): "missing the following expected columns from pg_stat_activity: %s", missing_columns ) self._log.debug("found available pg_stat_activity columns: %s", available_columns) - except psycopg2.errors.InvalidSchemaName as e: + except psycopg.errors.InvalidSchemaName as e: self._log.warning( "cannot collect activity due to invalid schema in dbname=%s: %s", self._config.dbname, repr(e) ) return None - except psycopg2.DatabaseError as e: + except psycopg.DatabaseError as e: # if the schema is valid then it's some problem with the function (missing, or invalid permissions, # incorrect definition) self._check.record_warning( @@ -348,42 +347,16 @@ def _filter_and_normalize_statement_rows(self, rows): insufficient_privilege_count = 0 total_count = 0 normalized_rows = [] - for raw_row in rows: + for row in rows: total_count += 1 - row = {} - with self._check._get_main_db() as conn: - encoding = conn.encoding if conn.encoding != "SQLASCII" else 'utf-8' - - for key, value in raw_row.items(): - if type(value) is not bytes: - row[key] = value - elif key == "query": - try: - # Attempt decoding query in potential encodings - row[key] = decode_with_encodings(value, self._config.query_encodings) - except Exception as e: - # Log the unable to decode query error - self._log.warning("Unable to decode query: %s | Error: %s", value, e) - break - else: - try: - # Decode other columns as database encoding, or default to utf-8 - try: - row[key] = value.decode(encoding) - except Exception: - # Fallback to trying utf-8 - row[key] = value.decode('utf-8', 'backslashreplace') - except Exception as e: - self._log.warning("Unable to decode column: %s: %s | Error: %s", key, value, e) - row[key] = "unknown" - # We later on fallback to having the statement be set to the backend_type so it's important to only - # set it if it is not None - if row.get("backend_type") is not None: - # backend_type is always a bytea type because it can contain non-UTF8 characters - try: - row["backend_type"] = row.get("backend_type").tobytes().decode('utf-8', 'backslashreplace') - except Exception: - row["backend_type"] = "unknown" + + # Manually decode backend_type to handle bad encodings in Azure + row['backend_type'] = ( + row['backend_type'].decode('utf-8', errors='backslashreplace') + if isinstance(row.get('backend_type'), bytes) + else row.get('backend_type') + ) + if not row.get('query'): continue if (not row.get('datname')) and row.get('backend_type', 'client backend') == 'client backend': @@ -418,7 +391,7 @@ def _filter_and_normalize_statement_rows(self, rows): def _normalize_row(self, row): normalized_row = dict(copy.copy(row)) obfuscated_query = None - backend_type = normalized_row.get('backend_type', 'client backend') + backend_type = normalized_row.get('backend_type', 'client backend') or 'client backend' try: if backend_type != 'client backend': obfuscated_query = backend_type @@ -494,7 +467,6 @@ def run_job(self): self.tags = [t for t in self._tags if not t.startswith('dd.internal')] self._tags_no_db = [t for t in self.tags if not t.startswith('db:')] self._collect_statement_samples() - self.db_pool.prune_connections() @tracked_method(agent_check_getter=agent_check_getter) def _collect_statement_samples(self): @@ -675,13 +647,13 @@ def _can_explain_statement(self, obfuscated_statement): def _get_db_explain_setup_state(self, dbname): # type: (str) -> Tuple[Optional[DBExplainError], Optional[Exception]] try: - self.db_pool.get_connection(dbname, self._conn_ttl_ms) - except psycopg2.OperationalError as e: + self.db_pool.get_connection(dbname) + except psycopg.OperationalError as e: self._log.warning( "cannot collect execution plans due to failed DB connection to dbname=%s: %s", dbname, repr(e) ) return DBExplainError.connection_error, e - except psycopg2.DatabaseError as e: + except psycopg.DatabaseError as e: self._log.warning( "cannot collect execution plans due to a database error in dbname=%s: %s", dbname, repr(e) ) @@ -689,14 +661,14 @@ def _get_db_explain_setup_state(self, dbname): try: result = self._run_explain(dbname, EXPLAIN_VALIDATION_QUERY, EXPLAIN_VALIDATION_QUERY) - except psycopg2.errors.InvalidSchemaName as e: + except psycopg.errors.InvalidSchemaName as e: self._log.warning("cannot collect execution plans due to invalid schema in dbname=%s: %s", dbname, repr(e)) self._emit_run_explain_error(dbname, DBExplainError.invalid_schema, e) return DBExplainError.invalid_schema, e - except psycopg2.errors.DatatypeMismatch as e: + except psycopg.errors.DatatypeMismatch as e: self._emit_run_explain_error(dbname, DBExplainError.datatype_mismatch, e) return DBExplainError.datatype_mismatch, e - except psycopg2.DatabaseError as e: + except psycopg.DatabaseError as e: # if the schema is valid then it's some problem with the function (missing, or invalid permissions, # incorrect definition) self._emit_run_explain_error(dbname, DBExplainError.failed_function, e) @@ -739,15 +711,17 @@ def _get_db_explain_setup_state_cached(self, dbname): def _run_explain(self, dbname, statement, obfuscated_statement): start_time = time.time() - with self.db_pool.get_connection(dbname, ttl_ms=self._conn_ttl_ms) as conn: + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") + with self.db_pool.get_connection(dbname) as conn: # When sending potentially non-ascii data, e.g. UTF8, we need to force # the client encoding to UTF-8 to match Python string encoding - if conn.encoding == 'SQLASCII': + if conn.info.encoding.lower() in ["ascii", "sqlascii", "sql_ascii"]: self._log.debug( "Setting client encoding to UTF-8 for dbname=%s, as the current encoding is SQLASCII", dbname ) - conn.set_client_encoding('utf-8') - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + conn.execute("SET client_encoding TO UTF8") + with conn.cursor() as cursor: self._log.debug( "Running query on dbname=%s: %s(%s)", dbname, self._explain_function, obfuscated_statement ) @@ -829,7 +803,7 @@ def _run_explain_safe(self, dbname, statement, obfuscated_statement, query_signa return self._explain_parameterized_queries.explain_statement( dbname, statement, obfuscated_statement, query_signature ) - e = psycopg2.errors.UndefinedParameter("Unable to explain parameterized query") + e = psycopg.errors.UndefinedParameter("Unable to explain parameterized query") self._log.debug( "Unable to collect execution plan, clients using the extended query protocol or prepared statements" " can't be explained due to the separation of the parsed query and raw bind parameters: %s", @@ -840,30 +814,30 @@ def _run_explain_safe(self, dbname, statement, obfuscated_statement, query_signa self._emit_run_explain_error(dbname, DBExplainError.parameterized_query, e) return error_response return self._run_explain(dbname, statement, obfuscated_statement), None, None - except psycopg2.errors.UndefinedTable as e: + except psycopg.errors.UndefinedTable as e: self._log.debug("Failed to collect execution plan: %s", repr(e)) error_response = None, DBExplainError.undefined_table, '{}'.format(type(e)) self._explain_errors_cache[query_signature] = error_response self._emit_run_explain_error(dbname, DBExplainError.undefined_table, e) return error_response - except psycopg2.errors.UndefinedFunction as e: + except psycopg.errors.UndefinedFunction as e: self._log.debug("Failed to collect execution plan: %s", repr(e)) error_response = None, DBExplainError.undefined_function, '{}'.format(type(e)) self._explain_errors_cache[query_signature] = error_response self._emit_run_explain_error(dbname, DBExplainError.undefined_function, e) return error_response - except psycopg2.errors.IndeterminateDatatype as e: + except psycopg.errors.IndeterminateDatatype as e: self._log.debug("Failed to collect execution plan: %s", repr(e)) error_response = None, DBExplainError.indeterminate_datatype, '{}'.format(type(e)) self._explain_errors_cache[query_signature] = error_response self._emit_run_explain_error(dbname, DBExplainError.indeterminate_datatype, e) return error_response - except psycopg2.errors.DatabaseError as e: + except psycopg.errors.DatabaseError as e: self._log.debug("Failed to collect execution plan: %s", repr(e)) error_response = None, DBExplainError.database_error, '{}'.format(type(e)) self._emit_run_explain_error(dbname, DBExplainError.database_error, e) - if isinstance(e, psycopg2.errors.ProgrammingError) and not isinstance( - e, psycopg2.errors.InsufficientPrivilege + if isinstance(e, psycopg.errors.ProgrammingError) and not isinstance( + e, psycopg.errors.InsufficientPrivilege ): # ProgrammingError is things like InvalidName, InvalidSchema, SyntaxError # we don't want to cache things like permission errors for a very long time because they can be fixed @@ -988,7 +962,10 @@ def _collect_plan_for_statement(self, row): def _collect_plans(self, rows): for row in rows: try: - if row['statement'] is None or row.get('backend_type', 'client backend') != 'client backend': + if ( + row['statement'] is None + or (row.get('backend_type', 'client backend') or 'client backend') != 'client backend' + ): continue yield from self._collect_plan_for_statement(row) except Exception: diff --git a/postgres/datadog_checks/postgres/statements.py b/postgres/datadog_checks/postgres/statements.py index e9d67ac66c4d2..bf8790389fcad 100644 --- a/postgres/datadog_checks/postgres/statements.py +++ b/postgres/datadog_checks/postgres/statements.py @@ -5,10 +5,11 @@ import copy import time +from typing import Tuple -import psycopg2 -import psycopg2.extras +import psycopg from cachetools import TTLCache +from psycopg.rows import dict_row from datadog_checks.base import is_affirmative from datadog_checks.base.utils.common import to_native_string @@ -17,8 +18,6 @@ from datadog_checks.base.utils.db.utils import DBMAsyncJob, default_json_event_encoding, obfuscate_sql_with_metadata from datadog_checks.base.utils.serialization import json from datadog_checks.base.utils.tracking import tracked_method -from datadog_checks.postgres.cursor import CommenterCursor, CommenterDictCursor -from datadog_checks.postgres.encoding import decode_with_encodings from .query_calls_cache import QueryCallsCache from .util import DatabaseConfigurationError, payload_pg_version, warning_with_tags @@ -157,7 +156,7 @@ def _row_key(row): class PostgresStatementMetrics(DBMAsyncJob): """Collects telemetry for SQL statements""" - def __init__(self, check, config, shutdown_callback): + def __init__(self, check, config): collection_interval = float( config.statement_metrics_config.get('collection_interval', DEFAULT_COLLECTION_INTERVAL) ) @@ -167,12 +166,11 @@ def __init__(self, check, config, shutdown_callback): check, run_sync=is_affirmative(config.statement_metrics_config.get('run_sync', False)), enabled=is_affirmative(config.statement_metrics_config.get('enabled', True)), - expected_db_exceptions=(psycopg2.errors.DatabaseError,), + expected_db_exceptions=(psycopg.errors.DatabaseError,), min_collection_interval=config.min_collection_interval, dbms="postgres", rate_limit=1 / float(collection_interval), job_name="query-metrics", - shutdown_callback=shutdown_callback, ) self._check = check self._metrics_collection_interval = collection_interval @@ -203,14 +201,19 @@ def __init__(self, check, config, shutdown_callback): ttl=60 * 60 / config.full_statement_text_samples_per_hour_per_query, ) - def _execute_query(self, cursor, query, params=()): + def _execute_query(self, query, params=(), binary=False, row_factory=None) -> Tuple[list, list]: + if self._cancel_event.is_set(): + raise Exception("Job loop cancelled. Aborting query.") try: - self._log.debug("Running query [%s] %s", query, params) - cursor.execute(query, params) - return cursor.fetchall() - except (psycopg2.ProgrammingError, psycopg2.errors.QueryCanceled) as e: + with self._check._get_main_db() as conn: + with conn.cursor(row_factory=row_factory) as cursor: + self._log.debug("Running query [%s] %s", query, params) + cursor.execute(query, params=params, binary=binary) + return cursor.fetchall(), cursor.description + except psycopg.Error as e: # A failed query could've derived from incorrect columns within the cache. It's a rare edge case, # but the next time the query is run, it will retrieve the correct columns. + self._log.warning("Failed to run query [%s] %s", query, params) self._stat_column_cache = [] raise e @@ -230,12 +233,12 @@ def _get_pg_stat_statements_columns(self): pg_stat_statements_view=self._config.pg_stat_statements_view, extra_clauses="LIMIT 0", ) - with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: - self._execute_query(cursor, query, params=(self._config.dbname,)) - col_names = [desc[0] for desc in cursor.description] if cursor.description else [] - self._stat_column_cache = col_names - return col_names + + _, description = self._execute_query(query) + col_names = [desc[0] for desc in description] if description else [] + self._stat_column_cache = col_names + self._log.debug("Fetched columns %s", col_names) + return col_names def _check_called_queries(self): pgss_view_without_query_text = self._config.pg_stat_statements_view @@ -246,20 +249,18 @@ def _check_called_queries(self): # For more info: https://www.postgresql.org/docs/current/pgstatstatements.html#PGSTATSTATEMENTS-FUNCS pgss_view_without_query_text = "pg_stat_statements(false)" - with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: - query = QUERYID_TO_CALLS_QUERY.format(pg_stat_statements_view=pgss_view_without_query_text) - rows = self._execute_query(cursor, query, params=(self._config.dbname,)) - self._query_calls_cache.set_calls(rows) - self._check.gauge( - "dd.postgresql.pg_stat_statements.calls_changed", - len(self._query_calls_cache.called_queryids), - tags=self.tags, - hostname=self._check.reported_hostname, - raw=True, - ) + query = QUERYID_TO_CALLS_QUERY.format(pg_stat_statements_view=pgss_view_without_query_text) + rows, _ = self._execute_query(query, row_factory=dict_row) + self._query_calls_cache.set_calls(rows) + self._check.gauge( + "dd.postgresql.pg_stat_statements.calls_changed", + len(self._query_calls_cache.called_queryids), + tags=self.tags, + hostname=self._check.reported_hostname, + raw=True, + ) - return self._query_calls_cache.called_queryids + return self._query_calls_cache.called_queryids def run_job(self): # do not emit any dd.internal metrics for DBM specific check code @@ -400,39 +401,33 @@ def _load_pg_stat_statements(self): "pg_database.datname NOT ILIKE %s" for _ in self._config.ignore_databases ) params = params + tuple(self._config.ignore_databases) - with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: - if conn.encoding == "SQLASCII": - # SQLASCII can truncate encodings across bytes, e.g. UTF8 multi-byte characters - # so we need to read in the data as bytes and then decode as best we can - psycopg2.extensions.register_type(psycopg2.extensions.BYTES, cursor) - if len(self._query_calls_cache.cache) > 0: - return self._execute_query( - cursor, - statements_query( - cols=', '.join(query_columns), - pg_stat_statements_view=self._config.pg_stat_statements_view, - filters=filters, - called_queryids=', '.join([str(i) for i in self._query_calls_cache.called_queryids]), - ), - params=params, - ) - else: - return self._execute_query( - cursor, - statements_query( - cols=', '.join(query_columns), - pg_stat_statements_view=self._config.pg_stat_statements_view, - filters=filters, - ), - params=params, - ) - except psycopg2.Error as e: + if len(self._query_calls_cache.cache) > 0: + rows, _ = self._execute_query( + statements_query( + cols=', '.join(query_columns), + pg_stat_statements_view=self._config.pg_stat_statements_view, + filters=filters, + called_queryids=', '.join([str(i) for i in self._query_calls_cache.called_queryids]), + ), + params=params, + row_factory=dict_row, + ) + return rows + else: + rows, _ = self._execute_query( + statements_query( + cols=', '.join(query_columns), + pg_stat_statements_view=self._config.pg_stat_statements_view, + filters=filters, + ), + params=params, + row_factory=dict_row, + ) + return rows + except psycopg.Error as e: error_tag = "error:database-{}".format(type(e).__name__) - if ( - isinstance(e, psycopg2.errors.ObjectNotInPrerequisiteState) - ) and 'pg_stat_statements must be loaded' in str(e.pgerror): + if (isinstance(e, psycopg.errors.ObjectNotInPrerequisiteState)) and 'pg_stat_statements' in str(e): error_tag = "error:database-{}-pg_stat_statements_not_loaded".format(type(e).__name__) self._check.record_warning( DatabaseConfigurationError.pg_stat_statements_not_loaded, @@ -448,7 +443,7 @@ def _load_pg_stat_statements(self): code=DatabaseConfigurationError.pg_stat_statements_not_loaded.value, ), ) - elif isinstance(e, psycopg2.errors.UndefinedTable) and 'pg_stat_statements' in str(e.pgerror): + elif isinstance(e, psycopg.errors.UndefinedTable) and 'pg_stat_statements' in str(e): error_tag = "error:database-{}-pg_stat_statements_not_created".format(type(e).__name__) self._check.record_warning( DatabaseConfigurationError.pg_stat_statements_not_created, @@ -490,33 +485,27 @@ def _emit_pg_stat_statements_dealloc(self): if self._check.version < V14: return try: - with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: - rows = self._execute_query( - cursor, - PG_STAT_STATEMENTS_DEALLOC, - ) - if rows: - dealloc = rows[0][0] - self._check.monotonic_count( - "pg_stat_statements.dealloc", - dealloc, - tags=self.tags, - hostname=self._check.reported_hostname, - ) - except psycopg2.Error as e: + rows, _ = self._execute_query( + PG_STAT_STATEMENTS_DEALLOC, + ) + if rows: + dealloc = rows[0][0] + self._check.monotonic_count( + "pg_stat_statements.dealloc", + dealloc, + tags=self.tags, + hostname=self._check.reported_hostname, + ) + except psycopg.Error as e: self._log.warning("Failed to query for pg_stat_statements_info: %s", e) @tracked_method(agent_check_getter=agent_check_getter) def _emit_pg_stat_statements_metrics(self): query = PG_STAT_STATEMENTS_COUNT_QUERY_LT_9_4 if self._check.version < V9_4 else PG_STAT_STATEMENTS_COUNT_QUERY try: - with self._check._get_main_db() as conn: - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: - rows = self._execute_query( - cursor, - query, - ) + rows, _ = self._execute_query( + query, + ) count = 0 if rows: count = rows[0][0] @@ -532,7 +521,7 @@ def _emit_pg_stat_statements_metrics(self): tags=self.tags, hostname=self._check.reported_hostname, ) - except psycopg2.Error as e: + except psycopg.Error as e: self._log.warning("Failed to query for pg_stat_statements count: %s", e) def _baseline_metrics_query_key(self, row): @@ -572,7 +561,7 @@ def _check_baseline_metrics_expiry(self): if ( self._last_baseline_metrics_expiry is None or self._last_baseline_metrics_expiry + self._config.baseline_metrics_expiry < time.time() - or len(self._baseline_metrics) > 3 * int(self._check.pg_settings.get("pg_stat_statements.max")) + or len(self._baseline_metrics) > 3 * int(self._check.pg_settings.get("pg_stat_statements.max", 10000)) ): self._baseline_metrics = {} self._query_calls_cache = QueryCallsCache() @@ -630,18 +619,11 @@ def _collect_metrics_rows(self): return rows def _normalize_queries(self, rows): - with self._check._get_main_db() as conn: - encoding = conn.encoding if conn.encoding != "SQLASCII" else 'utf-8' - normalized_rows = [] for row in rows: normalized_row = dict(copy.copy(row)) try: - query_text = ( - decode_with_encodings(row['query'], self._config.query_encodings) - if type(row['query']) is bytes - else row['query'] - ) + query_text = row['query'] statement = obfuscate_sql_with_metadata(query_text, self._obfuscate_options) except Exception as e: if self._config.log_unobfuscated_queries: @@ -653,21 +635,6 @@ def _normalize_queries(self, rows): obfuscated_query = statement['query'] normalized_row['query'] = obfuscated_query normalized_row['query_signature'] = compute_sql_signature(obfuscated_query) - for key in ['datname', 'rolname']: - value = row[key] - if type(value) is not bytes: - normalized_row[key] = value - continue - try: - # Decode other columns as database encoding, or default to utf-8 - try: - normalized_row[key] = value.decode(encoding) - except Exception: - # Fallback to trying utf-8 - normalized_row[key] = value.decode('utf-8', 'backslashreplace') - except Exception as e: - self._log.warning("Unable to decode column: %s: %s | Error: %s", key, value, e) - normalized_row[key] = "unknown" metadata = statement['metadata'] normalized_row['dd_tables'] = metadata.get('tables', None) diff --git a/postgres/datadog_checks/postgres/util.py b/postgres/datadog_checks/postgres/util.py index 63601b9c017d0..dadb05c93a49e 100644 --- a/postgres/datadog_checks/postgres/util.py +++ b/postgres/datadog_checks/postgres/util.py @@ -91,6 +91,8 @@ class DBExplainError(Enum): # PostgreSQL cannot determine the data type of a parameter in the query indeterminate_datatype = 'indeterminate_datatype' + unknown_error = 'unknown_error' + class DatabaseHealthCheckError(Exception): pass diff --git a/postgres/datadog_checks/postgres/version_utils.py b/postgres/datadog_checks/postgres/version_utils.py index 349f866054b4f..d8fadedf45739 100644 --- a/postgres/datadog_checks/postgres/version_utils.py +++ b/postgres/datadog_checks/postgres/version_utils.py @@ -6,7 +6,6 @@ from semver import VersionInfo from datadog_checks.base.log import get_check_logger -from datadog_checks.postgres.cursor import CommenterCursor V8_3 = VersionInfo.parse("8.3.0") V9 = VersionInfo.parse("9.0.0") @@ -32,7 +31,7 @@ def __init__(self): @staticmethod def get_raw_version(db): with db as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + with conn.cursor() as cursor: cursor.execute('SHOW SERVER_VERSION;') raw_version = cursor.fetchone()[0] return raw_version @@ -41,7 +40,7 @@ def is_aurora(self, db): if self._is_aurora is not None: return self._is_aurora with db as conn: - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + with conn.cursor() as cursor: cursor.execute( "SELECT 1 FROM pg_available_extension_versions " "WHERE name ILIKE '%aurora%' OR comment ILIKE '%aurora%' " diff --git a/postgres/pyproject.toml b/postgres/pyproject.toml index 73b67f49e252e..026334b3bf7fc 100644 --- a/postgres/pyproject.toml +++ b/postgres/pyproject.toml @@ -28,7 +28,7 @@ classifiers = [ "Private :: Do Not Upload", ] dependencies = [ - "datadog-checks-base>=37.10.0", + "datadog-checks-base>=37.17.1", ] dynamic = [ "version", @@ -40,7 +40,7 @@ deps = [ "azure-identity==1.23.0", "boto3==1.38.41", "cachetools==6.1.0", - "psycopg2-binary==2.9.9", + "psycopg[binary,pool]==3.2.7", "semver==3.0.4", ] diff --git a/postgres/tests/conftest.py b/postgres/tests/conftest.py index 395203e9592c9..476f2342463fd 100644 --- a/postgres/tests/conftest.py +++ b/postgres/tests/conftest.py @@ -4,7 +4,7 @@ import copy import os -import psycopg2 +import psycopg import pytest from semver import VersionInfo @@ -39,12 +39,20 @@ } +E2E_METADATA = { + 'start_commands': [ + 'apt update', + 'apt install -y --no-install-recommends build-essential python3-dev libpq-dev', + ], +} + + def connect_to_pg(): - psycopg2.connect(host=HOST, dbname=DB_NAME, user=USER, password=PASSWORD) + psycopg.connect(host=HOST, dbname=DB_NAME, user=USER, password=PASSWORD) if float(POSTGRES_VERSION) >= 10.0: - psycopg2.connect(host=HOST, dbname=DB_NAME, user=USER, port=PORT_REPLICA, password=PASSWORD) - psycopg2.connect(host=HOST, dbname=DB_NAME, user=USER, port=PORT_REPLICA2, password=PASSWORD) - psycopg2.connect(host=HOST, dbname=DB_NAME, user=USER, port=PORT_REPLICA_LOGICAL, password=PASSWORD) + psycopg.connect(host=HOST, dbname=DB_NAME, user=USER, port=PORT_REPLICA, password=PASSWORD) + psycopg.connect(host=HOST, dbname=DB_NAME, user=USER, port=PORT_REPLICA2, password=PASSWORD) + psycopg.connect(host=HOST, dbname=DB_NAME, user=USER, port=PORT_REPLICA_LOGICAL, password=PASSWORD) @pytest.fixture(scope='session') @@ -60,7 +68,7 @@ def dd_environment(e2e_instance): conditions=[WaitFor(connect_to_pg)], env_vars={"POSTGRES_IMAGE": POSTGRES_IMAGE, "POSTGRES_LOCALE": POSTGRES_LOCALE}, ): - yield e2e_instance + yield e2e_instance, E2E_METADATA @pytest.fixture diff --git a/postgres/tests/test_connection_pool.py b/postgres/tests/test_connection_pool.py new file mode 100644 index 0000000000000..2f7ea1bae2c9b --- /dev/null +++ b/postgres/tests/test_connection_pool.py @@ -0,0 +1,768 @@ +# (C) Datadog, Inc. 2025-present +# All rights reserved +# Licensed under a 3-clause BSD style license (see LICENSE) + +import queue +import threading +import time +from typing import Dict + +import pytest +from psycopg import Connection +from psycopg.errors import AdminShutdown + +from datadog_checks.postgres.connection_pool import LRUConnectionPoolManager, PostgresConnectionArgs + +from .utils import _get_superconn + + +def _create_conn_args(pg_instance, application_name="test_connection_pool"): + """ + Helper function to create PostgresConnectionArgs with common test configuration. + + Args: + pg_instance: The PostgreSQL instance configuration + application_name: The application name for the connection + + Returns: + PostgresConnectionArgs: Configured connection arguments + """ + return PostgresConnectionArgs( + application_name=application_name, + user=pg_instance["username"], + password=pg_instance["password"], + host=pg_instance["host"], + port=int(pg_instance["port"]), + ) + + +def _make_base_args(**overrides): + """Helper to create base connection arguments with defaults.""" + base = { + "application_name": "test_app", + "user": "testuser", + "password": "testpass", + "host": "localhost", + "port": 5432, + "ssl_mode": "allow", + } + base.update(overrides) + return base + + +def _make_expected_kwargs(**overrides): + """Helper to create expected kwargs with defaults.""" + base = { + "application_name": "test_app", + "user": "testuser", + "password": "testpass", + "host": "localhost", + "port": 5432, + "sslmode": "allow", + } + base.update(overrides) + + # Remove None values to match actual implementation behavior + return {k: v for k, v in base.items() if v is not None} + + +@pytest.mark.parametrize( + "init_args, dbname, expected", + [ + # Basic functionality tests + (_make_base_args(), "testdb", _make_expected_kwargs(dbname="testdb")), + (_make_base_args(), "override_db", _make_expected_kwargs(dbname="override_db")), + (_make_base_args(), "overridedb", _make_expected_kwargs(dbname="overridedb")), + # Optional parameter exclusion tests + (_make_base_args(password=None), "testdb", _make_expected_kwargs(dbname="testdb", password=None)), + (_make_base_args(port=None), "testdb", _make_expected_kwargs(dbname="testdb", port=None)), + ( + _make_base_args(password=""), + "testdb", + _make_expected_kwargs(dbname="testdb", password=None), # Empty string excluded + ), + # SSL configuration tests + ( + _make_base_args( + ssl_mode="require", + ssl_cert="/path/to/client.crt", + ssl_key="/path/to/client.key", + ssl_root_cert="/path/to/ca.crt", + ssl_password="sslkeypass", + ), + "ssldb", + _make_expected_kwargs( + dbname="ssldb", + sslmode="require", + sslcert="/path/to/client.crt", + sslkey="/path/to/client.key", + sslrootcert="/path/to/ca.crt", + sslpassword="sslkeypass", + ), + ), + ( + _make_base_args(ssl_mode="prefer", ssl_cert="/path/to/client.crt", ssl_key="/path/to/client.key"), + "override_ssl_db", + _make_expected_kwargs( + dbname="override_ssl_db", sslmode="prefer", sslcert="/path/to/client.crt", sslkey="/path/to/client.key" + ), + ), + ( + _make_base_args(ssl_mode="verify-full"), + "testdb", + _make_expected_kwargs(dbname="testdb", sslmode="verify-full"), + ), + # SSL parameter exclusion tests + ( + _make_base_args(ssl_mode="require", ssl_cert=None, ssl_key=None, ssl_root_cert=None, ssl_password=None), + "testdb", + _make_expected_kwargs(dbname="testdb", sslmode="require"), + ), + ( + _make_base_args( + ssl_mode="require", + ssl_cert="/path/to/client.crt", + ssl_key=None, + ssl_root_cert="/path/to/ca.crt", + ssl_password=None, + ), + "testdb", + _make_expected_kwargs( + dbname="testdb", sslmode="require", sslcert="/path/to/client.crt", sslrootcert="/path/to/ca.crt" + ), + ), + # Edge cases + ( + _make_base_args(password="", ssl_mode=""), + "testdb", + _make_expected_kwargs( + dbname="testdb", + password=None, # Empty string excluded + sslmode="", + ), + ), + ( + _make_base_args(), + "", # Empty string dbname + _make_expected_kwargs(dbname=""), # Empty string is valid + ), + # Socket connection tests (no host or password) + ( + _make_base_args(host=None, password=None), + "testdb", + _make_expected_kwargs(dbname="testdb", host=None, password=None), + ), + ( + _make_base_args(host=None, password=None, port=None), + "testdb", + _make_expected_kwargs(dbname="testdb", host=None, password=None, port=None), + ), + ( + _make_base_args(host="", password=""), + "testdb", + _make_expected_kwargs(dbname="testdb", host=None, password=None), # Empty strings excluded + ), + ], +) +def test_postgres_connection_args_as_kwargs(init_args, dbname, expected): + """ + Validates that PostgresConnectionArgs.as_kwargs() correctly outputs connection kwargs + under various combinations of initialization parameters and dbname. + + Test cases cover: + - Basic connection parameters (user, password, host, port) + - SSL configuration (ssl_mode, ssl_cert, ssl_key, ssl_root_cert, ssl_password) + - Edge cases (None values, empty strings, missing optional parameters) + - dbname parameter handling + - Parameter exclusion when values are None + """ + args = PostgresConnectionArgs(**init_args) + kwargs = args.as_kwargs(dbname=dbname) + + assert kwargs == expected + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_basic_connection(pg_instance: Dict[str, str]): + """ + Test basic connection acquisition, query execution, and pool stats + for a single dbname using LRUConnectionPoolManager. + """ + conn_args = _create_conn_args(pg_instance, "test_basic_connection") + manager = LRUConnectionPoolManager(max_db=3, base_conn_args=conn_args, pool_config={"min_size": 1, "max_size": 2}) + + try: + dbname = pg_instance["dbname"] + with manager.get_connection(dbname) as conn: + assert isinstance(conn, Connection) + + with conn.cursor() as cur: + cur.execute("SELECT 1") + result = cur.fetchone() + assert result == (1,) + + stats = manager.get_pool_stats(dbname) + assert stats is not None + # Basic presence checks + assert "last_used" in stats + assert isinstance(stats["last_used"], float) + + # Pool state + assert stats["pool_min"] == 0 + assert stats["pool_max"] == 2 + assert 1 <= stats["pool_size"] <= 2 + assert stats["pool_available"] >= 1 # Should be 1 or 2 after release + + # Usage metrics + assert stats["connections_num"] >= 1 + assert stats["requests_num"] == 1 + assert stats["requests_waiting"] == 0 + + finally: + manager.close_all() + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_lru_eviction_and_connection_rotation(pg_instance): + """ + Opens and closes multiple connections across dogs_n databases and asserts LRU behavior + and correct enforcement of min_size=0, max_size=2 per pool. + """ + conn_args = _create_conn_args(pg_instance, "test_lru_eviction_and_connection_rotation") + manager = LRUConnectionPoolManager(max_db=3, base_conn_args=conn_args) + + dbnames = [f"dogs_{i}" for i in range(5)] # 5 dbs, max pool limit is 3 + + # Open a connection to each dbname once + for dbname in dbnames: + with manager.get_connection(dbname) as conn: + assert isinstance(conn, Connection) + with conn.cursor() as cur: + cur.execute("SELECT 1") + result = cur.fetchone() + assert result == (1,) + + # After connecting to 5 DBs, only 3 should remain in the pool + assert len(manager.pools) == 3 + + # Check that those pools respect pool_config limits + for _dbname, (pool, _last_used, persistent) in manager.pools.items(): + stats = pool.get_stats() + assert stats["pool_min"] == 0 + assert stats["pool_max"] == 2 + assert stats["pool_size"] <= 2 + assert persistent is False + + manager.close_all() + # After closing, pool dict should be empty + assert len(manager.pools) == 0 + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_lru_eviction_order(pg_instance): + """ + Verifies that the least recently used dbname pool is evicted when the max_db limit is exceeded. + """ + conn_args = _create_conn_args(pg_instance, "test_lru_eviction_order") + manager = LRUConnectionPoolManager(max_db=3, base_conn_args=conn_args) + + try: + # Step 1–3: Fill up with 3 distinct pools + for db in ["dogs_0", "dogs_1", "dogs_2"]: + with manager.get_connection(db) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + + # Access dogs_0 again to make it most recently used + with manager.get_connection("dogs_0") as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + + # Step 4: Add a new pool — this should evict dogs_1 (oldest) + with manager.get_connection("dogs_3") as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + + # Step 5: Verify eviction + current_pools = list(manager.pools.keys()) + assert "dogs_1" not in current_pools + assert set(current_pools) == {"dogs_0", "dogs_2", "dogs_3"} + + finally: + manager.close_all() + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_max_idle_closes_and_reopens_connection(pg_instance): + """ + Tests that a connection is closed after max_idle seconds of idleness, + and that the pool reopens a usable connection on next use. + """ + conn_args = _create_conn_args(pg_instance, "test_max_idle_closes_and_reopens_connection") + + manager = LRUConnectionPoolManager( + max_db=1, + base_conn_args=conn_args, + pool_config={ + "max_idle": 0.5, # second timeout + }, + ) + + dbname = pg_instance["dbname"] + + try: + # Step 1: Open a connection and return it + with manager.get_connection(dbname) as conn: + assert isinstance(conn, Connection) + with conn.cursor() as cur: + cur.execute("SELECT 1") + + stats1 = manager.get_pool_stats(dbname) + first_conn_count = stats1["connections_num"] + assert first_conn_count == 1 + assert stats1["pool_available"] == 1 + + # Step 2: Wait longer than max_idle so idle conn is closed + time.sleep(2) + + # Step 3: Re-acquire a connection + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + + stats2 = manager.get_pool_stats(dbname) + assert stats2["connections_num"] >= 2, "Expected a new connection to be opened after idle timeout" + assert stats2["pool_size"] <= 1 + + finally: + manager.close_all() + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_statement_timeout_configuration(pg_instance: Dict[str, str]): + """ + Test that statement_timeout is properly configured on connections and that + timeout behavior works correctly. + """ + conn_args = _create_conn_args(pg_instance, "test_statement_timeout") + + manager = LRUConnectionPoolManager( + max_db=1, + base_conn_args=conn_args, + statement_timeout=1000, # 1 second + ) + + try: + dbname = pg_instance["dbname"] + with manager.get_connection(dbname) as conn: + assert isinstance(conn, Connection) + + with conn.cursor() as cur: + # Verify timeout is set correctly + cur.execute("SHOW statement_timeout") + timeout = cur.fetchone()[0] + assert timeout == "1s" + + # Test that a normal query works + cur.execute("SELECT 1") + result = cur.fetchone() + assert result == (1,) + + # Test timeout behavior with a long query + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + # This should timeout after 1 second + with pytest.raises(Exception) as exc_info: + cur.execute("SELECT pg_sleep(2)") + + # Verify it's a timeout-related exception + error_msg = str(exc_info.value).lower() + assert any(keyword in error_msg for keyword in ["timeout", "statement_timeout", "canceling"]) + + finally: + manager.close_all() + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_connection_termination_and_recovery(pg_instance): + """ + Simulates a server-side termination of a connection and verifies the pool + replaces it and continues working. + """ + conn_args = _create_conn_args(pg_instance, "test_connection_termination_and_recovery") + + manager = LRUConnectionPoolManager( + max_db=1, + base_conn_args=conn_args, + pool_config={ + "min_size": 0, + "max_size": 1, + "max_idle": 60, + "open": True, + }, + ) + + dbname = pg_instance["dbname"] + + try: + # Open a connection and get its PID + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT pg_backend_pid()") + pid = cur.fetchone()[0] + print(f"Target connection PID: {pid}") + + # Terminate the connection from superuser + with _get_superconn(pg_instance) as superconn: + with superconn.cursor() as cur: + cur.execute("SELECT pg_terminate_backend(%s)", (pid,)) + result = cur.fetchone()[0] + assert result is True, "pg_terminate_backend() failed" + + # Try using the connection again + try: + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 42") + assert cur.fetchone()[0] == 42 + except AdminShutdown: + # Connection was terminated, psycopg3 will not retry and raise an AdminShutdown error + # But we should still be able to successfully open a new connection next attempt + # This should be caught and handled by the application using the connection + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 42") + assert cur.fetchone()[0] == 42 + + stats = manager.get_pool_stats(dbname) + assert stats["connections_num"] >= 2, "Expected connection to be reopened after termination" + + finally: + manager.close_all() + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_persistent_pool_eviction_behavior(pg_instance): + """ + Ensures persistent pools are not evicted while non-persistent ones are available. + """ + conn_args = _create_conn_args(pg_instance, "test_persistent_pool_eviction_behavior") + + manager = LRUConnectionPoolManager(max_db=3, base_conn_args=conn_args) + + try: + # Fill pool with 3 connections + dbs = ["dogs_0", "dogs_1", "dogs_2"] + with manager.get_connection(dbs[0]): # non-persistent + pass + with manager.get_connection(dbs[1]): # non-persistent + pass + with manager.get_connection(dbs[2], persistent=True): # persistent + pass + + # Trigger eviction by adding one more + with manager.get_connection("dogs_3"): + pass + + pool_keys = list(manager.pools.keys()) + + # Assert persistent pool still exists + assert "dogs_2" in pool_keys + + # One of the non-persistent dbs must be evicted + evicted = set(dbs[:2]) - set(pool_keys) + assert len(evicted) == 1, "One non-persistent db should have been evicted" + + # Final pool should contain 3 entries + assert len(pool_keys) == 3 + + finally: + manager.close_all() + assert len(manager.pools) == 0 + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_eviction_when_all_pools_are_persistent(pg_instance): + """ + Ensures that if all existing pools are persistent, the least recently used one + is still evicted to make room for a new pool. + """ + conn_args = _create_conn_args(pg_instance, "test_eviction_when_all_pools_are_persistent") + + manager = LRUConnectionPoolManager(max_db=3, base_conn_args=conn_args) + + try: + # Open 3 persistent pools + dbs = ["dogs_0", "dogs_1", "dogs_2"] + for db in dbs: + with manager.get_connection(db, persistent=True): + pass + + # Re-access dogs_1 to update its recency + with manager.get_connection("dogs_1"): + pass + + # dogs_0 is now the least recently used persistent pool + + # Open a new pool to trigger eviction + with manager.get_connection("dogs_3", persistent=True): + pass + + current_pools = list(manager.pools.keys()) + + # Should have evicted dogs_0 + assert "dogs_0" not in current_pools, "Expected least recently used persistent pool to be evicted" + assert "dogs_1" in current_pools + assert "dogs_2" in current_pools + assert "dogs_3" in current_pools + assert len(current_pools) == 3 + + finally: + manager.close_all() + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_connection_proxy_exception_handling(pg_instance: Dict[str, str]): + """ + Test that ConnectionProxy properly handles exceptions and releases connections. + """ + conn_args = _create_conn_args(pg_instance, "test_proxy_exceptions") + + manager = LRUConnectionPoolManager(max_db=1, base_conn_args=conn_args) + + try: + dbname = pg_instance["dbname"] + + # Test normal usage first + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + result = cur.fetchone() + assert result == (1,) + + # Test exception handling - this should raise an exception + with pytest.raises(Exception) as exc_info: + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT * FROM nonexistent_table") + + # Verify it's a database-related exception + error_msg = str(exc_info.value).lower() + assert any(keyword in error_msg for keyword in ["relation", "table", "nonexistent", "does not exist"]) + + # Verify connection is still available after exception + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT 1") + result = cur.fetchone() + assert result == (1,) + + # Test that pool stats are still valid after exception + stats = manager.get_pool_stats(dbname) + assert stats is not None + assert stats["pool_available"] == 1 + + finally: + manager.close_all() + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_concurrent_access_and_thread_safety(pg_instance: Dict[str, str]): + """ + Test that the pool manager handles concurrent access safely from multiple threads. + """ + conn_args = _create_conn_args(pg_instance, "test_concurrent") + + manager = LRUConnectionPoolManager(max_db=2, base_conn_args=conn_args) + results = queue.Queue() + + def worker(dbname): + """Worker function that accesses a database and reports results.""" + try: + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT pg_sleep(0.2), 1") # simulate work + result = cur.fetchone() + if result[-1] == 1: + results.put(f"success_{dbname}") + else: + results.put(f"error_{dbname}: unexpected result {result}") + except Exception as e: + results.put(f"error_{dbname}: {e}") + + try: + # Start multiple threads accessing different databases + threads = [] + for i in range(10): + dbname = f"dogs_{i % 2}" # Only 2 unique dbs, 10 threads, to test connection contention + thread = threading.Thread(target=worker, args=(dbname,)) + threads.append(thread) + thread.start() + + # Wait for all threads to complete + for thread in threads: + thread.join() + + # Verify all operations succeeded + success_count = 0 + error_count = 0 + while not results.empty(): + result = results.get() + if result.startswith("success_"): + success_count += 1 + else: + error_count += 1 + print(f"Thread error: {result}") + + assert success_count == 10, ( + f"Expected all 10 operations to succeed, got {success_count} successes and {error_count} errors" + ) + + # Verify pool limits are respected + assert len(manager.pools) <= 5, f"Expected max 5 pools, got {len(manager.pools)}" + + # Verify all pools are functional after concurrent access + for dbname in list(manager.pools.keys()): + stats = manager.get_pool_stats(dbname) + assert stats is not None, f"Pool stats should be available for {dbname}" + assert stats["pool_available"] == 2, f"Pool should have available connections for {dbname}" + assert stats["pool_size"] == 2, f"Pool should have size 2 for {dbname}" + + finally: + manager.close_all() + assert len(manager.pools) == 0, "All pools should be closed" + + +@pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +def test_commenter_cursor_functionality(pg_instance: Dict[str, str]): + """ + Test that CommenterCursor properly prepends SQL comments and handles ignore_query_metric parameter + when used with LRUConnectionPoolManager. + """ + conn_args = _create_conn_args(pg_instance, "test_commenter_cursor") + manager = LRUConnectionPoolManager(max_db=1, base_conn_args=conn_args, pool_config={"min_size": 1, "max_size": 1}) + + try: + dbname = pg_instance["dbname"] + + # Test normal query execution with CommenterCursor + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT generate_series(1, 5) AS number") + result = cur.fetchall() + assert len(result) == 5 + assert all(isinstance(row[0], int) for row in result) + + # Verify SQL comment is prepended + _verify_sql_comment_prepended(pg_instance, "generate_series", False) + + # Test query with ignore_query_metric=True + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT generate_series(1, 3) AS number", ignore_query_metric=True) + result = cur.fetchall() + assert len(result) == 3 + assert all(isinstance(row[0], int) for row in result) + + # Verify SQL comment with DDIGNORE is prepended + _verify_sql_comment_prepended(pg_instance, "generate_series", True) + + # Test multiple queries to ensure CommenterCursor works consistently + with manager.get_connection(dbname) as conn: + with conn.cursor() as cur: + cur.execute("SELECT current_database()") + result = cur.fetchone() + assert result[0] == dbname + + _verify_sql_comment_prepended(pg_instance, "current_database", False) + + finally: + manager.close_all() + + +def _verify_sql_comment_prepended(pg_instance, query_pattern, ignore_query_metric): + """ + Verify that SQL comments are properly prepended to queries in pg_stat_activity. + """ + super_conn = _get_superconn(pg_instance) + try: + with super_conn.cursor() as cursor: + cursor.execute( + ( + "SELECT query FROM pg_stat_activity WHERE query LIKE %s " + "AND query NOT LIKE '%%pg_stat_activity%%' " + "ORDER BY query_start DESC LIMIT 1" + ), + (f"%{query_pattern}%",), + ) + result = cursor.fetchall() + assert len(result) > 0, f"No queries found matching pattern '{query_pattern}'" + + query_text = result[0][0] + # Decode bytes to string if necessary + if isinstance(query_text, bytes): + query_text = query_text.decode('utf-8') + + expected_comment = "/* service='datadog-agent' */" + + if ignore_query_metric: + expected_comment = f"/* DDIGNORE */ {expected_comment}" + + assert query_text.startswith(expected_comment), ( + f"Query should start with '{expected_comment}', but got: {query_text[:100]}..." + ) + finally: + super_conn.close() + + +def test_closed_state_and_pool_creation_prevention(): + """ + Test that the pool manager correctly tracks closed state and prevents new pool creation after closing. + + This test verifies: + 1. The is_closed() method returns the correct state + 2. New pools cannot be created after close_all() is called + 3. New connections cannot be created after close_all() is called + """ + # Create a pool manager with mock connection args (no real DB needed for this test) + conn_args = PostgresConnectionArgs( + application_name="test_closed_state", + user="testuser", + password="testpass", + host="localhost", + port=5432, + ) + + manager = LRUConnectionPoolManager(max_db=2, base_conn_args=conn_args) + + # Initially should not be closed + assert not manager.is_closed() + + # Close the manager + manager.close_all() + + # Should now be closed + assert manager.is_closed() + + # Attempting to get a connection should raise RuntimeError + with pytest.raises(RuntimeError, match="Pool manager is closed and cannot get connection"): + manager.get_connection("testdb") + + # Calling close_all() again should not cause issues (idempotent) + manager.close_all() + assert manager.is_closed() + + # Still should not be able to create new connections + with pytest.raises(RuntimeError, match="Pool manager is closed and cannot get connection"): + manager.get_connection("testdb") diff --git a/postgres/tests/test_connections.py b/postgres/tests/test_connections.py deleted file mode 100644 index 670373fd772cb..0000000000000 --- a/postgres/tests/test_connections.py +++ /dev/null @@ -1,372 +0,0 @@ -# (C) Datadog, Inc. 2023-present -# All rights reserved -# Licensed under a 3-clause BSD style license (see LICENSE) -import datetime -import pprint -import threading -import time -import uuid - -import psycopg2 -import pytest - -from datadog_checks.postgres import PostgreSql -from datadog_checks.postgres.connections import ConnectionPoolFullError, MultiDatabaseConnectionPool -from datadog_checks.postgres.util import DatabaseHealthCheckError - -from .common import HOST, PASSWORD_ADMIN, USER_ADMIN -from .utils import _get_superconn - - -@pytest.mark.integration -@pytest.mark.usefixtures('dd_environment') -def test_conn_pool(pg_instance): - """ - Test simple case of creating a connection pool, pruning a stale connection, - and closing all connections. - """ - check = PostgreSql('postgres', {}, [pg_instance]) - - pool = MultiDatabaseConnectionPool(check._new_connection) - db = pool._get_connection_raw('postgres', 1) - assert pool._stats.connection_opened == 1 - pool.prune_connections() - assert len(pool._conns) == 1 - assert pool._stats.connection_closed == 0 - - with db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor: - cursor.execute("select 1") - rows = cursor.fetchall() - assert len(rows) == 1 and rows[0][0] == 1 - - time.sleep(0.001) - pool.prune_connections() - assert len(pool._conns) == 0 - assert pool._stats.connection_closed == 1 - assert pool._stats.connection_closed_failed == 0 - assert pool._stats.connection_pruned == 1 - - db = pool._get_connection_raw('postgres', 999 * 1000) - assert len(pool._conns) == 1 - assert pool._stats.connection_opened == 2 - success = pool.close_all_connections() - assert success - assert len(pool._conns) == 0 - assert pool._stats.connection_closed == 2 - assert pool._stats.connection_closed_failed == 0 - assert pool._stats.connection_pruned == 1 - - -@pytest.mark.integration -@pytest.mark.usefixtures('dd_environment') -def test_conn_pool_no_leaks_on_close(pg_instance): - """ - Test a simple case of opening and closing many connections. There should be no leaked connections on the server. - """ - unique_id = str(uuid.uuid4()) # Used to isolate this test from others on the DB - - check = PostgreSql('postgres', {}, [pg_instance]) - check._config.application_name = unique_id - - # Used to make verification queries - pool2 = MultiDatabaseConnectionPool( - lambda dbname: psycopg2.connect(host=HOST, dbname=dbname, user=USER_ADMIN, password=PASSWORD_ADMIN) - ) - - # Iterate in the test many times to detect flakiness - for _ in range(20): - pool = MultiDatabaseConnectionPool(check._new_connection) - - def get_activity(): - """ - Fetches all pg_stat_activity rows generated by this test and connection to a "dogs%" database - """ - with pool2.get_connection('postgres', 1) as conn: - cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute( - "SELECT pid, datname, usename, state, query_start, state_change, application_name" - " FROM pg_stat_activity" - " WHERE datname LIKE 'dogs%%' AND application_name = %s", - (unique_id,), - ) - return cursor.fetchall() - - conn_count = 100 - for i in range(0, conn_count): - dbname = 'dogs_{}'.format(i) - db = pool._get_connection_raw(dbname, 10 * 1000) - with db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor: - cursor.execute("select current_database()") - rows = cursor.fetchall() - assert len(rows) == 1 - assert rows[0][0] == dbname - - assert pool._stats.connection_opened == conn_count - assert len(get_activity()) == conn_count - - pool.close_all_connections() - assert pool._stats.connection_closed == conn_count - assert pool._stats.connection_closed_failed == 0 - - # Ensure all the connections have been terminated on the server - attempts = 5 - while True: - attempts -= 1 - - rows = get_activity() - if len(rows) == 0: - break - - assert attempts >= 0, "Connections leaked! Leaked rows found:\n{}".format(pprint.pformat(rows)) - time.sleep(1) - - -@pytest.mark.integration -@pytest.mark.usefixtures('dd_environment') -def test_conn_pool_no_leaks_on_prune(pg_instance): - """ - Test a scenario where many connections are created. These connections should be open on the database - then should properly close on the pooler side and database when pruned and/or closed. - """ - unique_id = str(uuid.uuid4()) # Used to isolate this test from others on the DB - - check = PostgreSql('postgres', {}, [pg_instance]) - check._config.application_name = unique_id - - pool = MultiDatabaseConnectionPool(check._new_connection) - # Used to make verification queries - pool2 = MultiDatabaseConnectionPool( - lambda dbname: psycopg2.connect(host=HOST, dbname=dbname, user=USER_ADMIN, password=PASSWORD_ADMIN) - ) - ttl_long = 90 * 1000 - ttl_short = 1 - - def get_activity(): - """ - Fetches all pg_stat_activity rows generated by this test and connection to a "dogs%" database - """ - with pool2.get_connection('postgres', 1) as conn: - cursor = conn.cursor(cursor_factory=psycopg2.extras.DictCursor) - cursor.execute( - "SELECT pid, datname, usename, state, query_start, state_change, application_name" - " FROM pg_stat_activity" - " WHERE datname LIKE 'dogs%%' AND application_name = %s", - (unique_id,), - ) - return cursor.fetchall() - - def get_many_connections(count, ttl): - """ - Retrieves the number of connections from the pool with the specified TTL - """ - for i in range(0, count): - dbname = 'dogs_{}'.format(i) - db = pool._get_connection_raw(dbname, ttl) - with db.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor: - cursor.execute("select current_database()") - rows = cursor.fetchall() - assert len(rows) == 1 - assert rows[0][0] == dbname - - pool.close_all_connections() - - pool._stats.reset() - - # Create many connections with long-lived TTLs - get_many_connections(50, ttl_long) - assert len(pool._conns) == 50 - assert pool._stats.connection_opened == 50 - # Ensure those connections have the correct deadline and connection status - for i in range(0, 50): - dbname = 'dogs_{}'.format(i) - conn_info = pool._conns[dbname] - db = conn_info.connection - deadline = conn_info.deadline - approximate_deadline = datetime.datetime.now() + datetime.timedelta(milliseconds=ttl_long) - assert ( - approximate_deadline - datetime.timedelta(seconds=2) - < deadline - < approximate_deadline + datetime.timedelta(seconds=2) - ) - assert not db.closed - assert db.status == psycopg2.extensions.STATUS_READY - # Check that those pooled connections do exist on the database - rows = get_activity() - assert len(rows) == 50 - assert len({row['datname'] for row in rows}) == 50 - assert all(row['state'] == 'idle' for row in rows) - - pool._stats.reset() - - # Repeat this process many times and expect that only one connection is created per database - for _ in range(100): - get_many_connections(51, ttl_long) - assert pool._stats.connection_opened == 1 - - attempts_to_verify = 10 - # Loop here to prevent flakiness. Sometimes postgres doesn't immediately terminate backends. - # The test can be considered successful as long as the backend is eventually terminated. - for attempt in range(attempts_to_verify): - rows = get_activity() - server_pids = {row['pid'] for row in rows} - conns = [c.connection for c in pool._conns.values()] - conn_pids = {db.info.backend_pid for db in conns} - leaked_rows = [row for row in rows if row['pid'] in server_pids - conn_pids] - if not leaked_rows: - break - if attempt < attempts_to_verify - 1: - time.sleep(1) - continue - assert len(leaked_rows) == 0, 'Found leaked rows on the server not in the connection pool' - - assert len({row['datname'] for row in rows}) == 51 - assert len(rows) == 51, 'Possible leaked connections' - assert all(row['state'] == 'idle' for row in rows) - assert pool._stats.connection_opened == 1 - assert pool._stats.connection_closed == 0 - - pool._stats.reset() - - # Now update db connections with short-lived TTLs and expect them to self-prune - get_many_connections(55, ttl_short) - time.sleep(0.001) - pool.prune_connections() - - assert pool._stats.connection_opened == 55 - 51 - assert pool._stats.connection_closed == 55 - assert pool._stats.connection_pruned == 55 - assert pool._stats.connection_closed_failed == 0 - attempts_to_verify = 10 - for attempt in range(attempts_to_verify): - leaked_rows = get_activity() - if attempt < attempts_to_verify - 1: - time.sleep(1) - continue - assert len(leaked_rows) == 0, 'Found leaked rows remaining after TTL was updated to short TTL' - - # Final check that the server contains no leaked connections still open - rows = get_activity() - assert len(rows) == 0 - - -@pytest.mark.integration -@pytest.mark.usefixtures('dd_environment') -def test_conn_pool_single_context(pg_instance): - """ - Test creating a single connection. - """ - check = PostgreSql('postgres', {}, [pg_instance]) - - pool = MultiDatabaseConnectionPool(check._new_connection) - with pool.get_connection("dogs_0", 1000): - pass - - assert pool._stats.connection_opened == 1 - - expected_evicted = "dogs_0" - evicted = pool.evict_lru() - assert evicted == expected_evicted - assert pool._stats.connection_closed == 1 - - # ask for another connection again, error not raised - with pool.get_connection("dogs_1", 1000): - pass - - -@pytest.mark.integration -@pytest.mark.usefixtures('dd_environment') -def test_conn_pool_context_managed(pg_instance): - """ - Test context manager API for connection grabbing. - """ - - def pretend_to_run_query(pool, dbname): - with pool.get_connection(dbname, 10000): - time.sleep(5) - - limit = 30 - check = PostgreSql('postgres', {}, [pg_instance]) - - pool = MultiDatabaseConnectionPool(check._new_connection, limit) - threadpool = [] - for i in range(limit): - thread = threading.Thread(target=pretend_to_run_query, args=(pool, 'dogs_{}'.format(i))) - threadpool.append(thread) - thread.start() - - time.sleep(1) - assert pool._stats.connection_opened == limit - - # ask for one more connection - with pytest.raises(ConnectionPoolFullError): - with pool.get_connection('dogs_{}'.format(limit + 1), 1, 1): - pass - - # join threads - for thread in threadpool: - thread.join() - - # now can add a new connection, one will get kicked out of pool - with pool.get_connection('dogs_{}'.format(limit + 1), 60000): - pass - - assert pool._stats.connection_closed == 1 - - # close the rest - pool.close_all_connections() - assert pool._stats.connection_closed == limit + 1 - - -@pytest.mark.integration -@pytest.mark.usefixtures('dd_environment') -def test_conn_terminated_prematurely(pg_instance): - """ - Test db connection terminated prematurely - """ - - def _terminate_connection(conn, dbname): - # function to abruptly terminate a connection - with conn.cursor() as cursor: - cursor.execute( - "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND datname = %s", - (dbname,), - ) - - check = PostgreSql('postgres', {}, [pg_instance]) - conn = _get_superconn(pg_instance) - - check.check(pg_instance) - assert check._db is not None - assert not check._db.closed - - _terminate_connection(conn, pg_instance['dbname']) - time.sleep(1) - - assert check._db is not None - - # the connection is terminated on the server, psycopg has no way of knowing - # this check run will fail but the connection status should be updated - with pytest.raises(DatabaseHealthCheckError): - check.check(pg_instance) - - # connection status is updated to closed - assert check._db is not None - assert check._db.closed - - # new check run will re-open connection - check.check(pg_instance) - - -@pytest.mark.integration -@pytest.mark.usefixtures('dd_environment') -def test_conn_statement_timeout(pg_instance): - """ - Test db connection statement timeout is set at the session level - """ - pg_instance["query_timeout"] = 500 - check = PostgreSql('postgres', {}, [pg_instance]) - check._connect() - with pytest.raises(psycopg2.errors.QueryCanceled): - with check.db() as conn: - with conn.cursor() as cursor: - cursor.execute("SELECT pg_sleep(1)") diff --git a/postgres/tests/test_cursor.py b/postgres/tests/test_cursor.py index 97e81300e2715..2482d4d5b0c68 100644 --- a/postgres/tests/test_cursor.py +++ b/postgres/tests/test_cursor.py @@ -3,8 +3,6 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import pytest -from datadog_checks.postgres.cursor import CommenterCursor, CommenterDictCursor - from .utils import _get_superconn @@ -16,8 +14,8 @@ def test_integration_connection_with_commenter_cursor(integration_check, pg_inst check = integration_check(pg_instance) with check.db() as conn: - # verify CommenterCursor and CommenterDictCursor prepend the query with /* service='datadog-agent' */ - with conn.cursor(cursor_factory=CommenterCursor) as cursor: + conn.execute("SET client_encoding TO 'UTF8'") + with conn.cursor() as cursor: cursor.execute( 'SELECT generate_series(1, 10) AS number', ignore_query_metric=ignore, @@ -26,7 +24,7 @@ def test_integration_connection_with_commenter_cursor(integration_check, pg_inst assert isinstance(result[0], int) __check_prepand_sql_comment(pg_instance, ignore) - with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: + with conn.cursor() as cursor: cursor.execute( 'SELECT generate_series(1, 10) AS number', ignore_query_metric=ignore, @@ -42,6 +40,7 @@ def __check_prepand_sql_comment(pg_instance, ignore): # collect query_text from pg_stat_activity # assert /* service='datadog-agent' */ is present in the query super_conn = _get_superconn(pg_instance) + super_conn.execute("SET client_encoding TO 'UTF8'") with super_conn.cursor() as cursor: cursor.execute( ( diff --git a/postgres/tests/test_deadlock.py b/postgres/tests/test_deadlock.py index 1fbb8d1d70ccc..bc0a6a48112b3 100644 --- a/postgres/tests/test_deadlock.py +++ b/postgres/tests/test_deadlock.py @@ -2,12 +2,12 @@ # All rights reserved # Licensed under Simplified BSD License (see LICENSE) -import copy -import select +import threading import time -import psycopg2 +import psycopg import pytest +from psycopg import ClientCursor from .common import DB_NAME, HOST, POSTGRES_VERSION, _get_expected_tags @@ -48,27 +48,17 @@ def test_deadlock(aggregator, dd_run_check, integration_check, pg_instance): conn = check._new_connection(pg_instance['dbname']) cursor = conn.cursor() - def wait(conn): - while True: - state = conn.poll() - if state == psycopg2.extensions.POLL_OK: - break - elif state == psycopg2.extensions.POLL_WRITE: - select.select([], [conn.fileno()], []) - elif state == psycopg2.extensions.POLL_READ: - select.select([conn.fileno()], [], []) - else: - raise psycopg2.OperationalError("poll() returned %s" % state) - time.sleep(0.1) - - conn_args = {'host': HOST, 'dbname': DB_NAME, 'user': "bob", 'password': "bob"} - conn_args_async = copy.copy(conn_args) - conn_args_async["async_"] = 1 - conn1 = psycopg2.connect(**conn_args) - conn1.autocommit = False - - conn2 = psycopg2.connect(**conn_args_async) - wait(conn2) + def execute_in_thread(q, args): + with psycopg.connect( + host=HOST, dbname=DB_NAME, user="bob", password="bob", cursor_factory=ClientCursor + ) as tconn: + with tconn.cursor() as cur: + # this will block, and eventually throw when + # the deadlock is created + try: + cur.execute(q, args) + except psycopg.errors.DeadlockDetected: + pass appname = 'deadlock sess' appname1 = appname + '1' @@ -80,20 +70,23 @@ def wait(conn): cursor.execute(deadlock_count_sql, (DB_NAME,)) deadlocks_before = cursor.fetchone()[0] + conn_args = {'host': HOST, 'dbname': DB_NAME, 'user': "bob", 'password': "bob"} + conn1 = psycopg.connect(**conn_args, autocommit=False, cursor_factory=ClientCursor) + cur1 = conn1.cursor() cur1.execute(appname_sql, (appname1,)) cur1.execute(update_sql, (1,)) - cur2 = conn2.cursor() - cur2.execute( - """SET application_name=%s; + args = (appname2, 2, 1) + query = """SET application_name=%s; begin transaction; {}; {}; commit; -""".format(update_sql, update_sql), - (appname2, 2, 1), - ) +""".format(update_sql, update_sql) + # ... now execute the test query in a separate thread + lock_task = threading.Thread(target=execute_in_thread, args=(query, args)) + lock_task.start() lock_count_sql = """SELECT COUNT(1) FROM pg_catalog.pg_locks blocked_locks @@ -123,11 +116,10 @@ def wait(conn): try: cur1.execute(update_sql, (2,)) cur1.execute("commit") - except psycopg2.errors.DeadlockDetected: + except psycopg.errors.DeadlockDetected: pass conn1.close() - conn2.close() dd_run_check(check) diff --git a/postgres/tests/test_discovery.py b/postgres/tests/test_discovery.py index b80a5b0e67024..988d50a0f18ff 100644 --- a/postgres/tests/test_discovery.py +++ b/postgres/tests/test_discovery.py @@ -8,8 +8,8 @@ import time from contextlib import contextmanager -import psycopg2 -import psycopg2.sql +import psycopg +import psycopg.sql import pytest from datadog_checks.base import ConfigurationError @@ -75,7 +75,7 @@ @contextmanager def get_postgres_connection(dbname="postgres"): conn_args = {'host': HOST, 'dbname': dbname, 'user': USER_ADMIN, 'password': PASSWORD_ADMIN} - conn = psycopg2.connect(**conn_args) + conn = psycopg.connect(**conn_args) conn.autocommit = True yield conn @@ -162,7 +162,7 @@ def test_autodiscovery_refresh(integration_check, pg_instance): del pg_instance['dbname'] pg_instance["database_autodiscovery"]['refresh'] = 1 check = integration_check(pg_instance) - run_one_check(check) + run_one_check(check, cancel=False) assert check.autodiscovery is not None databases = check.autodiscovery.get_items() @@ -172,7 +172,7 @@ def test_autodiscovery_refresh(integration_check, pg_instance): with get_postgres_connection() as conn: cursor = conn.cursor() try: - cursor.execute(psycopg2.sql.SQL("CREATE DATABASE {}").format(psycopg2.sql.Identifier(database_to_find))) + cursor.execute(psycopg.sql.SQL("CREATE DATABASE {}").format(psycopg.sql.Identifier(database_to_find))) time.sleep(pg_instance["database_autodiscovery"]['refresh']) databases = check.autodiscovery.get_items() @@ -180,7 +180,7 @@ def test_autodiscovery_refresh(integration_check, pg_instance): finally: # Need to drop the new database to clean up the environment for next tests. cursor.execute( - psycopg2.sql.SQL("DROP DATABASE {} WITH (FORCE);").format(psycopg2.sql.Identifier(database_to_find)) + psycopg.sql.SQL("DROP DATABASE {} WITH (FORCE);").format(psycopg.sql.Identifier(database_to_find)) ) @@ -304,7 +304,7 @@ def _set_allow_connection(dbname: str, allow: bool): with get_postgres_connection() as conn: cursor = conn.cursor() cursor.execute( - psycopg2.sql.SQL("UPDATE pg_database SET datallowconn = %s WHERE datname = %s;"), + psycopg.sql.SQL("UPDATE pg_database SET datallowconn = %s WHERE datname = %s;"), (allow, dbname), ) conn.commit() diff --git a/postgres/tests/test_e2e.py b/postgres/tests/test_e2e.py index 69558eeb6ea00..587df431c1608 100644 --- a/postgres/tests/test_e2e.py +++ b/postgres/tests/test_e2e.py @@ -15,6 +15,7 @@ def test_e2e(check, dd_agent_check, pg_instance): aggregator = dd_agent_check(pg_instance, rate=True) conn = _get_conn(pg_instance) + conn.execute("SET client_encoding TO 'UTF8'") with conn.cursor() as cur: cur.execute("SHOW server_version;") check.raw_version = cur.fetchone()[0] diff --git a/postgres/tests/test_explain_parameterized_queries.py b/postgres/tests/test_explain_parameterized_queries.py index b92463143dbcb..8fd866bb29748 100644 --- a/postgres/tests/test_explain_parameterized_queries.py +++ b/postgres/tests/test_explain_parameterized_queries.py @@ -4,7 +4,7 @@ from unittest import mock -import psycopg2 +import psycopg import pytest from datadog_checks.base.utils.db.sql import compute_sql_signature @@ -12,6 +12,7 @@ from datadog_checks.postgres.version_utils import V12 from .common import DB_NAME +from .utils import requires_over_12 @pytest.fixture @@ -32,6 +33,8 @@ def dbm_instance(pg_instance): @pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +@requires_over_12 @pytest.mark.parametrize( "query,expected_explain_err_code", [ @@ -45,11 +48,7 @@ def dbm_instance(pg_instance): ) def test_explain_parameterized_queries(integration_check, dbm_instance, query, expected_explain_err_code): check = integration_check(dbm_instance) - check._connect() - check.check(dbm_instance) - if check.version < V12: - return plan_dict, explain_err_code, err = check.statement_samples._run_and_track_explain( DB_NAME, query, query, "7231596c8b5536d1" @@ -60,16 +59,19 @@ def test_explain_parameterized_queries(integration_check, dbm_instance, query, e explain_param_queries = check.statement_samples._explain_parameterized_queries # check that we deallocated the prepared statement after explaining - rows = explain_param_queries._execute_query_and_fetch_rows( - DB_NAME, - "SELECT * FROM pg_prepared_statements WHERE name = 'dd_{query_signature}'".format( - query_signature=compute_sql_signature(query) - ), - ) + with check.db_pool.get_connection(DB_NAME) as conn: + rows = explain_param_queries._execute_query_and_fetch_rows( + conn, + "SELECT * FROM pg_prepared_statements WHERE name = 'dd_{query_signature}'".format( + query_signature=compute_sql_signature(query) + ), + ) assert len(rows) == 0 @pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +@requires_over_12 @pytest.mark.parametrize( "query,expected_generic_values", [ @@ -83,30 +85,25 @@ def test_explain_parameterized_queries(integration_check, dbm_instance, query, e ) def test_explain_parameterized_queries_generic_params(integration_check, dbm_instance, query, expected_generic_values): check = integration_check(dbm_instance) - check._connect() - - check.check(dbm_instance) - if check.version < V12: - return query_signature = compute_sql_signature(query) explain_param_queries = check.statement_samples._explain_parameterized_queries - explain_param_queries._create_prepared_statement(DB_NAME, query, query, query_signature) - assert expected_generic_values == explain_param_queries._get_number_of_parameters_for_prepared_statement( - DB_NAME, query_signature - ) + with check.db_pool.get_connection(DB_NAME) as conn: + explain_param_queries._create_prepared_statement(conn, query, query, query_signature) + assert expected_generic_values == explain_param_queries._get_number_of_parameters_for_prepared_statement( + conn, query_signature + ) @pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") def test_explain_parameterized_queries_version_below_12(integration_check, dbm_instance): ''' For postgres versions below 12, we do not support explaining parameterized queries, because plan_cache_mode is not supported. We should return proper error. ''' check = integration_check(dbm_instance) - check._connect() - check.check(dbm_instance) if check.version >= V12: # this test is for versions below 12 to make sure we return proper error for unsupported versions @@ -121,21 +118,19 @@ def test_explain_parameterized_queries_version_below_12(integration_check, dbm_i assert plan_dict is None assert explain_err_code == DBExplainError.parameterized_query assert err is not None - assert err == "" + assert err == "" @pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +@requires_over_12 def test_explain_parameterized_queries_create_prepared_statement_exception(integration_check, dbm_instance): check = integration_check(dbm_instance) - check._connect() - check.check(dbm_instance) - if check.version < V12: - return with mock.patch( 'datadog_checks.postgres.explain_parameterized_queries.ExplainParameterizedQueries._create_prepared_statement', - side_effect=psycopg2.errors.DatabaseError("unexpected exception"), + side_effect=psycopg.errors.DatabaseError("unexpected exception"), ): plan_dict, explain_err_code, err = check.statement_samples._run_and_track_explain( DB_NAME, @@ -146,21 +141,20 @@ def test_explain_parameterized_queries_create_prepared_statement_exception(integ assert plan_dict is None assert explain_err_code == DBExplainError.failed_to_explain_with_prepared_statement assert err is not None - assert err == "" + assert err == "" @pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +@requires_over_12 def test_explain_parameterized_queries_explain_prepared_statement_exception(integration_check, dbm_instance): check = integration_check(dbm_instance) - check._connect() check.check(dbm_instance) - if check.version < V12: - return with mock.patch( 'datadog_checks.postgres.explain_parameterized_queries.ExplainParameterizedQueries._explain_prepared_statement', - side_effect=psycopg2.errors.DatabaseError("unexpected exception"), + side_effect=psycopg.errors.DatabaseError("unexpected exception"), ): query = "SELECT * FROM pg_settings WHERE name = $1" plan_dict, explain_err_code, err = check.statement_samples._run_and_track_explain( @@ -169,25 +163,25 @@ def test_explain_parameterized_queries_explain_prepared_statement_exception(inte assert plan_dict is None assert explain_err_code == DBExplainError.failed_to_explain_with_prepared_statement assert err is not None - assert err == "" - # check that we deallocated the prepared statement after explaining - rows = check.statement_samples._explain_parameterized_queries._execute_query_and_fetch_rows( - DB_NAME, - "SELECT * FROM pg_prepared_statements WHERE name = 'dd_{query_signature}'".format( - query_signature=compute_sql_signature(query) - ), - ) + assert err == "" + with check.db_pool.get_connection(DB_NAME) as conn: + # check that we deallocated the prepared statement after explaining + rows = check.statement_samples._explain_parameterized_queries._execute_query_and_fetch_rows( + conn, + "SELECT * FROM pg_prepared_statements WHERE name = 'dd_{query_signature}'".format( + query_signature=compute_sql_signature(query) + ), + ) assert len(rows) == 0 @pytest.mark.integration +@pytest.mark.usefixtures("dd_environment") +@requires_over_12 def test_explain_parameterized_queries_explain_prepared_statement_no_plan_returned(integration_check, dbm_instance): check = integration_check(dbm_instance) - check._connect() check.check(dbm_instance) - if check.version < V12: - return with mock.patch( 'datadog_checks.postgres.explain_parameterized_queries.ExplainParameterizedQueries._execute_query_and_fetch_rows', @@ -205,57 +199,45 @@ def test_explain_parameterized_queries_explain_prepared_statement_no_plan_return @pytest.mark.unit +@requires_over_12 def test_generate_prepared_statement_query_no_parameters(integration_check, dbm_instance): check = integration_check(dbm_instance) - check._connect() test_query_signature = "12345678" - check.check(dbm_instance) - if check.version < V12: - return - with mock.patch( 'datadog_checks.postgres.explain_parameterized_queries.ExplainParameterizedQueries._get_number_of_parameters_for_prepared_statement', return_value=0, ): prepared_statement_query = ( check.statement_samples._explain_parameterized_queries._generate_prepared_statement_query( - DB_NAME, test_query_signature + None, test_query_signature ) ) assert prepared_statement_query == f"EXECUTE dd_{test_query_signature}" @pytest.mark.unit +@requires_over_12 def test_generate_prepared_statement_query_three_parameters(integration_check, dbm_instance): check = integration_check(dbm_instance) - check._connect() test_query_signature = "12345678" - check.check(dbm_instance) - if check.version < V12: - return - with mock.patch( 'datadog_checks.postgres.explain_parameterized_queries.ExplainParameterizedQueries._get_number_of_parameters_for_prepared_statement', return_value=3, ): prepared_statement_query = ( check.statement_samples._explain_parameterized_queries._generate_prepared_statement_query( - DB_NAME, test_query_signature + None, test_query_signature ) ) assert prepared_statement_query == f"EXECUTE dd_{test_query_signature}(null,null,null)" -@pytest.mark.integration +@pytest.mark.unit +@requires_over_12 def test_create_prepared_statement_exception(integration_check, dbm_instance): check = integration_check(dbm_instance) - check._connect() - - check.check(dbm_instance) - if check.version < V12: - return query = "SELECT * FROM pg_settings WHERE name = $1" query_signature = compute_sql_signature(query) @@ -269,7 +251,7 @@ def test_create_prepared_statement_exception(integration_check, dbm_instance): ) -@pytest.mark.integration +@pytest.mark.unit @pytest.mark.parametrize( "query,statement_is_parameterized_query", [ @@ -286,6 +268,5 @@ def test_explain_parameterized_queries_is_parameterized_query( integration_check, dbm_instance, query, statement_is_parameterized_query ): check = integration_check(dbm_instance) - check._connect() explain_param_queries = check.statement_samples._explain_parameterized_queries assert statement_is_parameterized_query == explain_param_queries._is_parameterized_query(query) diff --git a/postgres/tests/test_pg_integration.py b/postgres/tests/test_pg_integration.py index 88f808a695f0f..7fc9b4792487b 100644 --- a/postgres/tests/test_pg_integration.py +++ b/postgres/tests/test_pg_integration.py @@ -6,7 +6,7 @@ import time import mock -import psycopg2 +import psycopg import pytest from datadog_checks.base.errors import ConfigurationError @@ -130,8 +130,7 @@ def test_initialization_tags(integration_check, pg_instance): def test_snapshot_xmin(aggregator, integration_check, pg_instance): - with psycopg2.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: - conn.set_session(autocommit=True) + with psycopg.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g", autocommit=True) as conn: with conn.cursor() as cur: if float(POSTGRES_VERSION) >= 13.0: query = 'select pg_snapshot_xmin(pg_current_snapshot());' @@ -148,9 +147,7 @@ def test_snapshot_xmin(aggregator, integration_check, pg_instance): aggregator.assert_metric('postgresql.snapshot.xmax', count=1, tags=expected_tags) assert aggregator.metrics('postgresql.snapshot.xmax')[0].value >= xmin - with psycopg2.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: - # Force autocommit - conn.set_session(autocommit=True) + with psycopg.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g", autocommit=True) as conn: with conn.cursor() as cur: _increase_txid(cur) @@ -292,7 +289,7 @@ def test_unsupported_replication(aggregator, integration_check, pg_instance): def format_with_error(value, **kwargs): if 'pg_is_in_recovery' in value: called.append(True) - raise psycopg2.errors.FeatureNotSupported("Not available") + raise psycopg.errors.FeatureNotSupported("Not available") return unpatched_fmt.format(value, **kwargs) # This simulate an error in the fmt function, as it's a bit hard to mock psycopg @@ -325,6 +322,7 @@ def test_can_connect_service_check(aggregator, integration_check, pg_instance): with pytest.raises(AttributeError): check.db = mock.MagicMock(side_effect=AttributeError('foo')) check.check(pg_instance) + # Since we can't connect to the host, we can't gather the replication role tags_without_role = _get_expected_tags( check, pg_instance, with_db=True, with_version=False, with_sys_id=False, with_cluster_name=False, role=None @@ -340,7 +338,7 @@ def test_can_connect_service_check(aggregator, integration_check, pg_instance): # Forth: connection health check failed with pytest.raises(DatabaseHealthCheckError): db = mock.MagicMock() - db.cursor().__enter__().execute.side_effect = psycopg2.OperationalError('foo') + db.cursor().__enter__().execute.side_effect = psycopg.OperationalError('foo') @contextlib.contextmanager def mock_db(): @@ -348,6 +346,7 @@ def mock_db(): check.db = mock_db check.check(pg_instance) + aggregator.assert_service_check('postgres.can_connect', count=1, status=PostgreSql.CRITICAL, tags=tags_without_role) aggregator.reset() @@ -398,7 +397,7 @@ def test_locks_metrics_no_relations(aggregator, integration_check, pg_instance): Since 4.0.0, to prevent tag explosion, lock metrics are not collected anymore unless relations are specified """ check = integration_check(pg_instance) - with psycopg2.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: + with psycopg.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: with conn.cursor() as cur: cur.execute('LOCK persons') check.run() @@ -612,13 +611,12 @@ def test_query_timeout(integration_check, pg_instance): pg_instance['query_timeout'] = 1000 check = integration_check(pg_instance) check._connect() - with pytest.raises(psycopg2.errors.QueryCanceled): + with pytest.raises(psycopg.errors.QueryCanceled): with check.db() as conn: with conn.cursor() as cursor: cursor.execute("select pg_sleep(2000)") -@pytest.mark.flaky(max_runs=10) def test_pg_control(aggregator, integration_check, pg_instance): check = integration_check(pg_instance) check.run() @@ -833,18 +831,6 @@ def test_database_instance_metadata(aggregator, pg_instance, dbm_enabled, report None, False, ), - ( - { - "instance_endpoint": "mydb.cfxgae8cilcf.us-east-1.rds.amazonaws.com", - "region": "us-east-1", - "managed_authentication": { - "enabled": True, - }, - }, - psycopg2.OperationalError, - 'password authentication failed', - True, - ), ( { 'region': 'us-east-1', @@ -864,27 +850,6 @@ def test_database_instance_metadata(aggregator, pg_instance, dbm_enabled, report None, False, ), - ( - { - 'region': 'us-east-1', - "managed_authentication": { - "enabled": 'true', - }, - }, - psycopg2.OperationalError, - 'password authentication failed', - True, - ), - ( - { - "managed_authentication": { - "enabled": 'true', - } - }, - ConfigurationError, - 'AWS region must be set when using AWS managed authentication', - None, # IAM auth requires region so this should fail - ), ], ) def test_database_instance_cloud_metadata_aws( @@ -937,18 +902,6 @@ def test_database_instance_cloud_metadata_aws( None, False, ), - ( - { - "deployment_type": "flexible_server", - "fully_qualified_domain_name": "my-postgres-database.database.windows.net", - }, - { - "client_id": "my-client-id", - }, - psycopg2.OperationalError, - 'password authentication failed', - True, - ), ( { "deployment_type": "flexible_server", @@ -979,50 +932,6 @@ def test_database_instance_cloud_metadata_aws( None, False, ), - ( - { - "deployment_type": "flexible_server", - "fully_qualified_domain_name": "my-postgres-database.database.windows.net", - "managed_authentication": { - "enabled": True, - "client_id": "my-client-id", - }, - }, - { - "client_id": "my-client-id", - }, - psycopg2.OperationalError, - 'password authentication failed', - True, - ), - ( - { - "deployment_type": "flexible_server", - "fully_qualified_domain_name": "my-postgres-database.database.windows.net", - "managed_authentication": { - "enabled": 'true', - "client_id": "my-client-id", - 'identity_scope': 'https://database.windows.net/.default', - }, - }, - None, - psycopg2.OperationalError, - 'password authentication failed', - True, - ), - ( - { - "deployment_type": "flexible_server", - "fully_qualified_domain_name": "my-postgres-database.database.windows.net", - "managed_authentication": { - "enabled": True, - }, - }, - None, - ConfigurationError, - 'Azure client_id must be set when using Azure managed authentication', - None, - ), ], ) def test_database_instance_cloud_metadata_azure( diff --git a/postgres/tests/test_pg_replication.py b/postgres/tests/test_pg_replication.py index 888e6f5543269..b7272eac99585 100644 --- a/postgres/tests/test_pg_replication.py +++ b/postgres/tests/test_pg_replication.py @@ -132,13 +132,11 @@ def test_conflicts_lock(aggregator, integration_check, pg_instance, pg_replica_i check = integration_check(pg_replica_instance2) replica_con = _get_superconn(pg_replica_instance2) - replica_con.set_session(autocommit=False) replica_cur = replica_con.cursor() replica_cur.execute('BEGIN;') replica_cur.execute('select * from persons;') conn = _get_superconn(pg_instance) - conn.set_session(autocommit=True) cur = conn.cursor() cur.execute('update persons SET personid = 1 where personid = 1;') cur.execute('vacuum full persons;') @@ -164,13 +162,11 @@ def test_conflicts_snapshot(aggregator, integration_check, pg_instance, pg_repli check = integration_check(pg_replica_instance2) replica2_con = _get_superconn(pg_replica_instance2) - replica2_con.set_session(autocommit=False) replica2_cur = replica2_con.cursor() replica2_cur.execute('BEGIN TRANSACTION ISOLATION LEVEL REPEATABLE READ;') replica2_cur.execute('select * from persons;') - conn = _get_superconn(pg_instance) - conn.set_session(autocommit=True) + conn = _get_superconn(pg_instance, autocommit=True) cur = conn.cursor() cur.execute('update persons SET personid = 1 where personid = 1;') time.sleep(1.2) diff --git a/postgres/tests/test_relations.py b/postgres/tests/test_relations.py index 9e330289bcbc1..648653073cba5 100644 --- a/postgres/tests/test_relations.py +++ b/postgres/tests/test_relations.py @@ -4,7 +4,7 @@ import threading -import psycopg2 +import psycopg import pytest from datadog_checks.base import ConfigurationError @@ -99,6 +99,7 @@ def test_relations_metrics_access_exclusive_lock(aggregator, integration_check, check = integration_check(pg_instance) conn = _get_superconn(pg_instance) + conn.execute("SET client_encoding TO 'UTF8'") cursor = conn.cursor() # Lock the persons table with an AccessExclusiveLock cursor.execute("BEGIN") # must be in a transaction to lock a table @@ -456,7 +457,7 @@ def check_with_lock(check, instance, lock_table=None): lock_statement = "LOCK persons" if lock_table is not None: lock_statement = "LOCK {}".format(lock_table) - with psycopg2.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: + with psycopg.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: with conn.cursor() as cur: cur.execute(lock_statement) check.check(instance) diff --git a/postgres/tests/test_replication_slot.py b/postgres/tests/test_replication_slot.py index c80c315680e8e..d5e4edc743928 100644 --- a/postgres/tests/test_replication_slot.py +++ b/postgres/tests/test_replication_slot.py @@ -3,7 +3,7 @@ # Licensed under a 3-clause BSD style license (see LICENSE) import time -import psycopg2 +import psycopg import pytest from datadog_checks.dev.docker import get_container_ip @@ -30,17 +30,19 @@ def test_physical_replication_slots(aggregator, integration_check, pg_instance): time.sleep(5) redo_lsn_age = 0 xmin_age_higher_bound = 1 - with psycopg2.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: - with conn.cursor() as cur: - cur.execute("select pg_wal_lsn_diff(pg_current_wal_lsn(), redo_lsn) from pg_control_checkpoint();") - redo_lsn_age = int(cur.fetchall()[0][0]) - cur.execute("select age(xmin) FROM pg_replication_slots;") - bound = cur.fetchall()[0][0] - xmin_age_higher_bound += int(bound) if bound is not None else 0 - - cur.execute("select * from pg_create_physical_replication_slot('phys_1');") - cur.execute("select * from pg_create_physical_replication_slot('phys_2', true);") - cur.execute("select * from pg_create_physical_replication_slot('phys_3', true, true);") + # Keep the connection open so that the temporary replication slots are not dropped + conn = psycopg.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") + with conn.cursor() as cur: + cur.execute("select pg_wal_lsn_diff(pg_current_wal_lsn(), redo_lsn) from pg_control_checkpoint();") + redo_lsn_age = int(cur.fetchall()[0][0]) + cur.execute("select age(xmin) FROM pg_replication_slots;") + bound = cur.fetchall()[0][0] + xmin_age_higher_bound += int(bound) if bound is not None else 0 + + cur.execute("select * from pg_create_physical_replication_slot('phys_1');") + cur.execute("select * from pg_create_physical_replication_slot('phys_2', true);") + cur.execute("select * from pg_create_physical_replication_slot('phys_3', true, true);") + check.check(pg_instance) # slot_name | slot_type | temporary | active | active_pid | xmin | restart_lsn @@ -94,11 +96,13 @@ def test_physical_replication_slots(aggregator, integration_check, pg_instance): count=1, ) + conn.close() + @requires_over_10 def test_logical_replication_slots(aggregator, integration_check, pg_instance): check = integration_check(pg_instance) - with psycopg2.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: + with psycopg.connect(host=HOST, dbname=DB_NAME, user="postgres", password="datad0g") as conn: with conn.cursor() as cur: cur.execute("SELECT pg_wal_lsn_diff(pg_current_wal_lsn(), restart_lsn) FROM pg_replication_slots;") restart_age = cur.fetchall()[0][0] diff --git a/postgres/tests/test_statements.py b/postgres/tests/test_statements.py index 58a2234cdb434..2569e92f805d0 100644 --- a/postgres/tests/test_statements.py +++ b/postgres/tests/test_statements.py @@ -3,15 +3,16 @@ # Licensed under Simplified BSD License (see LICENSE) import datetime import re -import select +import threading import time from collections import Counter, namedtuple from concurrent.futures.thread import ThreadPoolExecutor import mock -import psycopg2 +import psycopg import pytest from dateutil import parser +from psycopg import ClientCursor from semver import VersionInfo from datadog_checks.base.utils.db.sql import compute_sql_signature @@ -25,8 +26,8 @@ ) from datadog_checks.postgres.statements import ( PG_STAT_STATEMENTS_METRICS_COLUMNS, - PG_STAT_STATEMENTS_TIMING_COLUMNS, - PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17, + # PG_STAT_STATEMENTS_TIMING_COLUMNS, + # PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17, PostgresStatementMetrics, StatementMetrics, _row_key, @@ -45,7 +46,7 @@ _get_expected_replication_tags, _get_expected_tags, ) -from .utils import _get_conn, _get_superconn, requires_over_10, requires_over_13, run_one_check +from .utils import WaitGroup, _get_conn, _get_superconn, requires_over_10, requires_over_13, run_one_check pytestmark = [pytest.mark.integration, pytest.mark.usefixtures('dd_environment')] @@ -125,7 +126,9 @@ def _run_query(idx): password = "bob" dbname = "datadog_test" if dbname not in connections: - connections[dbname] = psycopg2.connect(host=HOST, dbname=dbname, user=user, password=password) + connections[dbname] = psycopg.connect( + host=HOST, dbname=dbname, user=user, password=password, cursor_factory=ClientCursor + ) args = ('two',) if idx == 1: @@ -241,7 +244,9 @@ def test_statement_metrics( def _run_queries(): for user, password, dbname, query, arg in SAMPLE_QUERIES: if dbname not in connections: - connections[dbname] = psycopg2.connect(host=HOST, dbname=dbname, user=user, password=password) + connections[dbname] = psycopg.connect( + host=HOST, dbname=dbname, user=user, password=password, autocommit=True, cursor_factory=ClientCursor + ) connections[dbname].cursor().execute(query, (arg,)) check = integration_check(dbm_instance) @@ -302,15 +307,15 @@ def _should_catch_query(dbname): assert row['query'] == expected_query available_columns = set(row.keys()) metric_columns = available_columns & PG_STAT_STATEMENTS_METRICS_COLUMNS - if track_io_timing_enabled: - if float(POSTGRES_VERSION) >= 17.0: - assert (available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS) == PG_STAT_STATEMENTS_TIMING_COLUMNS - else: - assert ( - available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17 - ) == PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17 - else: - assert (available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS) == set() + # if track_io_timing_enabled: + # if float(POSTGRES_VERSION) >= 17.0: + # assert (available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS) == PG_STAT_STATEMENTS_TIMING_COLUMNS + # else: + # assert ( + # available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17 + # ) == PG_STAT_STATEMENTS_TIMING_COLUMNS_LT_17 + # else: + # assert (available_columns & PG_STAT_STATEMENTS_TIMING_COLUMNS) == set() for col in metric_columns: assert type(row[col]) in (float, int) @@ -414,8 +419,6 @@ def test_statement_metrics_cloud_metadata( # don't need samples for this test dbm_instance['query_samples'] = {'enabled': False} dbm_instance['query_activity'] = {'enabled': False} - # very low collection interval for test purposes - dbm_instance['query_metrics'] = {'enabled': True, 'run_sync': True, 'collection_interval': 0.1} if input_cloud_metadata: for k, v in input_cloud_metadata.items(): dbm_instance[k] = v @@ -424,14 +427,14 @@ def test_statement_metrics_cloud_metadata( def _run_queries(): for user, password, dbname, query, arg in SAMPLE_QUERIES: if dbname not in connections: - connections[dbname] = psycopg2.connect(host=HOST, dbname=dbname, user=user, password=password) + connections[dbname] = psycopg.connect(host=HOST, dbname=dbname, user=user, password=password) connections[dbname].cursor().execute(query, (arg,)) check = integration_check(dbm_instance) check._connect() _run_queries() - run_one_check(check) + run_one_check(check, cancel=False) _run_queries() run_one_check(check) @@ -456,22 +459,20 @@ def test_wal_metrics(aggregator, integration_check, dbm_instance): # don't need samples for this test dbm_instance['query_samples'] = {'enabled': False} dbm_instance['query_activity'] = {'enabled': False} - # very low collection interval for test purposes - dbm_instance['query_metrics'] = {'enabled': True, 'run_sync': True, 'collection_interval': 0.1} connections = {} def _run_queries(): for user, password, dbname, query, arg in SAMPLE_QUERIES: if dbname not in connections: - connections[dbname] = psycopg2.connect(host=HOST, dbname=dbname, user=user, password=password) + connections[dbname] = psycopg.connect(host=HOST, dbname=dbname, user=user, password=password) connections[dbname].cursor().execute(query, (arg,)) check = integration_check(dbm_instance) check._connect() _run_queries() - run_one_check(check) + run_one_check(check, cancel=False) _run_queries() run_one_check(check) @@ -491,8 +492,6 @@ def test_statement_metrics_with_duplicates(aggregator, integration_check, dbm_in # don't need samples for this test dbm_instance['query_samples'] = {'enabled': False} dbm_instance['query_activity'] = {'enabled': False} - # very low collection interval for test purposes - dbm_instance['query_metrics'] = {'enabled': True, 'run_sync': True, 'collection_interval': 0.1} # The query signature matches the normalized query returned by the mock agent and would need to be # updated if the normalized query is updated @@ -501,7 +500,7 @@ def test_statement_metrics_with_duplicates(aggregator, integration_check, dbm_in normalized_query = 'select * from pg_stat_activity where application_name = ANY(array [ ? ])' def obfuscate_sql(query, options=None): - if query.startswith('select * from pg_stat_activity where application_name'): + if 'select * from pg_stat_activity where application_name' in query: return normalized_query return query @@ -534,7 +533,7 @@ def obfuscate_sql(query, options=None): @pytest.fixture def bob_conn(): - conn = psycopg2.connect(host=HOST, dbname=DB_NAME, user="bob", password="bob") + conn = psycopg.connect(host=HOST, dbname=DB_NAME, user="bob", password="bob") yield conn conn.close() @@ -607,7 +606,7 @@ def test_successful_explain( check._connect() # run check so all internal state is correctly initialized - run_one_check(check) + run_one_check(check, cancel=False) # clear out contents of aggregator so we measure only the metrics generated during this specific part of the test aggregator.reset() @@ -629,42 +628,42 @@ def test_successful_explain( [ ( "select * from fake_table", - "error:explain-undefined_table-", + "error:explain-undefined_table-", None, 1, None, ), ( "select * from fake_schema.fake_table", - "error:explain-undefined_table-", + "error:explain-undefined_table-", None, 1, None, ), ( "select * from pg_settings where name = $1", - "error:explain-parameterized_query-", + "error:explain-parameterized_query-", None, 1, None, ), ( "select * from pg_settings where name = 'this query is truncated' limi", - "error:explain-database_error-", + "error:explain-database_error-", None, 1, None, ), ( "select * from persons", - "error:explain-database_error-", + "error:explain-database_error-", "datadog.explain_statement_noaccess", failed_explain_test_repeat_count, None, ), ( "update persons set firstname='firstname' where personid in (2, 1); select pg_sleep(1);", - "error:explain-database_error-", + "error:explain-database_error-", None, 1, None, @@ -694,7 +693,7 @@ def test_failed_explain_handling( pytest.skip("not relevant for postgres {version}".format(version=POSTGRES_VERSION)) # run check so all internal state is correctly initialized - run_one_check(check) + run_one_check(check, cancel=False) # clear out contents of aggregator so we measure only the metrics generated during this specific part of the test aggregator.reset() @@ -759,7 +758,7 @@ def test_failed_explain_handling( "SELECT * FROM kennel WHERE id = %s", 123, "error:explain-no_plans_possible", - [{'code': 'invalid_schema', 'message': ""}], + [{'code': 'invalid_schema', 'message': ""}], StatementTruncationState.not_truncated.value, [], ), @@ -769,8 +768,8 @@ def test_failed_explain_handling( "dogs_nofunc", "SELECT * FROM kennel WHERE id = %s", 123, - "error:explain-failed_function-", - [{'code': 'failed_function', 'message': ""}], + "error:explain-failed_function-", + [{'code': 'failed_function', 'message': ""}], StatementTruncationState.not_truncated.value, [ "Unable to collect execution plans in dbname=dogs_nofunc. Check that the function " @@ -779,7 +778,7 @@ def test_failed_explain_handling( " for more details: function datadog.explain_statement(unknown) does not exist\nLINE 1: " "... DDIGNORE */ /* service='datadog-agent' */ SELECT datadog.ex...\n" " ^\nHINT: No function matches the given " - "name and argument types. You might need to add explicit type casts.\n\ncode=undefined-explain-function" + "name and argument types. You might need to add explicit type casts.\ncode=undefined-explain-function" " dbname=dogs_nofunc host=stubbed.hostname", ], ), @@ -844,9 +843,11 @@ def test_statement_samples_collect( check = integration_check(dbm_instance) check._connect() - conn = psycopg2.connect(host=HOST, dbname=dbname, user=user, password=password) - conn.set_client_encoding('utf8') - # we are able to see the full query (including the raw parameters) in pg_stat_activity because psycopg2 uses + conn = psycopg.connect( + host=HOST, dbname=dbname, user=user, password=password, autocommit=True, cursor_factory=ClientCursor + ) + conn.execute("SET client_encoding TO UTF8") + # we are able to see the full query (including the raw parameters) in pg_stat_activity because psycopg uses # the simple query protocol, sending the whole query as a plain string to postgres. # if a client is using the extended query protocol with prepare then the query would appear as # leave connection open until after the check has run to ensure we're able to see the query in @@ -884,7 +885,7 @@ def test_statement_samples_collect( hostname='stubbed.hostname', ) else: - assert len(matching) == 1, "missing captured event" + assert len(matching) == 1, "missing captured event for query: {query}".format(query=query) event = matching[0] assert event['db']['query_truncated'] == expected_statement_truncated assert set(event['ddtags'].split(',')) == set(tags) @@ -920,7 +921,6 @@ def test_statement_samples_collect( assert event['db']['plan']['raw_signature'] is not None, "missing raw plan signature" else: assert len(raw_plan_events) == 0 - finally: conn.close() @@ -950,11 +950,7 @@ def test_statement_metadata( ): """Tests for metadata in both samples and metrics""" dbm_instance['pg_stat_statements_view'] = pg_stat_statements_view - dbm_instance['query_samples']['run_sync'] = True dbm_instance['query_metrics']['run_sync'] = True - # This prevents DBMAsync from skipping job executions, as a job should not be executed - # more frequently than its collection period. - dbm_instance['query_samples']['collection_interval'] = CLOSE_TO_ZERO_INTERVAL # If query or normalized_query changes, the query_signatures for both will need to be updated as well. query = ''' @@ -975,7 +971,7 @@ def obfuscate_sql(query, options=None): check = integration_check(dbm_instance) check._connect() - conn = psycopg2.connect(host=HOST, dbname="datadog_test", user="bob", password="bob") + conn = psycopg.connect(host=HOST, dbname="datadog_test", user="bob", password="bob") cursor = conn.cursor() # Execute the query with the mocked obfuscate_sql. The result should produce an event payload with the metadata. with mock.patch.object(datadog_agent, 'obfuscate_sql', passthrough=True) as mock_agent: @@ -983,11 +979,11 @@ def obfuscate_sql(query, options=None): cursor.execute( query, ) - run_one_check(check) + run_one_check(check, cancel=False) cursor.execute( query, ) - run_one_check(check) + run_one_check(check, cancel=False) # Test samples metadata, metadata in samples is an object under `db`. samples = aggregator.get_event_platform_events("dbm-samples") @@ -1037,16 +1033,11 @@ def test_statement_reported_hostname( reported_hostname, expected_hostname, ): - dbm_instance['query_samples']['run_sync'] = True - dbm_instance['query_metrics']['run_sync'] = True - # This prevents DBMAsync from skipping job executions, as a job should not be executed - # more frequently than its collection period. - dbm_instance['query_samples']['collection_interval'] = 0.0000001 dbm_instance['reported_hostname'] = reported_hostname check = integration_check(dbm_instance) - run_one_check(check) + run_one_check(check, cancel=False) run_one_check(check) samples = aggregator.get_event_platform_events("dbm-samples") @@ -1064,29 +1055,27 @@ def test_statement_reported_hostname( @pytest.mark.parametrize("pg_stat_activity_view", ["pg_stat_activity", "datadog.pg_stat_activity()"]) @pytest.mark.parametrize( - "user,password,dbname,query,blocking_query,arg,expected_out,expected_keys,expected_conn_out", + "user,password,dbname,query,blocking_query,expected_out,expected_keys,expected_conn_out", [ ( "bob", "bob", "datadog_test", - "BEGIN TRANSACTION; SET application_name='test_snapshot'; SELECT city FROM persons WHERE city = %s;", + "BEGIN TRANSACTION; SELECT city FROM persons WHERE city = 'hello';", "LOCK TABLE persons IN ACCESS EXCLUSIVE MODE", - "hello", { 'datname': 'datadog_test', 'usename': 'bob', 'state': 'active', - 'query_signature': '4bd870d5ce614fd', - 'statement': "BEGIN TRANSACTION; SET application_name='test_snapshot'; " - "SELECT city FROM persons WHERE city = 'hello';", + 'query_signature': '9382c42e92099c04', + 'statement': "BEGIN TRANSACTION; SELECT city FROM persons WHERE city = 'hello';", 'query_truncated': StatementTruncationState.not_truncated.value, }, ["now", "xact_start", "query_start", "pid", "client_port", "client_addr", "backend_type", "blocking_pids"], { 'usename': 'bob', 'state': 'active', - 'application_name': 'test_snapshot', + 'application_name': '', 'datname': 'datadog_test', 'connections': 1, }, @@ -1095,43 +1084,43 @@ def test_statement_reported_hostname( "bob", "bob", "datadog_test", - "BEGIN TRANSACTION; SET application_name='test_snapshot'; SELECT city as city0, city as city1, " - "city as city2, city as city3, city as city4, city as city5, city as city6, city as city7, " - "city as city8, city as city9, city as city10, city as city11, city as city12, city as city13, " - "city as city14, city as city15, city as city16, city as city17, city as city18, city as city19, " - "city as city20, city as city21, city as city22, city as city23, city as city24, city as city25, " - "city as city26, city as city27, city as city28, city as city29, city as city30, city as city31, " - "city as city32, city as city33, city as city34, city as city35, city as city36, city as city37, " - "city as city38, city as city39, city as city40, city as city41, city as city42, city as city43, " - "city as city44, city as city45, city as city46, city as city47, city as city48, city as city49, " - "city as city50, city as city51, city as city52, city as city53, city as city54, city as city55, " - "city as city56, city as city57, city as city58, city as city59, city as city60, city as city61 " - "FROM persons WHERE city = %s;", + "BEGIN TRANSACTION; SELECT city as city0, city as city1, city as city2, city as city3, " + "city as city4, city as city5, city as city6, city as city7, city as city8, city as city9, " + "city as city10, city as city11, city as city12, city as city13, city as city14, city as city15, " + "city as city16, city as city17, city as city18, city as city19, city as city20, city as city21, " + "city as city22, city as city23, city as city24, city as city25, city as city26, city as city27, " + "city as city28, city as city29, city as city30, city as city31, city as city32, city as city33, " + "city as city34, city as city35, city as city36, city as city37, city as city38, city as city39, " + "city as city40, city as city41, city as city42, city as city43, city as city44, city as city45, " + "city as city46, city as city47, city as city48, city as city49, city as city50, city as city51, " + "city as city52, city as city53, city as city54, city as city55, city as city56, city as city57, " + "city as city58, city as city59, city as city60, city as city61 " + "FROM persons WHERE city = 'hello';", "LOCK TABLE persons IN ACCESS EXCLUSIVE MODE", - "hello", { 'datname': 'datadog_test', 'usename': 'bob', 'state': 'active', - 'query_signature': 'f79596b3cba3247a', - 'statement': "BEGIN TRANSACTION; SET application_name='test_snapshot'; SELECT city as city0, " - "city as city1, city as city2, city as city3, city as city4, city as city5, city as city6, " - "city as city7, city as city8, city as city9, city as city10, city as city11, city as city12, " - "city as city13, city as city14, city as city15, city as city16, city as city17, city as city18, " - "city as city19, city as city20, city as city21, city as city22, city as city23, city as city24, " - "city as city25, city as city26, city as city27, city as city28, city as city29, city as city30, " - "city as city31, city as city32, city as city33, city as city34, city as city35, city as city36, " - "city as city37, city as city38, city as city39, city as city40, city as city41, city as city42, " - "city as city43, city as city44, city as city45, city as city46, city as city47, city as city48, " - "city as city49, city as city50, city as city51, city as city52, city as city53, city as city54, " - "city as city55, city as city56, city as city57, city as city58, city as city59, city as", + 'query_signature': 'e1429b86c013a78e', + 'statement': "BEGIN TRANSACTION; SELECT city as city0, city as city1, city as city2, city as city3, " + "city as city4, city as city5, city as city6, city as city7, city as city8, city as city9, " + "city as city10, city as city11, city as city12, city as city13, city as city14, city as city15, " + "city as city16, city as city17, city as city18, city as city19, city as city20, city as city21, " + "city as city22, city as city23, city as city24, city as city25, city as city26, city as city27, " + "city as city28, city as city29, city as city30, city as city31, city as city32, city as city33, " + "city as city34, city as city35, city as city36, city as city37, city as city38, city as city39, " + "city as city40, city as city41, city as city42, city as city43, city as city44, city as city45, " + "city as city46, city as city47, city as city48, city as city49, city as city50, city as city51, " + "city as city52, city as city53, city as city54, city as city55, city as city56, city as city57, " + "city as city58, city as city59, city as city60, city as city61 " + "FROM persons WHE", 'query_truncated': StatementTruncationState.truncated.value, }, ["now", "xact_start", "query_start", "pid", "client_port", "client_addr", "backend_type", "blocking_pids"], { 'usename': 'bob', 'state': 'active', - 'application_name': 'test_snapshot', + 'application_name': '', 'datname': 'datadog_test', 'connections': 1, }, @@ -1149,7 +1138,6 @@ def test_activity_snapshot_collection( dbname, query, blocking_query, - arg, expected_out, expected_keys, expected_conn_out, @@ -1165,41 +1153,30 @@ def test_activity_snapshot_collection( check = integration_check(dbm_instance) check._connect() - conn = psycopg2.connect(host=HOST, dbname=dbname, user=user, password=password, async_=1) - blocking_conn = psycopg2.connect(host=HOST, dbname=dbname, user="blocking_bob", password=password) + blocking_conn = psycopg.connect(host=HOST, dbname=dbname, user="blocking_bob", password=password, autocommit=False) + conn = psycopg.connect(host=HOST, dbname=dbname, user=user, password=password, autocommit=False) + wg = WaitGroup() - def wait(conn): - while True: - state = conn.poll() - if state == psycopg2.extensions.POLL_OK: - break - elif state == psycopg2.extensions.POLL_WRITE: - select.select([], [conn.fileno()], []) - elif state == psycopg2.extensions.POLL_READ: - select.select([conn.fileno()], [], []) - else: - raise psycopg2.OperationalError("poll() returned %s" % state) + def execute_in_thread(q): + with conn.cursor() as cursor: + cursor.execute(q) + wg.done() - # we are able to see the full query (including the raw parameters) in pg_stat_activity because psycopg2 uses + # we are able to see the full query (including the raw parameters) in pg_stat_activity because psycopg uses # the simple query protocol, sending the whole query as a plain string to postgres. # if a client is using the extended query protocol with prepare then the query would appear as # leave connection open until after the check has run to ensure we're able to see the query in # pg_stat_activity try: # first lock the table, which will cause the test query to be blocked - blocking_conn.autocommit = False blocking_conn.cursor().execute(blocking_query) - # ... now execute the test query - wait(conn) - conn.cursor().execute(query, (arg,)) - run_one_check(check) + # ... now execute the test query in a separate thread + t = threading.Thread(target=execute_in_thread, args=(query,)) + wg.add(1) + t.start() + check.check(dbm_instance) dbm_activity_event = aggregator.get_event_platform_events("dbm-activity") - if POSTGRES_VERSION.split('.')[0] == "9" and pg_stat_activity_view == "pg_stat_activity": - # cannot catch any queries from other users - # only can see own queries - return - event = dbm_activity_event[0] assert event['host'] == "stubbed.hostname" assert event['ddsource'] == "postgres" @@ -1207,14 +1184,7 @@ def wait(conn): assert event['ddagentversion'] == datadog_agent.get_version() assert len(event['postgres_activity']) > 0 # find bob's query and blocking_bob's query - bobs_query = next( - ( - q - for q in event['postgres_activity'] - if q.get('usename', None) == "bob" and q.get('application_name', None) == 'test_snapshot' - ), - None, - ) + bobs_query = next((q for q in event['postgres_activity'] if q.get('usename', None) == "bob"), None) blocking_bobs_query = next( (q for q in event['postgres_activity'] if q.get('usename', None) == "blocking_bob"), None ) @@ -1259,50 +1229,40 @@ def wait(conn): # find bob's connections. bobs_conns = None for query_json in event['postgres_connections']: - if ( - 'usename' in query_json - and query_json['usename'] == "bob" - and query_json['application_name'] == 'test_snapshot' - ): + if 'usename' in query_json and query_json['usename'] == "bob": bobs_conns = query_json - break assert bobs_conns is not None for key in expected_conn_out: assert expected_conn_out[key] == bobs_conns[key] - assert set(event['ddtags']) == set(expected_tags) - - if POSTGRES_VERSION == '9.5': - # rest of test is to confirm blocking behavior - # which we cannot collect in pg v9.5 at this time - return + assert sorted(event['ddtags']) == sorted(expected_tags) # ... now run the check again after closing blocking_bob's conn. # this means we should report bob as no longer blocked # close blocking_bob's tx blocking_conn.close() + + # wait for query to complete, but commit has not been called, + # so it should remain open and idle + wg.wait() + # Wait collection interval to make sure dbm events are reported time.sleep(dbm_instance['query_activity']['collection_interval']) - run_one_check(check) + check.check(dbm_instance) dbm_activity_event = aggregator.get_event_platform_events("dbm-activity") event = dbm_activity_event[1] assert len(event['postgres_activity']) > 0 + # find bob's query bobs_query = None for query_json in event['postgres_activity']: - if ( - 'usename' in query_json - and query_json['usename'] == "bob" - and query_json['application_name'] == 'test_snapshot' - ): + if 'usename' in query_json and query_json['usename'] == "bob": bobs_query = query_json - break assert bobs_query is not None assert len(bobs_query['blocking_pids']) == 0 # state should be idle now that it's no longer blocked assert bobs_query['state'] == "idle in transaction" - finally: blocking_conn.close() conn.close() @@ -1322,7 +1282,7 @@ def test_activity_raw_statement_collection(aggregator, integration_check, dbm_in check = integration_check(dbm_instance) check._connect() - conn = psycopg2.connect(host=HOST, dbname='datadog_test', user='bob', password='bob') + conn = psycopg.connect(host=HOST, dbname='datadog_test', user='bob', password='bob') query = "BEGIN TRANSACTION; SELECT * FROM persons WHERE city like 'hello';" try: conn.cursor().execute(query) @@ -1476,15 +1436,15 @@ def test_truncate_activity_rows(integration_check, dbm_instance, active_rows, ex [ ( "select * from fake_table", - "error:explain-undefined_table-", + "error:explain-undefined_table-", DBExplainError.undefined_table, - "", + "", ), ( "select * from pg_settings where name = $1", - "error:explain-parameterized_query-", + "error:explain-parameterized_query-", DBExplainError.parameterized_query, - "", + "", ), ( "SELECT city as city0, city as city1, city as city2, city as city3, " @@ -1520,7 +1480,7 @@ def test_statement_run_explain_errors( check = integration_check(dbm_instance) check._connect() - run_one_check(check) + run_one_check(check, cancel=False) _, explain_err_code, err = check.statement_samples._run_and_track_explain( "datadog_test", query, query, "7231596c8b5536d1" ) @@ -1576,7 +1536,7 @@ def test_statement_run_explain_parameterized_queries( if check.version < V12: return - run_one_check(check) + run_one_check(check, cancel=False) _, explain_err_code, err = check.statement_samples._run_and_track_explain( "datadog_test", query, query, "7231596c8b5536d1" ) @@ -1596,7 +1556,9 @@ def test_statement_samples_dbstrict(aggregator, integration_check, dbm_instance, connections = [] for user, password, dbname, query, arg in SAMPLE_QUERIES: - conn = psycopg2.connect(host=HOST, dbname=dbname, user=user, password=password) + conn = psycopg.connect( + host=HOST, dbname=dbname, user=user, password=password, cursor_factory=ClientCursor, autocommit=True + ) conn.cursor().execute(query, (arg,)) connections.append(conn) @@ -1787,7 +1749,7 @@ def _sample_key(e): @pytest.mark.parametrize("query_activity_enabled", [True, False]) @pytest.mark.parametrize( "user,password,dbname,query,arg", - [("bob", "bob", "datadog_test", "BEGIN TRANSACTION; SELECT city FROM persons WHERE city = %s;", "hello")], + [("bob", "bob", "datadog_test", "SELECT city FROM persons WHERE city = %s;", "hello")], ) def test_disabled_activity_or_explain_plans( aggregator, @@ -1815,10 +1777,9 @@ def test_disabled_activity_or_explain_plans( check = integration_check(dbm_instance) check._connect() - conn = psycopg2.connect(host=HOST, dbname=dbname, user=user, password=password) + conn = psycopg.connect(host=HOST, dbname=dbname, user=user, password=password, cursor_factory=ClientCursor) try: - conn.autocommit = True conn.cursor().execute(query, (arg,)) run_one_check(check) dbm_activity = aggregator.get_event_platform_events("dbm-activity") @@ -1867,7 +1828,7 @@ def test_async_job_cancel_cancel(aggregator, integration_check, dbm_instance): assert not check.statement_metrics._job_loop_future.running(), "metrics thread should be stopped" # if the thread doesn't start until after the cancel signal is set then the db connection will never # be created in the first place - assert check.db_pool._conns.get(dbm_instance['dbname']) is None, "db connection should be gone" + assert check.db_pool.pools.get(dbm_instance['dbname']) is None, "db connection should be gone" for job in ['query-metrics', 'query-samples']: aggregator.assert_metric( "dd.postgres.async_job.cancel", @@ -1911,81 +1872,11 @@ def test_statement_samples_config_invalid_number(integration_check, pg_instance, integration_check(pg_instance) -class ObjectNotInPrerequisiteState(psycopg2.errors.ObjectNotInPrerequisiteState): - """ - A fake ObjectNotInPrerequisiteState that allows setting pg_error on construction since ObjectNotInPrerequisiteState - has it as read-only and not settable at construction-time - """ - - def __init__(self, pg_error): - self.pg_error = pg_error - - def __getattribute__(self, attr): - if attr == 'pgerror': - return self.pg_error - else: - return super(ObjectNotInPrerequisiteState, self).__getattribute__(attr) - - def __str__(self): - return self.pg_error - - -class UndefinedTable(psycopg2.errors.UndefinedTable): - """ - A fake UndefinedTable that allows setting pg_error on construction since UndefinedTable - has it as read-only and not settable at construction-time - """ - - def __init__(self, pg_error): - self.pg_error = pg_error - - def __getattribute__(self, attr): - if attr == 'pgerror': - return self.pg_error - else: - return super(UndefinedTable, self).__getattribute__(attr) - - def __str__(self): - return self.pg_error - - @pytest.mark.parametrize( "error,metric_columns,expected_error_tag,expected_warnings", [ ( - ObjectNotInPrerequisiteState('pg_stat_statements must be loaded via shared_preload_libraries'), - [], - 'error:database-ObjectNotInPrerequisiteState-pg_stat_statements_not_loaded', - [ - 'Unable to collect statement metrics because pg_stat_statements extension is ' - "not loaded in database 'datadog_test'. See https://docs.datadoghq.com/database_monitoring/" - 'setup_postgres/troubleshooting#pg-stat-statements-not-loaded' - ' for more details\ncode=pg-stat-statements-not-loaded dbname=datadog_test host=stubbed.hostname', - ], - ), - ( - UndefinedTable('ERROR: relation "pg_stat_statements" does not exist'), - [], - 'error:database-UndefinedTable-pg_stat_statements_not_created', - [ - 'Unable to collect statement metrics because pg_stat_statements is not ' - "created in database 'datadog_test'. See https://docs.datadoghq.com/database_monitoring/" - 'setup_postgres/troubleshooting#pg-stat-statements-not-created' - ' for more details\ncode=pg-stat-statements-not-created dbname=datadog_test host=stubbed.hostname', - ], - ), - ( - ObjectNotInPrerequisiteState('cannot insert into view'), - [], - 'error:database-ObjectNotInPrerequisiteState', - [ - "Unable to collect statement metrics because of an error running queries in database 'datadog_test'. " - "See https://docs.datadoghq.com/database_monitoring/troubleshooting for help: cannot insert into view\n" - "dbname=datadog_test host=stubbed.hostname" - ], - ), - ( - psycopg2.errors.DatabaseError('connection reset'), + psycopg.errors.DatabaseError('connection reset'), [], 'error:database-DatabaseError', [ @@ -2031,6 +1922,52 @@ def test_statement_metrics_database_errors( assert check.warnings == expected_warnings +@pytest.mark.parametrize( + "error,metric_columns,expected_error_tag,expected_warnings", + [ + ( + 'not-created', + [], + 'error:database-UndefinedTable-pg_stat_statements_not_created', + [ + 'Unable to collect statement metrics because pg_stat_statements is not ' + "created in database 'datadog_test'. See https://docs.datadoghq.com/database_monitoring/" + 'setup_postgres/troubleshooting#pg-stat-statements-not-created' + ' for more details\ncode=pg-stat-statements-not-created dbname=datadog_test host=stubbed.hostname', + ], + ), + ], +) +def test_statement_metrics_database_extension_errors( + aggregator, integration_check, dbm_instance, error, metric_columns, expected_error_tag, expected_warnings +): + # don't need samples for this test + dbm_instance['query_samples']['enabled'] = False + dbm_instance['query_activity']['enabled'] = False + check = integration_check(dbm_instance) + + # Break databased on purpose to simulate the extension not being loaded or created + if error == 'not-created': + superconn = _get_superconn(dbm_instance) + with superconn.cursor() as cur: + cur.execute("DROP EXTENSION pg_stat_statements CASCADE;") + + run_one_check(check) + + # Restore extension for next test + cur.execute("CREATE EXTENSION IF NOT EXISTS pg_stat_statements SCHEMA public;") + + expected_tags = _get_expected_tags( + check, dbm_instance, with_host=False, with_db=True, agent_hostname='stubbed.hostname' + ) + [expected_error_tag] + + aggregator.assert_metric( + 'dd.postgres.statement_metrics.error', value=1.0, count=1, tags=expected_tags, hostname='stubbed.hostname' + ) + + assert check.warnings == expected_warnings + + @pytest.mark.parametrize( "pg_stat_statements_max_threshold,expected_warnings", [ @@ -2148,22 +2085,20 @@ def test_plan_time_metrics(aggregator, integration_check, dbm_instance): # don't need samples for this test dbm_instance['query_samples'] = {'enabled': False} dbm_instance['query_activity'] = {'enabled': False} - # very low collection interval for test purposes - dbm_instance['query_metrics'] = {'enabled': True, 'run_sync': True, 'collection_interval': 0.1} connections = {} def _run_queries(): for user, password, dbname, query, arg in SAMPLE_QUERIES: if dbname not in connections: - connections[dbname] = psycopg2.connect(host=HOST, dbname=dbname, user=user, password=password) + connections[dbname] = psycopg.connect(host=HOST, dbname=dbname, user=user, password=password) connections[dbname].cursor().execute(query, (arg,)) check = integration_check(dbm_instance) check._connect() _run_queries() - run_one_check(check) + run_one_check(check, cancel=False) _run_queries() run_one_check(check) @@ -2184,7 +2119,7 @@ def _run_queries(): @pytest.mark.unit def test_get_query_metrics_payload_rows(): config = PostgresConfig({"host": "host", "username": "user"}, {}, None) - statement_metrics = PostgresStatementMetrics({}, config, None) + statement_metrics = PostgresStatementMetrics({}, config) wrapper = {} TestCase = namedtuple('TestCase', 'rows max_size expected') @@ -2234,7 +2169,6 @@ def test_metrics_encoding( integration_check, dbm_instance, ): - dbm_instance['query_metrics'] = {'enabled': True, 'run_sync': True, 'collection_interval': 0.1} if POSTGRES_LOCALE == 'C': dbm_instance['query_encodings'] = ['latin1', 'utf-8'] dbm_instance['query_samples'] = {'enabled': False} @@ -2243,9 +2177,9 @@ def test_metrics_encoding( check = integration_check(dbm_instance) check._connect() - with psycopg2.connect(host=HOST, dbname=DB_NAME, user='bob', password='bob') as conn: + with psycopg.connect(host=HOST, dbname=DB_NAME, user='bob', password='bob') as conn: with conn.cursor() as cursor: - conn.set_client_encoding('latin1') + conn.execute("SET client_encoding TO LATIN1") # This should be funké in latin1 query = b"select 'funk\xe9' as funk\xe9;" cursor.execute(query) @@ -2279,9 +2213,9 @@ def test_samples_encoding( check = integration_check(dbm_instance) check._connect() - with psycopg2.connect(host=HOST, dbname=DB_NAME, user='bob', password='bob') as conn: + with psycopg.connect(host=HOST, dbname=DB_NAME, user='bob', password='bob') as conn: with conn.cursor() as cursor: - conn.set_client_encoding('latin1') + conn.execute("SET client_encoding TO LATIN1") # This should be funké in latin1 query = b"select 'funk\xe9' as funk\xe9;" cursor.execute(query) diff --git a/postgres/tests/test_unit.py b/postgres/tests/test_unit.py index 1b0a9905621cc..a4b9d1c6bb88b 100644 --- a/postgres/tests/test_unit.py +++ b/postgres/tests/test_unit.py @@ -4,7 +4,7 @@ import copy import mock -import psycopg2 +import psycopg import pytest from pytest import fail from semver import VersionInfo @@ -124,10 +124,10 @@ def test_query_timeout_connection_string(aggregator, integration_check, pg_insta check = integration_check(pg_instance) try: - check.db_pool.get_connection(pg_instance['dbname'], 100) - except psycopg2.ProgrammingError as e: + check.db_pool.get_connection(pg_instance['dbname']) + except psycopg.ProgrammingError as e: fail(str(e)) - except psycopg2.OperationalError: + except psycopg.OperationalError: # could not connect to server because there is no server running pass diff --git a/postgres/tests/utils.py b/postgres/tests/utils.py index 89760c888f4c6..dcf6ce13c41da 100644 --- a/postgres/tests/utils.py +++ b/postgres/tests/utils.py @@ -4,7 +4,7 @@ import threading import time -import psycopg2 +import psycopg import pytest from .common import PASSWORD_ADMIN, POSTGRES_VERSION, USER_ADMIN @@ -47,22 +47,24 @@ ) -def _get_conn(db_instance, dbname=None, user=None, password=None, application_name='test'): - conn = psycopg2.connect( +def _get_conn(db_instance, dbname=None, user=None, password=None, application_name='test', autocommit=True): + conn = psycopg.connect( host=db_instance['host'], port=db_instance['port'], dbname=dbname or db_instance['dbname'], user=user or db_instance['username'], password=password or db_instance['password'], application_name=application_name, + autocommit=autocommit, ) - conn.autocommit = True return conn # Get a connection with superuser -def _get_superconn(db_instance, application_name='test'): - return _get_conn(db_instance, user=USER_ADMIN, password=PASSWORD_ADMIN, application_name=application_name) +def _get_superconn(db_instance, application_name='test', autocommit=True): + return _get_conn( + db_instance, user=USER_ADMIN, password=PASSWORD_ADMIN, application_name=application_name, autocommit=autocommit + ) def lock_table(pg_instance, table, lock_mode): @@ -114,7 +116,7 @@ def run_query(): cur.execute(stmt) try: cur.execute(query) - except psycopg2.errors.QueryCanceled: + except psycopg.errors.QueryCanceled: pass conn.close() @@ -143,3 +145,28 @@ def run_one_check(check, cancel=True): check.statement_metrics._job_loop_future.result() if check.metadata_samples._job_loop_future is not None: check.metadata_samples._job_loop_future.result() + + +# WaitGroup is used like go's sync.WaitGroup +class WaitGroup(object): + def __init__(self): + self.count = 0 + self.cv = threading.Condition() + + def add(self, n): + self.cv.acquire() + self.count += n + self.cv.release() + + def done(self): + self.cv.acquire() + self.count -= 1 + if self.count == 0: + self.cv.notify_all() + self.cv.release() + + def wait(self): + self.cv.acquire() + while self.count > 0: + self.cv.wait() + self.cv.release() diff --git a/silverstripe_cms/changelog.d/21173.added b/silverstripe_cms/changelog.d/21173.added new file mode 100644 index 0000000000000..ea06e9e0efd6b --- /dev/null +++ b/silverstripe_cms/changelog.d/21173.added @@ -0,0 +1 @@ +Upgrade to psycopg3 diff --git a/silverstripe_cms/datadog_checks/silverstripe_cms/database_client.py b/silverstripe_cms/datadog_checks/silverstripe_cms/database_client.py index 9dbc1ef9b0d4b..41d74b0835e97 100644 --- a/silverstripe_cms/datadog_checks/silverstripe_cms/database_client.py +++ b/silverstripe_cms/datadog_checks/silverstripe_cms/database_client.py @@ -2,9 +2,10 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -import psycopg2 -import psycopg2.extras +import psycopg import pymysql +from psycopg import ClientCursor +from psycopg.rows import dict_row from datadog_checks.base.errors import ConfigurationError @@ -44,19 +45,23 @@ def create_connection(self) -> None: ) else: # PostgreSQL connection settings - self.connection = psycopg2.connect( + self.connection = psycopg.connect( host=self.db_host, port=self.db_port, user=self.db_username, password=self.db_password, dbname=self.db_name, - cursor_factory=psycopg2.extras.DictCursor, + cursor_factory=ClientCursor, connect_timeout=DB_CONNECTION_TIMEOUT_IN_SECONDS, + autocommit=True, ) message = f"Successfully authenticated with the {self.db_type} database." self.log.info(LOG_TEMPLATE.format(host=self.db_host, message=message)) - self.cursor = self.connection.cursor() - except (pymysql.Error, psycopg2.Error) as db_err: + if self.db_type == MYSQL: + self.cursor = self.connection.cursor() + else: + self.cursor = self.connection.cursor(row_factory=dict_row) + except (pymysql.Error, psycopg.Error) as db_err: err_message = ( f"Authentication failed for provided credentials. Please check the provided credentials." f" | Error={db_err}." @@ -70,7 +75,7 @@ def close_connection(self) -> None: self.connection.close() message = "Connection closed successfully." self.log.info(LOG_TEMPLATE.format(host=self.db_host, message=message)) - except (pymysql.Error, psycopg2.Error) as db_err: + except (pymysql.Error, psycopg.Error) as db_err: err_message = f"Error occurred while closing the connection. | Error={db_err}." self.log.error(LOG_TEMPLATE.format(host=self.db_host, message=err_message)) @@ -98,6 +103,6 @@ def execute_query(self, query: str) -> list: try: self.cursor.execute(query) return self.cursor.fetchall() - except (pymysql.Error, psycopg2.Error) as db_err: + except (pymysql.Error, psycopg.Error) as db_err: err_message = f"Error occurred while executing query: {query}. | Error={db_err}." self.log.error(LOG_TEMPLATE.format(host=self.db_host, message=err_message)) diff --git a/silverstripe_cms/pyproject.toml b/silverstripe_cms/pyproject.toml index 7e52a7c8b026e..0448a97fb6b30 100644 --- a/silverstripe_cms/pyproject.toml +++ b/silverstripe_cms/pyproject.toml @@ -37,7 +37,7 @@ dynamic = [ [project.optional-dependencies] deps = [ - "psycopg2-binary==2.9.9", + "psycopg[binary,pool]==3.2.7", "PyMySQL==1.1.1" ]