Refactor telemtry signals (#2225)

* refactor: align telemetry signals with otel semantic conventions

* refactor: unify spans names

- make aspire part of dev stack

* feature: data frame cache telemetry

* feature: integrate flow-php/telemetry with flow-php/postgresql

* feature: telemetry attributes limits

- updated collector to send its own telemetry to aspire
- added asipre/collector to compose.yml.dist
- unified resource name in otlp bridge tests
- unified postgresql/symfony bundle span naming

* feature: introduce name to dataframes

- unify naming conventions across monorepo for telemetry signals

* fix: incorrect tracers behavior about closing spans

- introduce Scope to ContextStorage
- make tracers use Scope when their stack is empty

* refactor: unified naming conventions for filesystem telemetry

* feature: added dataframe name to dataframe metrics/logs/traces attributes

* refactor: unify telemetry conventions for caches

* feature: phpunit telemetry extension

* feature: collect telemetry from local infrastructure

* fix: codecov configuration

* fix: failing tests due to missing otel collector config

* fix: website failing tests

* fix: codding standards

* fix: failing tests

* fix: upgraded otel collector version on ci/cd

fix: otel collector config on ci/cd

Author: norberttech

.codecov.yaml

@@ -43,6 +43,18 @@ component_management:
       name: lib-snappy
       paths:
         - src/lib/snappy/**
+    - component_id: lib-dremel
+      name: lib-dremel
+      paths:
+        - src/lib/dremel/**
+    - component_id: lib-postgresql
+      name: lib-postgresql
+      paths:
+        - src/lib/postgresql/**
+    - component_id: lib-telemetry
+      name: lib-telemetry
+      paths:
+        - src/lib/telemetry/**
     - component_id: bridge-filesystem-async-aws
       name: bridge-filesystem-async-aws
       paths:
@@ -63,6 +75,26 @@ component_management:
       name: symfony-http-foundation
       paths:
         - src/bridge/symfony/http-foundation/**
+    - component_id: bridge-psr18-telemetry
+      name: bridge-psr18-telemetry
+      paths:
+        - src/bridge/psr18/telemetry/**
+    - component_id: bridge-psr7-telemetry
+      name: bridge-psr7-telemetry
+      paths:
+        - src/bridge/psr7/telemetry/**
+    - component_id: bridge-telemetry-otlp
+      name: bridge-telemetry-otlp
+      paths:
+        - src/bridge/telemetry/otlp/**
+    - component_id: bridge-symfony-http-foundation-telemetry
+      name: bridge-symfony-http-foundation-telemetry
+      paths:
+        - src/bridge/symfony/http-foundation-telemetry/**
+    - component_id: bridge-symfony-telemetry-bundle
+      name: bridge-symfony-telemetry-bundle
+      paths:
+        - src/bridge/symfony/telemetry-bundle/**
     - component_id: adapter-chartjs
       name: adapter-chartjs
       paths:
@@ -110,4 +142,16 @@ component_management:
     - component_id: adapter-xml
       name: adapter-xml
       paths:
-        - src/adapter/etl-adapter-xml/**
+        - src/adapter/etl-adapter-xml/**
+    - component_id: adapter-avro
+      name: adapter-avro
+      paths:
+        - src/adapter/etl-adapter-avro/**
+    - component_id: adapter-excel
+      name: adapter-excel
+      paths:
+        - src/adapter/etl-adapter-excel/**
+    - component_id: adapter-postgresql
+      name: adapter-postgresql
+      paths:
+        - src/adapter/etl-adapter-postgresql/**

.github/workflows/job-tests.yml

@@ -76,7 +76,7 @@ jobs:
           docker run -d --name otel-collector \
             -p 4317:4317 -p 4318:4318 -p 8888:8888 -p 13133:13133 \
             -v ${{ github.workspace }}/docker/otel-collector-test-config.yaml:/etc/otelcol-contrib/config.yaml:ro \
-            otel/opentelemetry-collector-contrib:0.115.1
+            otel/opentelemetry-collector-contrib:0.144.0
           # Wait for container to be ready
           for i in {1..30}; do
             if curl -sf http://localhost:13133/ > /dev/null 2>&1; then

.php-cs-fixer.php

@@ -31,6 +31,8 @@
         __DIR__ . '/web/landing/bin',
         __DIR__ . '/examples',
         __DIR__ . '/tools/rector/src',
+        __DIR__ . '/tools/phpunit/extension/telemetry/src',
+        __DIR__ . '/tools/phpunit/extension/telemetry/tests',
     ])
     ->exclude([
         'Flow/Parquet/ThriftModel',

compose.yml.dist

@@ -1,3 +1,7 @@
+networks:
+    flow-php:
+        driver: bridge
+
 services:
     postgres:
         image: postgres:18-alpine
@@ -8,6 +12,8 @@ services:
             - POSTGRES_USER=postgres
             - POSTGRES_PASSWORD=postgres
             - POSTGRES_DB=postgres
+        networks:
+            - flow-php
     mysql:
         image: mysql:8.0.31
         container_name: flow-php-mysql
@@ -18,6 +24,8 @@ services:
             MYSQL_PASSWORD: mysql
             MYSQL_DATABASE: mysql
             MYSQL_ROOT_PASSWORD: root
+        networks:
+            - flow-php
     elasticsearch:
         image: elasticsearch:7.17.2
         container_name: flow-php-elasticsearch
@@ -28,6 +36,8 @@ services:
             - discovery.type=single-node
             - bootstrap.memory_lock=true
             - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
+        networks:
+            - flow-php
     meilisearch:
         image: getmeili/meilisearch:latest
         container_name: flow-php-meilisearch
@@ -36,6 +46,8 @@ services:
         environment:
             - MEILI_MASTER_KEY=masterKey
             - MEILI_NO_ANALYTICS=true
+        networks:
+            - flow-php
     azurite:
         image: mcr.microsoft.com/azure-storage/azurite
         container_name: flow-php-azurite
@@ -48,11 +60,15 @@ services:
             - 10000:10000
         volumes:
             - ./var/azurite:/workspace
+        networks:
+            - flow-php
     redis:
         image: redis:latest
         container_name: flow-php-redis
         ports:
             - "6379:6379"
+        networks:
+            - flow-php
     localstack:
         image: localstack/localstack:latest
         container_name: flow-php-localstack
@@ -64,16 +80,36 @@ services:
             - DATA_DIR=/var/localstack/data
         volumes:
             - "./var/localstack:/var/localstack"
+        networks:
+            - flow-php
+    aspire:
+        image: mcr.microsoft.com/dotnet/aspire-dashboard:latest
+        container_name: flow-php-aspire
+        ports:
+            - "18888:18888"
+            - "18889:18889"
+        environment:
+            - DOTNET_DASHBOARD_UNSECURED_ALLOW_ANONYMOUS=true
+            - ASPIRE_DASHBOARD_OTLP_ENDPOINT_URL=http://0.0.0.0:18889
+        networks:
+            - flow-php
     otel-collector:
-        image: otel/opentelemetry-collector-contrib:0.115.1
+        image: otel/opentelemetry-collector-contrib:0.144.0
         container_name: flow-php-otel-collector
         command: ["--config=/etc/otel-collector-config.yaml"]
         ports:
             - "4317:4317"
             - "4318:4318"
             - "8888:8888"
         volumes:
-            - "./docker/otel-collector-test-config.yaml:/etc/otel-collector-config.yaml:ro"
+            - "./docker/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro"
             - "./var/otel:/var/otel"
+        depends_on:
+            - aspire
+            - postgres
+            - mysql
+            - redis
+        networks:
+            - flow-php
 
 

composer.json

@@ -113,7 +113,8 @@
         "flow-php/symfony-telemetry-bundle": "self.version",
         "flow-php/telemetry": "self.version",
         "flow-php/telemetry-otlp-bridge": "self.version",
-        "flow-php/types": "self.version"
+        "flow-php/types": "self.version",
+        "flow-php/phpunit-telemetry": "self.version"
     },
     "minimum-stability": "dev",
     "prefer-stable": true,
@@ -166,6 +167,9 @@
             ],
             "Flow\\ETL\\Adapter\\Doctrine\\": [
                 "src/adapter/etl-adapter-doctrine/src/Flow/ETL/Adapter/Doctrine"
+            ],
+            "Flow\\Tool\\PHPUnit\\Telemetry\\": [
+                "tools/phpunit/extension/telemetry/src/Flow/Tool/PHPUnit/Telemetry"
             ]
         },
         "files": [
@@ -258,6 +262,9 @@
             ],
             "Flow\\ETL\\Adapter\\Doctrine\\Tests\\": [
                 "src/adapter/etl-adapter-doctrine/tests/Flow/ETL/Adapter/Doctrine/Tests"
+            ],
+            "Flow\\Tool\\PHPUnit\\Telemetry\\Tests\\": [
+                "tools/phpunit/extension/telemetry/tests/Flow/Tool/PHPUnit/Telemetry/Tests"
             ]
         }
     },
@@ -464,6 +471,9 @@
             "tools/phpunit/vendor/bin/phpunit --testsuite=adapter-postgresql-unit --log-junit ./var/phpunit/logs/adapter-postgresql-unit.junit.xml --coverage-clover=./var/phpunit/coverage/clover/adapter-postgresql-unit.coverage.xml",
             "tools/phpunit/vendor/bin/phpunit --testsuite=adapter-postgresql-integration --log-junit ./var/phpunit/logs/adapter-postgresql-integration.junit.xml --coverage-clover=./var/phpunit/coverage/clover/adapter-postgresql-integration.coverage.xml"
         ],
+        "test:tool:phpunit-telemetry": [
+            "tools/phpunit/vendor/bin/phpunit --testsuite=tool-phpunit-telemetry-unit --log-junit ./var/phpunit/logs/tool-phpunit-telemetry-unit.junit.xml --coverage-clover=./var/phpunit/coverage/clover/tool-phpunit-telemetry-unit.coverage.xml"
+        ],
         "test:docs": [
             "docker run -t --rm -v $PWD:/app norberttech/md-link-linter --exclude=vendor --exclude=.scratchpad --exclude=examples --exclude=documentation ."
         ],

docker/.gitignore

@@ -1,2 +1,3 @@
-otel-collector-dev-config.yaml
+otel-collector-*-config.yaml
+!otel-collector-test-config.yaml
 signoz/

docker/otel-collector-config.yaml

@@ -0,0 +1,140 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: 0.0.0.0:4317
+      http:
+        endpoint: 0.0.0.0:4318
+
+  # PostgreSQL metrics receiver
+  # Docs: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/postgresqlreceiver
+  postgresql:
+    endpoint: postgres:5432
+    transport: tcp
+    username: postgres
+    password: postgres
+    databases:
+      - postgres
+    collection_interval: 10s
+    tls:
+      insecure: true
+
+  # MySQL metrics receiver
+  # Docs: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/mysqlreceiver
+  mysql:
+    endpoint: mysql:3306
+    username: root
+    password: root
+    database: mysql
+    collection_interval: 10s
+    tls:
+      insecure: true
+
+  # Redis metrics receiver
+  # Docs: https://github.com/open-telemetry/opentelemetry-collector-contrib/tree/main/receiver/redisreceiver
+  redis:
+    endpoint: redis:6379
+    collection_interval: 10s
+    tls:
+      insecure: true
+
+exporters:
+  debug:
+    verbosity: basic
+  otlp_grpc/aspire:
+    endpoint: aspire:18889
+    tls:
+      insecure: true
+
+processors:
+  # Add service.name to PostgreSQL metrics
+  resource/postgresql:
+    attributes:
+      - key: service.name
+        value: postgresql
+        action: upsert
+      - key: service.namespace
+        value: flow-php
+        action: upsert
+
+  # Add service.name to MySQL metrics
+  resource/mysql:
+    attributes:
+      - key: service.name
+        value: mysql
+        action: upsert
+      - key: service.namespace
+        value: flow-php
+        action: upsert
+
+  # Add service.name to Redis metrics
+  resource/redis:
+    attributes:
+      - key: service.name
+        value: redis
+        action: upsert
+      - key: service.namespace
+        value: flow-php
+        action: upsert
+
+extensions:
+  health_check:
+    endpoint: 0.0.0.0:13133
+
+service:
+  extensions: [health_check]
+  telemetry:
+    traces:
+      sampler:
+        trace_id_ratio_based:
+          ratio: 0.1
+      processors:
+        - batch:
+            exporter:
+              otlp:
+                protocol: http/protobuf
+                endpoint: http://localhost:4318
+    metrics:
+      level: detailed
+      readers:
+        - pull:
+            exporter:
+              prometheus:
+                host: '0.0.0.0'
+                port: 8888
+        - periodic:
+            exporter:
+              otlp:
+                protocol: grpc
+                endpoint: http://aspire:18889
+                insecure: true
+    logs:
+      level: warn
+      processors:
+        - batch:
+            exporter:
+              otlp:
+                protocol: http/protobuf
+                endpoint: http://localhost:4318
+  pipelines:
+    traces:
+      receivers: [otlp]
+      exporters: [debug, otlp_grpc/aspire]
+    metrics:
+      receivers: [otlp]
+      exporters: [debug, otlp_grpc/aspire]
+    metrics/postgresql:
+      receivers: [postgresql]
+      processors: [resource/postgresql]
+      exporters: [debug, otlp_grpc/aspire]
+    metrics/mysql:
+      receivers: [mysql]
+      processors: [resource/mysql]
+      exporters: [debug, otlp_grpc/aspire]
+    metrics/redis:
+      receivers: [redis]
+      processors: [resource/redis]
+      exporters: [debug, otlp_grpc/aspire]
+    logs:
+      receivers: [otlp]
+      exporters: [debug, otlp_grpc/aspire]

docker/otel-collector-test-config.yaml

@@ -15,15 +15,16 @@ extensions:
     endpoint: 0.0.0.0:13133
 
 service:
-  extensions: [health_check]
   telemetry:
     metrics:
+      level: detailed
       readers:
         - pull:
             exporter:
               prometheus:
                 host: '0.0.0.0'
                 port: 8888
+  extensions: [health_check]
   pipelines:
     traces:
       receivers: [otlp]

documentation/components/libs/postgresql.md

@@ -350,6 +350,7 @@ See [Type System](/documentation/components/libs/postgresql/client-types.md) for
 - [Type System](/documentation/components/libs/postgresql/client-types.md) - Value converters, TypedValue, custom types
 - [Query Plan Analysis](/documentation/components/libs/postgresql/client-explain.md) - EXPLAIN ANALYZE, plan insights,
   performance debugging
+- [Telemetry](/documentation/components/libs/postgresql/client-telemetry.md) - OpenTelemetry tracing and metrics
 
 ---