Feature: Enhance observability

.gitignore

@@ -48,6 +48,7 @@ yarn-error.log*
 !.env.example
 *.env
 .env*
+!dev-network/assets/**/.env.*
 
 # Typescript
 tsconfig.tsbuildinfo

.prettierrc

@@ -25,7 +25,11 @@
 			"options": {
 				"plugins": ["@trivago/prettier-plugin-sort-imports"],
 				"parser": "typescript",
-				"importOrder": ["<THIRD_PARTY_MODULES>", "^[./]"],
+				"importOrder": [
+					".*startOpenTelemetry*",
+					"<THIRD_PARTY_MODULES>",
+					"^[./]"
+				],
 				"importOrderParserPlugins": ["typescript", "decorators-legacy"],
 				"importOrderSeparation": true,
 				"trailingComma": "es5",

dev-network/Dockerfile.base

@@ -27,6 +27,7 @@ COPY --chown=node:node ./packages/broker/package.json ./packages/broker/
 COPY --chown=node:node ./packages/validator/package.json ./packages/validator/
 COPY --chown=node:node ./packages/heartbeat/package.json ./packages/heartbeat/
 COPY --chown=node:node ./packages/benchmarks/package.json ./packages/benchmarks/
+COPY --chown=node:node ./packages/observer/package.json ./packages/observer/
 
 RUN pnpm install
 

dev-network/Dockerfile.observer

@@ -0,0 +1,8 @@
+FROM logstore-base
+
+USER root
+
+USER node
+WORKDIR /home/node/logstore
+
+CMD [ "start-in-docker" ]

dev-network/assets/broker/.env.broker-1

@@ -3,5 +3,7 @@ LOG_LEVEL=debug
 # node-fetch tweak to call arweave.net with self-signed certificate
 # https://stackoverflow.com/a/52479399
 NODE_TLS_REJECT_UNAUTHORIZED=0
+METRICS_URL=http://opentelemetry-collector:4317
+TRACING_URL=http://opentelemetry-collector:4317
 
 BROKER_METADATA='{ "http": "http://logstore-broker-1:7771" }'

dev-network/assets/broker/.env.broker-2

@@ -3,5 +3,7 @@ LOG_LEVEL=debug
 # node-fetch tweak to call arweave.net with self-signed certificate
 # https://stackoverflow.com/a/52479399
 NODE_TLS_REJECT_UNAUTHORIZED=0
+METRICS_URL=http://opentelemetry-collector:4317
+TRACING_URL=http://opentelemetry-collector:4317
 
 BROKER_METADATA='{ "http": "http://logstore-broker-2:7772" }'

dev-network/assets/broker/.env.broker-3

@@ -3,5 +3,7 @@ LOG_LEVEL=debug
 # node-fetch tweak to call arweave.net with self-signed certificate
 # https://stackoverflow.com/a/52479399
 NODE_TLS_REJECT_UNAUTHORIZED=0
+METRICS_URL=http://opentelemetry-collector:4317
+TRACING_URL=http://opentelemetry-collector:4317
 
 BROKER_METADATA='{ "http": "http://logstore-broker-3:7773" }'

dev-network/assets/observer/.env.observer

@@ -0,0 +1,7 @@
+LOG_LEVEL=debug
+
+# node-fetch tweak to call arweave.net with self-signed certificate
+# https://stackoverflow.com/a/52479399
+NODE_TLS_REJECT_UNAUTHORIZED=0
+METRICS_URL=http://opentelemetry-collector:4317
+TRACING_URL=http://opentelemetry-collector:4317

dev-network/assets/observer/observer.json

@@ -0,0 +1,56 @@
+{
+	"$schema": "https://schema.streamr.network/config-v2.schema.json",
+	"client": {
+		"logLevel": "debug",
+		"auth": {
+			"privateKey": "0x2222222222222222222222222222222222222222222222222222222222222222"
+		},
+		"network": {
+			"id": "0x1563915e194d8cfba1943570603f7606a3115508",
+			"trackers": {
+				"contractAddress": "0xBFCF120a8fD17670536f1B27D9737B775b2FD4CF"
+			},
+			"location": {
+				"latitude": 60.19,
+				"longitude": 24.95,
+				"country": "Finland",
+				"city": "Helsinki"
+			},
+			"webrtcDisallowPrivateAddresses": false
+		},
+		"contracts": {
+			"streamRegistryChainAddress": "0x6cCdd5d866ea766f6DF5965aA98DeCCD629ff222",
+			"streamStorageRegistryChainAddress": "0xd04af489677001444280366Dd0885B03dAaDe71D",
+			"storageNodeRegistryChainAddress": "0x231b810D98702782963472e1D60a25496999E75D",
+			"logStoreNodeManagerChainAddress": "0x85ac4C8E780eae81Dd538053D596E382495f7Db9",
+			"logStoreStoreManagerChainAddress": "0x8560200b8E7477FB09281A0566B50fa6E7a66a34",
+			"streamRegistryChainRPCs": {
+				"chainId": 8997,
+				"rpcs": [
+					{
+						"url": "http://10.200.10.1:8546"
+					}
+				]
+			},
+			"mainChainRPCs": {
+				"chainId": 8995,
+				"rpcs": [
+					{
+						"url": "http://10.200.10.1:8545"
+					}
+				]
+			},
+			"theGraphUrl": "http://10.200.10.1:8000/subgraphs/name/streamr-dev/network-contracts",
+			"logStoreTheGraphUrl": "http://10.200.10.1:8000/subgraphs/name/logstore-dev/network-contracts"
+		},
+		"metrics": false
+	},
+	"pool": {
+		"id": "0",
+		"url": "http://logstore-kyve:1317",
+		"pollInterval": 60000
+	},
+	"plugins": {
+		"observer": {}
+	}
+}

dev-network/assets/observer/start-in-docker.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+pnpm --filter @logsn/observer start:prod

dev-network/assets/opentelemetry-collector/otel-collector-config.yaml

@@ -0,0 +1,23 @@
+receivers:
+  otlp:
+    protocols:
+      grpc:
+        endpoint: ":4317"
+
+processors:
+  batch:
+
+exporters:
+  prometheus:
+    endpoint: "0.0.0.0:8889"
+  otlphttp:
+    endpoint: http://tempo:4318
+
+service:
+  pipelines:
+    metrics:
+      receivers: [ otlp ]
+      exporters: [ prometheus ]
+    traces:
+      receivers: [ otlp ]
+      exporters: [ otlphttp ]

dev-network/assets/prometheus/prometheus.yml

@@ -0,0 +1,5 @@
+scrape_configs:
+  - job_name: 'otel-collector'
+    scrape_interval: 15s
+    static_configs:
+      - targets: ['opentelemetry-collector:8889']

dev-network/assets/tempo/tempo.yaml

@@ -0,0 +1,34 @@
+auth_enabled: false  # Authentication. TODO
+
+server:
+  http_listen_port: 3200  # Port for HTTP API
+
+ingester:
+  lifecycler:
+    ring:
+      kvstore:
+        store: inmemory  # In-memory ring store. For production, consider using etcd or Consul.
+      replication_factor: 1  # Number of the replicas for the ingested data.
+    final_sleep: 0s  # Time to sleep between shutdown and exiting the ingester.
+
+distributor:
+  receivers:  # Supported protocols for tracing ingestion.
+    jaeger:
+      protocols:
+        grpc:
+        thrift_binary:
+        thrift_compact:
+    zipkin:
+    otlp:
+      protocols:
+        grpc:
+        http:
+
+storage:
+  trace:
+    backend: local  # local storage backend. For production, we might want to use cloud
+    wal:
+      path: /tmp/tempo/wal  # Write-ahead log (WAL) directory path. Make sure to mount this directory in Docker.
+    local:
+      path: /tmp/tempo/traces  # Traces storage directory path. Make sure to mount this directory in Docker.
+

dev-network/bin/scripts/connect.sh

@@ -20,7 +20,9 @@ $SSH \
   -L 443:localhost:443 \
   -L 1317:localhost:1317 \
   -L 3000:localhost:3000 \
+  -L 3300:localhost:3300 \
   -L 4001:localhost:4001 \
+  -L 4317:localhost:4317 \
   -L 5432:localhost:5432 \
   -L 7771:localhost:7771 \
   -L 7772:localhost:7772 \

dev-network/docker-compose.yml

@@ -339,6 +339,105 @@ services:
       logstore-deploy-contracts:
         condition: service_completed_successfully
 
+  logstore-observer:
+    container_name: logstore-observer
+    hostname: logstore-observer
+    image: logstore-observer
+    build:
+      context: ../
+      dockerfile: ./dev-network/Dockerfile.observer
+    restart: always
+    networks:
+      - streamr-network
+    env_file:
+      - ./assets/observer/.env.observer
+    volumes:
+      - type: bind
+        source: ./assets/observer/start-in-docker.sh
+        target: /usr/local/bin/start-in-docker
+      - type: bind
+        source: ./assets/observer/observer.json
+        target: /home/node/.logstore/config/observer.json
+    depends_on:
+      logstore-base:
+        condition: service_completed_successfully
+      logstore-deploy-subgraph:
+        condition: service_completed_successfully
+      logstore-deploy-contracts:
+        condition: service_completed_successfully
+
+  logstore-grafana:
+    image: grafana/grafana:latest
+    container_name: logstore-grafana
+    hostname: logstore-grafana
+    restart: always
+    networks:
+      - streamr-network
+    ports:
+      - '3300:3000'
+    volumes:
+      - grafana-storage:/var/lib/grafana
+    depends_on:
+      - logstore-prometheus
+
+  opentelemetry-collector:
+    image: otel/opentelemetry-collector:0.85.0
+    container_name: opentelemetry-collector
+    hostname: opentelemetry-collector
+    restart: always
+    ports:
+      - '4317:4317'
+    networks:
+      - streamr-network
+    volumes:
+      - ./assets/opentelemetry-collector/otel-collector-config.yaml:/etc/otel-collector-config.yaml:ro
+    command: ['--config=/etc/otel-collector-config.yaml']
+
+  logstore-prometheus:
+    image: prom/prometheus:v2.47.0
+    container_name: logstore-prometheus
+    hostname: logstore-prometheus
+    restart: always
+    networks:
+      - streamr-network
+    command:
+      [
+        '--config.file=/etc/prometheus/prometheus.yml',
+        '--storage.tsdb.path=/prometheus',
+        '--web.enable-admin-api',
+      ]
+    ports:
+      - '9090:9090'
+    volumes:
+      - ./assets/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - type: volume
+        source: data-logstore-prometheus
+        target: /prometheus
+        volume:
+          nocopy: false
+    depends_on:
+      - opentelemetry-collector
+
+  tempo:
+    image: grafana/tempo:main-6c7e07d # Use the latest version
+    container_name: tempo
+    hostname: tempo
+    restart: always
+    networks:
+      - streamr-network
+    ports:
+      - '3200:3200' # Tempo's default port
+    command: ['-config.file=/etc/tempo/tempo.yaml']
+    volumes:
+      - ./assets/tempo/tempo.yaml:/etc/tempo/tempo.yaml:ro
+      - type: volume
+        source: data-tempo
+        target: /tmp/tempo
+        volume:
+          nocopy: false
+    depends_on:
+      - opentelemetry-collector
+
   logstore-vector:
     container_name: logstore-vector
     hostname: logstore-vector
@@ -365,3 +464,6 @@ volumes:
   data-heartbeat:
   data-logstore-deploy-contracts:
   data-logstore-deploy-subgraph:
+  data-logstore-prometheus:
+  grafana-storage:
+  data-tempo:

dev-network/docs/observer.md

@@ -0,0 +1,6 @@
+# LogStore DevNetwork - Observer
+
+| Properby    | Value                                                              |
+| ----------- | ------------------------------------------------------------------ |
+| Address     | `0x1563915e194d8cfba1943570603f7606a3115508`                       |
+| Private Key | `2222222222222222222222222222222222222222222222222222222222222222` |

dev-network/docs/ports.md

@@ -7,10 +7,12 @@
 | 443   | Streamr APP                     | streamr-dev-nginx                    | Streamr  |
 | 1317  | KYVE REST API                   | logstore-nginx                       | LogStore |
 | 1984  | Arweave                         | logstore-arweave                     | LogStore |
+| 3300  | Grafana                         | logstore-grafana                     | Logstore |
 | 3306  | MySql                           | streamr-dev-mysql                    | Streamr  |
 | 3333  |                                 | streamr-dev-platform                 | Streamr  |
 | 3334  |                                 | streamr-dev-network-explorer         | Streamr  |
 | 4001  |                                 | streamr-dev-stream-metrics-index     | Streamr  |
+| 4317  | OpenTelemetry Collector         | opentelemetry-collector              | LogStore |
 | 5001  |                                 | streamr-dev-ipfs                     | Streamr  |
 | 5432  |                                 | streamr-dev-postgres                 | Streamr  |
 | 6379  | Redis                           | streamr-dev-redis                    | Streamr  |

package.json

@@ -35,6 +35,6 @@
 		"prettier-2": "npm:prettier@^2",
 		"prettier-plugin-solidity": "^1.1.3",
 		"turbo": "^1.8.3",
-		"typescript": "^4.9.5"
+		"typescript": "^5.2.2"
 	}
 }

packages/broker/jest.config.js

@@ -1,7 +1,11 @@
-/** @type {import('ts-jest').JestConfigWithTsJest} */
+// we must use require here because the OTelEnvironment needs some env vars
+const { config } = require('dotenv');
+config();
+
+/** @type {import("ts-jest").JestConfigWithTsJest} */
 module.exports = {
 	preset: 'ts-jest/presets/js-with-ts',
-	testEnvironment: 'node',
+	testEnvironment: './test/OTelEnvironment.ts',
 	clearMocks: true,
 	// can't use prettier 3 with jest
 	prettierPath: require.resolve('prettier-2'),
@@ -10,5 +14,6 @@ module.exports = {
 			tsconfig: 'tsconfig.jest.json',
 		},
 	},
-	setupFilesAfterEnv: ['jest-extended/all', 'dotenv/config'],
+	setupFiles: ['dotenv/config'],
+	setupFilesAfterEnv: ['jest-extended/all'],
 };

packages/broker/package.json

@@ -43,6 +43,16 @@
 		"@streamr/network-tracker": "8.1",
 		"@streamr/protocol": "^8.1.0",
 		"@streamr/utils": "^8.1.0",
+		"@opentelemetry/api": "^1.6.0",
+		"@opentelemetry/context-async-hooks": "^1.17.0",
+		"@opentelemetry/auto-instrumentations-node": "^0.39.2",
+		"@opentelemetry/sdk-node": "^0.43.0",
+		"@opentelemetry/exporter-metrics-otlp-grpc": "^0.43.0",
+		"@opentelemetry/sdk-metrics": "^1.17.0",
+		"@opentelemetry/exporter-trace-otlp-grpc": "^0.43.0",
+		"@opentelemetry/instrumentation-http": "^0.43.0",
+		"@opentelemetry/semantic-conventions": "^1.17.0",
+		"@opentelemetry/resources": "^1.17.0",
 		"ajv": "8.8.2",
 		"ajv-formats": "^2.1.1",
 		"axios": "^1.4.0",
@@ -76,6 +86,7 @@
 		"@types/heap": "^0.2.31",
 		"@types/inquirer": "^8.1.3",
 		"@types/jest": "^29.5.0",
+		"jest-environment-node": "^29.6.2",
 		"@types/lodash": "^4.14.191",
 		"@types/merge2": "^1.4.0",
 		"@types/node-fetch": "^2.6.2",