Merge pull request #1108 from code0-tech/882-add-opentelemetry-support

Add OpenTelemetry support

Author: Taucher2003

Gemfile

@@ -95,3 +95,10 @@ gem 'json-schema', '~> 6.0'
 gem 'triangulum', '0.26.2'
 
 gem 'benchmark'
+
+# OpenTelemetry
+gem 'opentelemetry-exporter-otlp', '~> 0.34.0' # we need this to get traces
+gem 'opentelemetry-exporter-otlp-logs', '~> 0.5.1' # we need this to get logs
+gem 'opentelemetry-exporter-otlp-metrics', '~> 0.10.0' # we need this to get metrics
+gem 'opentelemetry-instrumentation-all', '~> 0.94.0' # we need this for logs and traces
+gem 'opentelemetry-instrumentation-logger', '~> 0.4.0' # we need this to get logs

Gemfile.lock

@@ -212,6 +212,214 @@ GEM
       mini_portile2 (~> 2.8.2)
       racc (~> 1.4)
     open3 (0.2.1)
+    opentelemetry-api (1.10.0)
+      logger
+    opentelemetry-common (0.25.0)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-exporter-otlp (0.34.0)
+      google-protobuf (>= 3.18)
+      googleapis-common-protos-types (~> 1.3)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-sdk (~> 1.10)
+      opentelemetry-semantic_conventions
+    opentelemetry-exporter-otlp-logs (0.5.1)
+      google-protobuf (>= 3.18)
+      googleapis-common-protos-types (~> 1.3)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-logs-api (~> 0.1)
+      opentelemetry-logs-sdk (~> 0.1)
+      opentelemetry-sdk
+      opentelemetry-semantic_conventions
+    opentelemetry-exporter-otlp-metrics (0.10.0)
+      google-protobuf (>= 3.18, < 5.0)
+      googleapis-common-protos-types (~> 1.3)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-metrics-api (~> 0.2)
+      opentelemetry-metrics-sdk (~> 0.5)
+      opentelemetry-sdk (~> 1.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-helpers-mysql (0.6.0)
+      opentelemetry-api (~> 1.7)
+      opentelemetry-common (~> 0.21)
+    opentelemetry-helpers-sql (0.4.0)
+      opentelemetry-api (~> 1.7)
+    opentelemetry-helpers-sql-processor (0.5.0)
+      opentelemetry-api (~> 1.0)
+      opentelemetry-common (~> 0.21)
+    opentelemetry-instrumentation-action_mailer (0.8.1)
+      opentelemetry-instrumentation-active_support (~> 0.10)
+    opentelemetry-instrumentation-action_pack (0.18.0)
+      opentelemetry-instrumentation-rack (~> 0.29)
+    opentelemetry-instrumentation-action_view (0.13.0)
+      opentelemetry-instrumentation-active_support (~> 0.10)
+    opentelemetry-instrumentation-active_job (0.13.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-active_model_serializers (0.25.0)
+      opentelemetry-instrumentation-active_support (>= 0.7.0)
+    opentelemetry-instrumentation-active_record (0.13.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-active_storage (0.5.1)
+      opentelemetry-instrumentation-active_support (~> 0.10)
+    opentelemetry-instrumentation-active_support (0.12.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-all (0.94.0)
+      opentelemetry-instrumentation-active_model_serializers (~> 0.25.0)
+      opentelemetry-instrumentation-anthropic (~> 0.5.0)
+      opentelemetry-instrumentation-aws_lambda (~> 0.7.0)
+      opentelemetry-instrumentation-aws_sdk (~> 0.12.0)
+      opentelemetry-instrumentation-bunny (~> 0.25.0)
+      opentelemetry-instrumentation-concurrent_ruby (~> 0.25.0)
+      opentelemetry-instrumentation-dalli (~> 0.30.0)
+      opentelemetry-instrumentation-delayed_job (~> 0.26.0)
+      opentelemetry-instrumentation-ethon (~> 0.29.0)
+      opentelemetry-instrumentation-excon (~> 0.29.0)
+      opentelemetry-instrumentation-faraday (~> 0.33.0)
+      opentelemetry-instrumentation-grape (~> 0.7.0)
+      opentelemetry-instrumentation-graphql (~> 0.32.0)
+      opentelemetry-instrumentation-grpc (~> 0.5.0)
+      opentelemetry-instrumentation-gruf (~> 0.6.0)
+      opentelemetry-instrumentation-http (~> 0.30.0)
+      opentelemetry-instrumentation-http_client (~> 0.29.0)
+      opentelemetry-instrumentation-httpx (~> 0.8.0)
+      opentelemetry-instrumentation-koala (~> 0.24.0)
+      opentelemetry-instrumentation-lmdb (~> 0.26.0)
+      opentelemetry-instrumentation-mongo (~> 0.26.0)
+      opentelemetry-instrumentation-mysql2 (~> 0.34.0)
+      opentelemetry-instrumentation-net_http (~> 0.29.0)
+      opentelemetry-instrumentation-pg (~> 0.36.0)
+      opentelemetry-instrumentation-que (~> 0.13.0)
+      opentelemetry-instrumentation-racecar (~> 0.7.0)
+      opentelemetry-instrumentation-rack (~> 0.31.0)
+      opentelemetry-instrumentation-rails (~> 0.42.0)
+      opentelemetry-instrumentation-rake (~> 0.6.0)
+      opentelemetry-instrumentation-rdkafka (~> 0.10.0)
+      opentelemetry-instrumentation-redis (~> 0.29.0)
+      opentelemetry-instrumentation-resque (~> 0.9.0)
+      opentelemetry-instrumentation-restclient (~> 0.28.0)
+      opentelemetry-instrumentation-ruby_kafka (~> 0.25.0)
+      opentelemetry-instrumentation-sidekiq (~> 0.29.0)
+      opentelemetry-instrumentation-sinatra (~> 0.30.0)
+      opentelemetry-instrumentation-trilogy (~> 0.69.0)
+    opentelemetry-instrumentation-anthropic (0.5.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-aws_lambda (0.7.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-aws_sdk (0.12.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-base (0.26.1)
+      opentelemetry-api (~> 1.7)
+      opentelemetry-common (~> 0.21)
+      opentelemetry-registry (~> 0.1)
+    opentelemetry-instrumentation-bunny (0.25.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-concurrent_ruby (0.25.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-dalli (0.30.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-delayed_job (0.26.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-ethon (0.29.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-excon (0.29.1)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-faraday (0.33.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-grape (0.7.0)
+      opentelemetry-instrumentation-rack (~> 0.29)
+    opentelemetry-instrumentation-graphql (0.32.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-grpc (0.5.1)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-gruf (0.6.1)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-http (0.30.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-http_client (0.29.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-httpx (0.8.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-koala (0.24.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-lmdb (0.26.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-logger (0.4.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+      opentelemetry-logs-api (~> 0.1)
+    opentelemetry-instrumentation-mongo (0.26.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-mysql2 (0.34.0)
+      opentelemetry-helpers-mysql
+      opentelemetry-helpers-sql
+      opentelemetry-helpers-sql-processor
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-net_http (0.29.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-pg (0.36.0)
+      opentelemetry-helpers-sql
+      opentelemetry-helpers-sql-processor
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-que (0.13.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-racecar (0.7.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-rack (0.31.1)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-rails (0.42.0)
+      opentelemetry-instrumentation-action_mailer (~> 0.7)
+      opentelemetry-instrumentation-action_pack (~> 0.17)
+      opentelemetry-instrumentation-action_view (~> 0.12)
+      opentelemetry-instrumentation-active_job (~> 0.11)
+      opentelemetry-instrumentation-active_record (~> 0.12)
+      opentelemetry-instrumentation-active_storage (~> 0.4)
+      opentelemetry-instrumentation-active_support (~> 0.11)
+      opentelemetry-instrumentation-concurrent_ruby (~> 0.25)
+    opentelemetry-instrumentation-rake (0.6.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-rdkafka (0.10.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-redis (0.29.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-resque (0.9.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-restclient (0.28.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-ruby_kafka (0.25.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-sidekiq (0.29.0)
+      opentelemetry-instrumentation-base (~> 0.25)
+    opentelemetry-instrumentation-sinatra (0.30.0)
+      opentelemetry-instrumentation-rack (~> 0.29)
+    opentelemetry-instrumentation-trilogy (0.69.0)
+      opentelemetry-helpers-mysql
+      opentelemetry-helpers-sql
+      opentelemetry-helpers-sql-processor
+      opentelemetry-instrumentation-base (~> 0.25)
+      opentelemetry-semantic_conventions (>= 1.8.0)
+    opentelemetry-logs-api (0.4.0)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-logs-sdk (0.6.0)
+      opentelemetry-api (~> 1.2)
+      opentelemetry-logs-api (~> 0.1)
+      opentelemetry-sdk (~> 1.3)
+    opentelemetry-metrics-api (0.6.0)
+      opentelemetry-api (~> 1.0)
+    opentelemetry-metrics-sdk (0.15.0)
+      opentelemetry-api (~> 1.1)
+      opentelemetry-metrics-api (~> 0.2)
+      opentelemetry-sdk (~> 1.2)
+    opentelemetry-registry (0.6.0)
+      opentelemetry-api (~> 1.1)
+    opentelemetry-sdk (1.12.0)
+      logger
+      opentelemetry-api (~> 1.1)
+      opentelemetry-common (~> 0.20)
+      opentelemetry-registry (~> 0.2)
+      opentelemetry-semantic_conventions
+    opentelemetry-semantic_conventions (1.41.0)
+      opentelemetry-api (~> 1.0)
     parallel (1.28.0)
     parser (3.3.11.1)
       ast (~> 2.4.1)
@@ -429,6 +637,11 @@ DEPENDENCIES
   image_processing (>= 1.2)
   json-schema (~> 6.0)
   lograge (~> 0.14.0)
+  opentelemetry-exporter-otlp (~> 0.34.0)
+  opentelemetry-exporter-otlp-logs (~> 0.5.1)
+  opentelemetry-exporter-otlp-metrics (~> 0.10.0)
+  opentelemetry-instrumentation-all (~> 0.94.0)
+  opentelemetry-instrumentation-logger (~> 0.4.0)
   pg (~> 1.1)
   pry (~> 0.16.0)
   pry-byebug (~> 3.10)

app/grpc/concerns/grpc_stream_handler.rb

@@ -4,6 +4,12 @@ module GrpcStreamHandler
   include Code0::ZeroTrack::Loggable
   extend ActiveSupport::Concern
 
+  StreamItem = Struct.new(:data, :otel_context, keyword_init: true)
+
+  def self.tracer
+    @tracer ||= ::OpenTelemetry.tracer_provider.tracer('sagittarius-grpc-stream')
+  end
+
   class_methods do
     def grpc_stream(method)
       define_method(method) do |_, call|
@@ -17,17 +23,23 @@ def grpc_stream(method)
                                       grpc_object: grpc_object)
 
         encoded_data = send('encoders')[method].call(grpc_object)
-        encoded_data64 = Base64.encode64(encoded_data)
+        encoded_data64 = Base64.encode64(encoded_data).delete("\n")
 
         GrpcStreamHandler.logger.info(message: 'Encoded data', runtime_id: runtime_id, method: method,
                                       encoded_data: encoded_data64)
 
+        carrier = {}
+        OpenTelemetry.propagation.inject(carrier)
+        trace_context64 = Base64.encode64(carrier.to_json).delete("\n")
+
+        notification_payload = "#{self},#{method},#{runtime_id},#{encoded_data64},#{trace_context64}"
+
         ActiveRecord::Base.connection.raw_connection
-                          .exec("NOTIFY grpc_streams, '#{self},#{method},#{runtime_id},#{encoded_data64}'")
+                          .exec("NOTIFY grpc_streams, '#{notification_payload}'")
       end
       define_singleton_method("end_#{method}") do |runtime_id|
         ActiveRecord::Base.connection.raw_connection
-                          .exec("NOTIFY grpc_streams, '#{self},#{method},#{runtime_id},end'")
+                          .exec("NOTIFY grpc_streams, '#{self},#{method},#{runtime_id},end,'")
       end
     end
   end
@@ -48,11 +60,18 @@ def self.listen!
 
         conn.wait_for_notify(1) do |_, _, payload|
           logger.info(message: 'Received notification', payload: payload)
-          class_name, method_name, runtime_id, encoded_data64 = payload.split(',')
+          parts = payload.split(',')
+          class_name = parts[0]
+          method_name = parts[1]
+          runtime_id = parts[2]
+          encoded_data64 = parts[3]
+          trace_context64 = parts[4]
 
           clazz = class_name.constantize
           method_name = method_name.to_sym
 
+          otel_context = extract_otel_context(trace_context64)
+
           if encoded_data64 == 'end'
             decoded_data = :end
           else
@@ -62,7 +81,7 @@ def self.listen!
 
           queues = GrpcStreamHandler.yielders.dig(clazz, method_name, runtime_id.to_i)
           queues&.each do |queue|
-            queue << decoded_data
+            queue << StreamItem.new(data: decoded_data, otel_context: otel_context)
           rescue StandardError => e
             logger.error(message: 'Error while yielding data', error: e.message, backtrace: e.backtrace)
           end
@@ -73,6 +92,15 @@ def self.listen!
     end
   end
 
+  def self.extract_otel_context(trace_context64)
+    return nil if trace_context64.blank?
+
+    carrier = JSON.parse(Base64.decode64(trace_context64))
+    OpenTelemetry.propagation.extract(carrier)
+  rescue StandardError
+    nil
+  end
+
   def create_enumerator(clazz, method, runtime_id, _call)
     logger.debug(message: 'Creating enumerator', runtime_id: runtime_id, clazz: clazz, method: method)
 
@@ -82,7 +110,7 @@ def create_enumerator(clazz, method, runtime_id, _call)
     method_queues = queues[method] ||= {}
     runtime_queues = method_queues[runtime_id] ||= []
 
-    runtime_queues.each { |existing_queue| existing_queue << :end }
+    runtime_queues.each { |existing_queue| existing_queue << StreamItem.new(data: :end, otel_context: nil) }
     runtime_queues.clear
     runtime_queues << queue
 
@@ -92,19 +120,27 @@ def create_enumerator(clazz, method, runtime_id, _call)
       loop do
         item = queue.pop(timeout: 1)
         next if item.nil?
-        break if item == :end
-
-        begin
-          y << item
-          ApplicationRecord.connection_pool.with_connection do
-            Runtime.update(runtime_id, last_heartbeat: Time.zone.now)
+        break if item.data == :end
+
+        otel_context = item.otel_context || OpenTelemetry::Context.current
+        OpenTelemetry::Context.with_current(otel_context) do
+          GrpcStreamHandler.tracer.in_span("#{clazz}/#{method} send") do |span|
+            y << item.data
+            ApplicationRecord.connection_pool.with_connection do
+              Runtime.update(runtime_id, last_heartbeat: Time.zone.now)
+            end
+          rescue ActiveRecord::ActiveRecordError => e
+            logger.warn(message: 'Failed to update runtime heartbeat', exception: e.message, backtrace: e.backtrace)
+          rescue GRPC::Core::CallError => e
+            logger.info(message: 'Stream was closed from client side (probably)')
+
+            span.set_attribute('rpc.response.status_code', 'UNKNOWN')
+            span.set_attribute('error.type', e.class.name)
+            span.status = ::OpenTelemetry::Trace::Status.error(e.message)
+            span.record_exception(e)
+
+            raise
           end
-        rescue ActiveRecord::ActiveRecordError => e
-          logger.warn(message: 'Failed to update runtime heartbeat', exception: e.message, backtrace: e.backtrace)
-        rescue GRPC::Core::CallError
-          logger.info(message: 'Stream was closed from client side (probably)')
-
-          raise
         end
       end
     ensure

app/grpc/execution_handler.rb

@@ -38,7 +38,7 @@ def test(requests, call)
     rescue StandardError => e
       logger.error(message: 'Error reading execution stream', error: e.message,
                    backtrace: e.backtrace)
-      outbound_queue << :end if outbound_queue
+      outbound_queue << GrpcStreamHandler::StreamItem.new(data: :end, otel_context: nil) if outbound_queue
     ensure
       logger.info(message: 'Execution runtime request stream closed')
     end