Fix lograge custom request fields

Author: ndbroadbent

.rubocop.yml

@@ -13,11 +13,18 @@ AllCops:
   SuggestExtensions:
     rubocop-rails: false
   Exclude:
-    - node_modules/**/*
+    - '**/node_modules/**/*'
     - public/**/*
     - vendor/**/*
     - tmp/**/*
+    - coverage/**/*
+    - coverage_rails/**/*
+    - docs/public/**/*
     - docspring/**/*
+    - convox_racks_terraform/**/*
+    - terraform-aws-logstruct/**/*
+    - terraform-example/**/*
+    - terraform-provider-logstruct/**/*
     - sorbet/rbi/dsl/**/*
     - sorbet/rbi/gems/**/*
     - sorbet/rbi/annotations/**/*

README.md

@@ -52,6 +52,7 @@ Once initialized (and enabled), the gem automatically includes its modules into
 - `ActiveSupport::TaggedLogging` is patched to support both Hashes and Strings (only when LogStruct is enabled)
 - `ActionMailer::Base` includes error handling and event logging modules
 - We configure `Lograge` for request logging
+- Lograge request logs include request metadata (request_id, source_ip, user_agent, referer, host, content_type, accept) and custom fields from `lograge_custom_options`
 - A Rack middleware is inserted to catch and log errors, including security violations (IP spoofing, CSRF, blocked hosts, etc.)
 - Structured logging is set up for ActiveJob, Sidekiq, Shrine, etc.
 - Rails `config.filter_parameters` are merged into LogStruct's filters and then cleared (to avoid double filtering). Configure sensitive keys via `LogStruct.config.filters`.

docs/app/docs/configuration/page.tsx

@@ -233,6 +233,12 @@ end`}
       </p>
 
       <RubyCodeExample name="lograge_custom_options" />
+      <p className="text-neutral-600 dark:text-neutral-300 mt-4">
+        Custom options are merged into the request log output at the top level. LogStruct also
+        captures request metadata like <code>request_id</code>, <code>source_ip</code>,
+        <code>user_agent</code>, <code>referer</code>, <code>host</code>, <code>content_type</code>,
+        and <code>accept</code> automatically when present.
+      </p>
 
       <HeadingWithAnchor id="custom-string-scrubbing">Custom String Scrubbing</HeadingWithAnchor>
       <p className="text-neutral-600 dark:text-neutral-300 mb-4">

docs/jest.config.ts

@@ -1,5 +1,5 @@
 import type { Config } from 'jest';
-import nextJest from 'next/jest';
+import nextJest from 'next/jest.js';
 
 const createJestConfig = nextJest({
   // Provide the path to your Next.js app

docs/lib/log-generation/sample-data.ts

@@ -193,6 +193,9 @@ export const SampleByLogField: Readonly<Record<LogField, (gen: RandomGen) => unk
   [LogField.UserAgent]: (_gen: RandomGen) => 'Mozilla/5.0',
   [LogField.Referer]: (_gen: RandomGen) => 'https://example.com',
   [LogField.RequestId]: SampleHelpers.hex8,
+  [LogField.Host]: (_gen: RandomGen) => 'example.test',
+  [LogField.ContentType]: (_gen: RandomGen) => 'application/json',
+  [LogField.Accept]: (_gen: RandomGen) => 'application/json',
   [LogField.Controller]: (_gen: RandomGen) => 'HomeController',
   [LogField.Action]: (_gen: RandomGen) => 'index',
   [LogField.View]: (gen: RandomGen) => gen.randomFloat(0, 200),

lib/log_struct/enums/log_field.rb

@@ -33,6 +33,9 @@ class LogField < T::Enum
       UserAgent = new(:user_agent)
       Referer = new(:referer)
       RequestId = new(:request_id)
+      Host = new(:host)
+      ContentType = new(:content_type)
+      Accept = new(:accept)
 
       # HTTP-specific fields
       Format = new(:format)

lib/log_struct/integrations/lograge.rb

@@ -13,6 +13,29 @@ module Integrations
     module Lograge
       extend IntegrationInterface
 
+      LOGRAGE_KNOWN_KEYS = T.let(
+        [
+          :method,
+          :path,
+          :format,
+          :controller,
+          :action,
+          :status,
+          :duration,
+          :view,
+          :db,
+          :params,
+          :request_id,
+          :source_ip,
+          :user_agent,
+          :referer,
+          :host,
+          :content_type,
+          :accept
+        ].freeze,
+        T::Array[Symbol]
+      )
+
       class << self
         extend T::Sig
 
@@ -38,30 +61,9 @@ def configure_lograge(logstruct_config)
             # The struct is converted to JSON by our Formatter (after filtering, etc.)
             config.lograge.formatter = T.let(
               lambda do |data|
-                # Coerce common fields to expected types
-                status = ((s = data[:status]) && s.respond_to?(:to_i)) ? s.to_i : s
-                duration_ms = ((d = data[:duration]) && d.respond_to?(:to_f)) ? d.to_f : d
-                view = ((v = data[:view]) && v.respond_to?(:to_f)) ? v.to_f : v
-                db = ((b = data[:db]) && b.respond_to?(:to_f)) ? b.to_f : b
-
-                params = data[:params]
-                params = params.deep_symbolize_keys if params&.respond_to?(:deep_symbolize_keys)
-
-                Log::Request.new(
-                  http_method: data[:method]&.to_s,
-                  path: data[:path]&.to_s,
-                  format: data[:format]&.to_sym,
-                  controller: data[:controller]&.to_s,
-                  action: data[:action]&.to_s,
-                  status: status,
-                  duration_ms: duration_ms,
-                  view: view,
-                  database: db,
-                  params: params,
-                  timestamp: Time.now
-                )
+                LogStruct::Integrations::Lograge.build_request_log(data)
               end,
-              T.proc.params(hash: T::Hash[Symbol, T.untyped]).returns(Log::Request)
+              T.proc.params(hash: T::Hash[T.any(Symbol, String), T.untyped]).returns(Log::Request)
             )
 
             # Add custom options to lograge
@@ -100,6 +102,7 @@ def process_headers(event, options)
           return if headers.blank?
 
           options[:user_agent] = headers["HTTP_USER_AGENT"]
+          options[:referer] = headers["HTTP_REFERER"]
           options[:content_type] = headers["CONTENT_TYPE"]
           options[:accept] = headers["HTTP_ACCEPT"]
         end
@@ -114,6 +117,66 @@ def apply_custom_options(event, options)
           # The proc can modify the options hash directly
           custom_options_proc.call(event, options)
         end
+
+        sig { params(data: T::Hash[T.any(Symbol, String), T.untyped]).returns(Log::Request) }
+        def build_request_log(data)
+          normalized_data = normalize_lograge_data(data)
+
+          # Coerce common fields to expected types
+          status = ((s = normalized_data[:status]) && s.respond_to?(:to_i)) ? s.to_i : s
+          duration_ms = ((d = normalized_data[:duration]) && d.respond_to?(:to_f)) ? d.to_f : d
+          view = ((v = normalized_data[:view]) && v.respond_to?(:to_f)) ? v.to_f : v
+          db = ((b = normalized_data[:db]) && b.respond_to?(:to_f)) ? b.to_f : b
+
+          params = normalized_data[:params]
+          params = params.deep_symbolize_keys if params&.respond_to?(:deep_symbolize_keys)
+
+          additional_data = extract_additional_data(normalized_data)
+
+          Log::Request.new(
+            http_method: normalized_data[:method]&.to_s,
+            path: normalized_data[:path]&.to_s,
+            format: normalized_data[:format]&.to_sym,
+            controller: normalized_data[:controller]&.to_s,
+            action: normalized_data[:action]&.to_s,
+            status: status,
+            duration_ms: duration_ms,
+            view: view,
+            database: db,
+            params: params,
+            request_id: normalized_data[:request_id]&.to_s,
+            source_ip: normalized_data[:source_ip]&.to_s,
+            user_agent: normalized_data[:user_agent]&.to_s,
+            referer: normalized_data[:referer]&.to_s,
+            host: normalized_data[:host]&.to_s,
+            content_type: normalized_data[:content_type]&.to_s,
+            accept: normalized_data[:accept]&.to_s,
+            additional_data: additional_data,
+            timestamp: Time.now
+          )
+        end
+
+        sig { params(data: T::Hash[T.any(Symbol, String), T.untyped]).returns(T::Hash[Symbol, T.untyped]) }
+        def normalize_lograge_data(data)
+          data.each_with_object({}) do |(key, value), normalized|
+            normalized[key.to_s.to_sym] = value
+          end
+        end
+
+        sig { params(data: T::Hash[Symbol, T.untyped]).returns(T.nilable(T::Hash[Symbol, T.untyped])) }
+        def extract_additional_data(data)
+          extras = T.let({}, T::Hash[Symbol, T.untyped])
+          data.each do |key, value|
+            next if LOGRAGE_KNOWN_KEYS.include?(key)
+            next if value.nil?
+
+            extras[key] = value
+          end
+
+          return nil if extras.empty?
+
+          extras
+        end
       end
     end
   end

lib/log_struct/log/request.rb

@@ -44,6 +44,14 @@ class Request < T::Struct
       const :view, T.nilable(Float), default: nil
       const :database, T.nilable(Float), default: nil
       const :params, T.nilable(T::Hash[Symbol, T.untyped]), default: nil
+      const :host, T.nilable(String), default: nil
+      const :content_type, T.nilable(String), default: nil
+      const :accept, T.nilable(String), default: nil
+
+      # Additional data
+      include LogStruct::Log::Interfaces::AdditionalDataField
+      const :additional_data, T.nilable(T::Hash[T.any(String, Symbol), T.untyped]), default: nil
+      include LogStruct::Log::Shared::MergeAdditionalDataFields
 
       # Request fields (optional)
       include LogStruct::Log::Interfaces::RequestFields
@@ -70,6 +78,9 @@ def to_h
         h[LogField::View] = view unless view.nil?
         h[LogField::Database] = database unless database.nil?
         h[LogField::Params] = params unless params.nil?
+        h[LogField::Host] = host unless host.nil?
+        h[LogField::ContentType] = content_type unless content_type.nil?
+        h[LogField::Accept] = accept unless accept.nil?
         h
       end
     end

lib/log_struct/semantic_logger/formatter.rb

@@ -86,7 +86,17 @@ def call(log, logger)
       private
 
       # Extract a LogStruct from the various places it might be stored in a SemanticLogger::Log
-      sig { params(log: ::SemanticLogger::Log).returns(T.nilable(LogStruct::Log::Interfaces::CommonFields)) }
+      sig do
+        params(log: ::SemanticLogger::Log).returns(
+          T.nilable(
+            T.any(
+              LogStruct::Log::Interfaces::CommonFields,
+              LogStruct::Log::Interfaces::PublicCommonFields,
+              T::Struct
+            )
+          )
+        )
+      end
       def extract_logstruct(log)
         # Check payload first (most common path for structured logging)
         if log.payload.is_a?(Hash) && log.payload[:payload].is_a?(LogStruct::Log::Interfaces::CommonFields)
@@ -97,6 +107,10 @@ def extract_logstruct(log)
           return log.payload
         end
 
+        if log.payload.is_a?(LogStruct::Log::Interfaces::PublicCommonFields)
+          return log.payload
+        end
+
         # Check message - this is where structs end up when passed directly to logger.info(struct)
         if log.message.is_a?(LogStruct::Log::Interfaces::CommonFields)
           return T.cast(log.message, LogStruct::Log::Interfaces::CommonFields)
@@ -106,7 +120,7 @@ def extract_logstruct(log)
         if log.payload.is_a?(Hash) && log.payload[:payload].is_a?(T::Struct)
           struct = log.payload[:payload]
           if struct.respond_to?(:source) && struct.respond_to?(:event)
-            return T.unsafe(struct)
+            return struct
           end
         end
 

schemas/log_sources/request.yml

@@ -4,6 +4,7 @@ consts:
   source: Source::Rails
 
 add_request_fields: true
+additional_data: true
 
 events:
   Request:
@@ -17,3 +18,6 @@ events:
       View: Float
       Database: Float
       Params: 'T::Hash[Symbol, T.untyped]'
+      Host: String
+      ContentType: String
+      Accept: String