feat: add log_format_extra to enrich the default logger format (#13568)

Author: shreemaan-abhishek

apisix/balancer.lua

@@ -252,6 +252,11 @@ local function pick_server(route, ctx)
         local node = up_conf.nodes[1]
         ctx.balancer_ip = node.host
         ctx.balancer_port = node.port
+        ctx.upstream_unresolved_host = node.domain or node.host
+        -- also expose it as an nginx var so it can be used in the access log (http only)
+        if is_http and ctx.var then
+            ctx.var.upstream_unresolved_host = ctx.upstream_unresolved_host
+        end
         node.upstream_host = parse_server_for_upstream_host(node, ctx.upstream_scheme)
         return node
     end
@@ -337,6 +342,11 @@ local function pick_server(route, ctx)
     res.domain = domain
     ctx.balancer_ip = res.host
     ctx.balancer_port = res.port
+    ctx.upstream_unresolved_host = res.domain or res.host
+    -- also expose it as an nginx var so it can be used in the access log (http only)
+    if is_http and ctx.var then
+        ctx.var.upstream_unresolved_host = ctx.upstream_unresolved_host
+    end
     ctx.server_picker = server_picker
     res.upstream_host = parse_server_for_upstream_host(res, ctx.upstream_scheme)
 

apisix/cli/ngx_tpl.lua

@@ -810,6 +810,7 @@ http {
 
             set $upstream_scheme             'http';
             set $upstream_host               $http_host;
+            set $upstream_unresolved_host    '';
             set $upstream_uri                '';
             set $request_line                '';
             set $ctx_ref                     '';

apisix/core/ctx.lua

@@ -190,6 +190,7 @@ do
     local ngx_var_names = {
         upstream_scheme            = true,
         upstream_host              = true,
+        upstream_unresolved_host   = true,
         upstream_upgrade           = true,
         upstream_connection        = true,
         upstream_uri               = true,
@@ -244,6 +245,8 @@ do
         route_name = true,
         service_id = true,
         service_name = true,
+        -- the upstream host before DNS resolution (configured domain/host)
+        upstream_unresolved_host = true,
     }
 
     local mt = {

apisix/plugins/clickhouse-logger.lua

@@ -42,6 +42,7 @@ local schema = {
         name = {type = "string", default = "clickhouse logger"},
         ssl_verify = {type = "boolean", default = true},
         log_format = {type = "object"},
+        log_format_extra = {type = "object"},
         include_req_body = {type = "boolean", default = false},
         include_req_body_expr = {
             type = "array",
@@ -72,6 +73,9 @@ local schema = {
 local metadata_schema = {
     type = "object",
     properties = {
+        log_format_extra = {
+            type = "object"
+        },
         log_format = {
             type = "object"
         },

apisix/plugins/elasticsearch-logger.lua

@@ -55,6 +55,7 @@ local schema = {
             required = {"index"}
         },
         log_format = {type = "object"},
+        log_format_extra = {type = "object"},
         auth = {
             type = "object",
             properties = {
@@ -121,6 +122,9 @@ local schema = {
 local metadata_schema = {
     type = "object",
     properties = {
+        log_format_extra = {
+            type = "object"
+        },
         log_format = {
             type = "object"
         },

apisix/plugins/file-logger.lua

@@ -34,6 +34,7 @@ local schema = {
             type = "string"
         },
         log_format = {type = "object"},
+        log_format_extra = {type = "object"},
         include_req_body = {type = "boolean", default = false},
         include_req_body_expr = {
             type = "array",
@@ -71,6 +72,9 @@ local metadata_schema = {
         },
         log_format = {
             type = "object"
+        },
+        log_format_extra = {
+            type = "object"
         }
     }
 }

apisix/plugins/google-cloud-logging.lua

@@ -18,6 +18,7 @@
 local core            = require("apisix.core")
 local plugin          = require("apisix.plugin")
 local tostring        = tostring
+local pairs           = pairs
 local http            = require("resty.http")
 local log_util        = require("apisix.utils.log-util")
 local bp_manager_mod  = require("apisix.utils.batch-processor-manager")
@@ -98,6 +99,7 @@ local schema = {
             default = "apisix.apache.org%2Flogs"
         },
         log_format = {type = "object"},
+        log_format_extra = {type = "object"},
     },
     oneOf = {
         { required = { "auth_config" } },
@@ -109,6 +111,9 @@ local schema = {
 local metadata_schema = {
     type = "object",
     properties = {
+        log_format_extra = {
+            type = "object"
+        },
         log_format = {
             type = "object"
         },
@@ -190,7 +195,7 @@ end
 
 
 local function get_logger_entry(conf, ctx, oauth)
-    local entry, customized = log_util.get_log_entry(plugin_name, conf, ctx)
+    local entry, customized, extra = log_util.get_log_entry(plugin_name, conf, ctx)
     local google_entry
     if not customized then
         google_entry = {
@@ -210,6 +215,14 @@ local function get_logger_entry(conf, ctx, oauth)
                 service_id = entry.service_id,
             },
         }
+        -- the fixed payload above drops everything else, so add log_format_extra
+        if extra then
+            for k, v in pairs(extra) do
+                if google_entry.jsonPayload[k] == nil then
+                    google_entry.jsonPayload[k] = v
+                end
+            end
+        end
     else
         google_entry = {
             jsonPayload = entry,

apisix/plugins/http-logger.lua

@@ -34,6 +34,7 @@ local schema = {
         auth_header = {type = "string"},
         timeout = {type = "integer", minimum = 1, default = 3},
         log_format = {type = "object"},
+        log_format_extra = {type = "object"},
         include_req_body = {type = "boolean", default = false},
         include_req_body_expr = {
             type = "array",
@@ -64,6 +65,9 @@ local schema = {
 local metadata_schema = {
     type = "object",
     properties = {
+        log_format_extra = {
+            type = "object"
+        },
         log_format = {
             type = "object"
         },

apisix/plugins/kafka-logger.lua

@@ -40,6 +40,7 @@ local schema = {
             enum = {"default", "origin"},
         },
         log_format = {type = "object"},
+        log_format_extra = {type = "object"},
         -- deprecated, use "brokers" instead
         broker_list = {
             type = "object",
@@ -146,6 +147,9 @@ local schema = {
 local metadata_schema = {
     type = "object",
     properties = {
+        log_format_extra = {
+            type = "object"
+        },
         log_format = {
             type = "object"
         },

apisix/plugins/loggly.lua

@@ -95,6 +95,7 @@ local schema = {
             default = true
         },
         log_format = {type = "object"},
+        log_format_extra = {type = "object"},
         severity_map = {
             type = "object",
             description = "upstream response code vs syslog severity mapping",
@@ -143,6 +144,9 @@ local metadata_schema = {
             minimum = 1,
             default= defaults.timeout
         },
+        log_format_extra = {
+            type = "object"
+        },
         log_format = {
             type = "object",
         }