Add support for custom logging field (#12872)

This adds TSLogFieldRegister to enable plugins to add or redefine access log fields.

Author: maskit

doc/developer-guide/api/functions/TSLifecycleHookAdd.en.rst

@@ -120,6 +120,12 @@ Types
 
       Invoked with the event :enumerator:`TS_EVENT_LIFECYCLE_SHUTDOWN` and ``nullptr`` data.
 
+   .. cpp:enumerator:: TS_LIFECYCLE_LOG_INITIALIZED_HOOK
+
+      Called after |TS| logging system is initialized but before logging configuration is loaded.
+
+      Invoked with the event :enumerator:`TS_EVENT_LIFECYCLE_LOG_INITIALIZED` and ``nullptr`` data.
+
 .. struct:: TSPluginMsg
 
    The data for the plugin message event :enumerator:`TS_EVENT_LIFECYCLE_MSG`.

doc/developer-guide/api/functions/TSLogFieldRegister.en.rst

@@ -0,0 +1,99 @@
+.. Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed
+   with this work for additional information regarding copyright
+   ownership.  The ASF licenses this file to you under the Apache
+   License, Version 2.0 (the "License"); you may not use this file
+   except in compliance with the License.  You may obtain a copy of
+   the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+   implied.  See the License for the specific language governing
+   permissions and limitations under the License.
+
+.. include:: ../../../common.defs
+
+.. default-domain:: cpp
+
+TSLogFieldRegister
+******************
+
+Registers a custom log field, or modifies an existing log field with a new definition.
+
+Synopsis
+========
+
+.. code-block:: cpp
+
+    #include <ts/ts.h>
+
+.. function:: TSReturnCode TSLogFieldRegister(std::string_view name, std::string_view symbol, TSLogType type, TSLogMarshalCallback marshal_cb, TSLogUnmarshalCallback unmarshal_cb, bool replace = false);
+
+.. enum:: TSLogType
+
+   Specify the type of a log field
+
+   .. enumerator:: TS_LOG_TYPE_INT
+
+      Integer field.
+
+   .. enumerator:: TS_LOG_TYPE_STRING
+
+      String field.
+
+   .. enumerator:: TS_LOG_TYPE_ADDR
+
+      Address field. It supports IPv4 address, IPv6 address, and Unix Domain Socket address (path).
+
+.. type:: int (*TSLogMarshalCallback)(TSHttpTxn, char *);
+
+   Callback signature for functions to marshal log fields.
+
+.. type:: std::tuple<int, int> (*TSLogUnmarshalCallback)(char **, char *, int);
+
+   Callback signature for functions to unmarshal log fields.
+
+.. function:: int TSLogStringMarshal(char *buf, std::string_view str);
+.. function:: int TSLogIntMarshal(char *buf, int64_t value);
+.. function:: int TSLogAddrMarshal(char *buf, sockaddr *addr);
+.. function:: std::tuple<int, int> TSLogStringUnmarshal(char **buf, char *dest, int len);
+.. function:: std::tuple<int, int> TSLogIntUnmarshal(char **buf, char *dest, int len);
+.. function:: std::tuple<int, int> TSLogAddrUnmarshal(char **buf, char *dest, int len);
+
+   Predefined marshaling and unmarshaling functions.
+
+Description
+===========
+
+The function registers or modifies a log field for access log. This is useful if you want to log something that |TS| does not expose,
+log plugin state, or redefine existing log fields.
+
+The `name` is a human friendly name, and only used for debugging. The `symbol` is the keyword you'd want to use on logging.yaml for
+the log field. It needs to be unique unless you are replacing an existing field by passing `true` to the optional argument
+`replace`, otherwise the API call fails.
+
+The `type` is the data type of a log field. You can log any data as a string value, but please note that aggregating functions such
+as AVG and SUM are only available for integer log fields.
+
+In many cases, you don't need to write code for marshaling and unmarshaling from scratch. The predefined functions are provided for
+your convenience, and you only needs to pass a value that you want to log,
+
+Example:
+
+    .. code-block:: cpp
+
+       TSLogFieldRegister("Example", "exmpl", TS_LOG_TYPE_INT,
+       [](TSHttpTxn txnp, char *buf) -> int {
+         return TSLogIntMarshal(buf, 123);
+       },
+       TSLogIntUnmarshal);
+
+Return Values
+=============
+
+:func:`TSLogFieldRegister` returns :enumerator:`TS_SUCCESS` if it successfully registeres a new field, or :enumerator:`TS_ERROR` if it
+fails due to symbol conflict. If :arg:`replace` is set to `true`, the function resolve the conflict by replacing the existing
+field definition with a new one, and returns :enumerator:`TS_SUCCESS`.

doc/developer-guide/api/types/TSEvent.en.rst

@@ -198,6 +198,10 @@ Enumeration Members
 
    The |TS| process has is shutting down.
 
+.. enumerator:: TS_EVENT_LIFECYCLE_LOG_INITIALIZED
+
+   The logging system is initialized.
+
 .. enumerator:: TS_EVENT_INTERNAL_60200
 
 .. enumerator:: TS_EVENT_INTERNAL_60201

example/plugins/c-api/CMakeLists.txt

@@ -65,3 +65,4 @@ add_atsplugin(statistic ./statistic/statistic.cc)
 add_atsplugin(protocol_stack ./protocol_stack/protocol_stack.cc)
 add_atsplugin(client_context_dump ./client_context_dump/client_context_dump.cc)
 target_link_libraries(client_context_dump PRIVATE OpenSSL::SSL libswoc::libswoc)
+add_atsplugin(custom_logfield ./custom_logfield/custom_logfield.cc)

example/plugins/c-api/custom_logfield/custom_logfield.cc

@@ -0,0 +1,230 @@
+/** @file
+
+  This plugin demonstrates custom log field registration and usage.
+  It populates custom log fields from per-transaction user arguments.
+
+  @section license License
+
+  Licensed to the Apache Software Foundation (ASF) under one
+  or more contributor license agreements.  See the NOTICE file
+  distributed with this work for additional information
+  regarding copyright ownership.  The ASF licenses this file
+  to you under the Apache License, Version 2.0 (the
+  "License"); you may not use this file except in compliance
+  with the License.  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+ */
+
+#include <inttypes.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+
+#include <ts/ts.h>
+#include <ts/remap.h>
+
+DbgCtl dbg_ctl{"custom_logfield"};
+
+char PLUGIN_NAME[]    = "custom_logfield";
+char VENDOR_NAME[]    = "Apache Software Foundation";
+char SUPPORT_EMAIL[]  = "dev@trafficserver.apache.org";
+char USER_ARG_CSTM[]  = "cstm_field";
+char USER_ARG_CSTMI[] = "cstmi_field";
+char USER_ARG_CSSN[]  = "cssn_field";
+
+int
+write_text_from_user_arg(TSHttpTxn txnp, char *buf, const char *user_arg_name)
+{
+  int len = 0;
+  int index;
+
+  if (TSUserArgIndexNameLookup(TS_USER_ARGS_TXN, user_arg_name, &index, nullptr) == TS_SUCCESS) {
+    Dbg(dbg_ctl, "User Arg Index: %d", index);
+    if (char *value = static_cast<char *>(TSUserArgGet(txnp, index)); value) {
+      Dbg(dbg_ctl, "Value: %s", value);
+      len = strlen(value);
+      if (buf) {
+        TSstrlcpy(buf, value, len + 1);
+      }
+    }
+  }
+  return len + 1;
+}
+
+int
+marshal_function_cstm(TSHttpTxn txnp, char *buf)
+{
+  if (buf) {
+    Dbg(dbg_ctl, "Marshaling a custom field cstm");
+  } else {
+    Dbg(dbg_ctl, "Marshaling a custom field cstm for size calculation");
+  }
+  return write_text_from_user_arg(txnp, buf, USER_ARG_CSTM);
+}
+
+int
+marshal_function_cssn(TSHttpTxn txnp, char *buf)
+{
+  if (buf) {
+    Dbg(dbg_ctl, "Marshaling a built-in field cssn");
+  } else {
+    Dbg(dbg_ctl, "Marshaling a built-in field cssn for size calculation");
+  }
+  return write_text_from_user_arg(txnp, buf, USER_ARG_CSSN);
+}
+
+int
+marshal_function_cstmi(TSHttpTxn txnp, char *buf)
+{
+  // This implementation is just to demonstrate marshaling an integer value.
+  // Predefined marshal function, TSLogIntMarshal, works for simple integer values
+
+  int index;
+
+  if (buf) {
+    Dbg(dbg_ctl, "Marshaling a custom field cstmi");
+  } else {
+    Dbg(dbg_ctl, "Marshaling a custom field cstmi for size calculation");
+  }
+
+  if (buf) {
+    if (TSUserArgIndexNameLookup(TS_USER_ARGS_TXN, USER_ARG_CSTMI, &index, nullptr) == TS_SUCCESS) {
+      Dbg(dbg_ctl, "User Arg Index: %d", index);
+      if (int64_t value = reinterpret_cast<int64_t>(TSUserArgGet(txnp, index)); value) {
+        Dbg(dbg_ctl, "Value: %" PRId64, value);
+        *(reinterpret_cast<int64_t *>(buf)) = value;
+      }
+    }
+  }
+  return sizeof(int64_t);
+}
+
+std::tuple<int, int>
+unmarshal_function_string(char **buf, char *dest, int len)
+{
+  Dbg(dbg_ctl, "Unmarshaling a string field");
+
+  // This implementation is just to demonstrate unmarshaling a string value.
+  // Predefined unmarshal function, TSLogStringUnmarshal, works for simple string values
+
+  int l = strlen(*buf);
+  Dbg(dbg_ctl, "Dest buf size: %d", len);
+  Dbg(dbg_ctl, "Unmarshaled value length: %d", l);
+  if (l < len) {
+    memcpy(dest, *buf, l);
+    Dbg(dbg_ctl, "Unmarshaled value: %.*s", l, dest);
+    return {
+      l, // The length of data read from buf
+      l  // The length of data written to dest
+    };
+  } else {
+    return {-1, -1};
+  }
+}
+
+int
+lifecycle_event_handler(TSCont /* contp ATS_UNUSED */, TSEvent event, void * /* edata ATS_UNUSED */)
+{
+  TSAssert(event == TS_EVENT_LIFECYCLE_LOG_INITIALIZED);
+
+  // This registers a custom log field "cstm".
+  Dbg(dbg_ctl, "Registering cstm log field");
+  TSLogFieldRegister("custom log field", "cstm", TS_LOG_TYPE_STRING, marshal_function_cstm, unmarshal_function_string);
+
+  // This replaces marshaling and unmarshaling functions for a built-in log field "cssn".
+  Dbg(dbg_ctl, "Overriding cssn log field");
+  TSLogFieldRegister("modified cssn", "cssn", TS_LOG_TYPE_STRING, marshal_function_cssn, TSLogStringUnmarshal, true);
+
+  // This registers a custom log field "cstmi"
+  Dbg(dbg_ctl, "Registering cstmi log field");
+  TSLogFieldRegister("custom integer log field", "cstmi", TS_LOG_TYPE_INT, marshal_function_cstmi, TSLogIntUnmarshal);
+
+  // This replaces marshaling and unmarshaling functions for a built-in log field "chi".
+  Dbg(dbg_ctl, "Overriding chi log field");
+  TSLogFieldRegister(
+    "modified cssn", "chi", TS_LOG_TYPE_ADDR,
+    [](TSHttpTxn /* txnp */, char *buf) -> int {
+      sockaddr_in addr;
+      addr.sin_family      = AF_INET;
+      addr.sin_port        = htons(80);
+      addr.sin_addr.s_addr = inet_addr("192.168.0.1");
+      return TSLogAddrMarshal(buf, reinterpret_cast<sockaddr *>(&addr));
+    },
+    TSLogAddrUnmarshal, true);
+
+  return TS_SUCCESS;
+}
+
+void
+TSPluginInit(int /* argc ATS_UNUSED */, const char ** /* argv ATS_UNUSED */)
+{
+  Dbg(dbg_ctl, "Initializing plugin");
+
+  TSPluginRegistrationInfo info = {PLUGIN_NAME, VENDOR_NAME, SUPPORT_EMAIL};
+  if (TSPluginRegister(&info) != TS_SUCCESS) {
+    TSError("[%s](%s) Plugin registration failed. \n", PLUGIN_NAME, __FUNCTION__);
+  }
+
+  TSCont cont = TSContCreate(lifecycle_event_handler, nullptr);
+  TSLifecycleHookAdd(TS_LIFECYCLE_LOG_INITIALIZED_HOOK, cont);
+
+  int argIndex;
+  TSUserArgIndexReserve(TS_USER_ARGS_TXN, USER_ARG_CSTM, "This is for cstm log field", &argIndex);
+  Dbg(dbg_ctl, "User Arg Index: %d", argIndex);
+  TSUserArgIndexReserve(TS_USER_ARGS_TXN, USER_ARG_CSSN, "This is for cssn log field", &argIndex);
+  Dbg(dbg_ctl, "User Arg Index: %d", argIndex);
+  TSUserArgIndexReserve(TS_USER_ARGS_TXN, USER_ARG_CSTMI, "This is for cstmi log field", &argIndex);
+  Dbg(dbg_ctl, "User Arg Index: %d", argIndex);
+}
+
+TSReturnCode
+TSRemapInit(TSRemapInterface *, char *, int)
+{
+  return TS_SUCCESS;
+}
+
+TSReturnCode
+TSRemapNewInstance(int, char **, void **, char *, int)
+{
+  return TS_SUCCESS;
+}
+
+void
+TSRemapDeleteInstance(void *)
+{
+}
+
+TSRemapStatus
+TSRemapDoRemap(void *, TSHttpTxn txn, TSRemapRequestInfo *)
+{
+  Dbg(dbg_ctl, "Remapping");
+
+  int index;
+
+  // Store a string value for cstm field
+  if (TSUserArgIndexNameLookup(TS_USER_ARGS_TXN, USER_ARG_CSTM, &index, nullptr) == TS_SUCCESS) {
+    Dbg(dbg_ctl, "User Arg Index: %d", index);
+    TSUserArgSet(txn, index, const_cast<char *>("abc"));
+  }
+
+  // Store a string value for cssn field
+  if (TSUserArgIndexNameLookup(TS_USER_ARGS_TXN, USER_ARG_CSSN, &index, nullptr) == TS_SUCCESS) {
+    Dbg(dbg_ctl, "User Arg Index: %d", index);
+    TSUserArgSet(txn, index, const_cast<char *>("xyz"));
+  }
+
+  // Store an integer value for cstmi field
+  if (TSUserArgIndexNameLookup(TS_USER_ARGS_TXN, USER_ARG_CSTMI, &index, nullptr) == TS_SUCCESS) {
+    Dbg(dbg_ctl, "User Arg Index: %d", index);
+    TSUserArgSet(txn, index, reinterpret_cast<void *>(43));
+  }
+
+  return TSREMAP_NO_REMAP;
+}

include/proxy/logging/Log.h

@@ -155,6 +155,7 @@ class Log
   // main interface
   static void init(int configFlags = 0);
   static void init_fields();
+  static void load_config();
 
   static bool
   transaction_logging_enabled()

include/proxy/logging/LogAccess.h

@@ -307,6 +307,10 @@ class LogAccess
   int marshal_milestones_csv(char *buf);
 
   void set_http_header_field(LogField::Container container, char *field, char *buf, int len);
+
+  // Plugin
+  int marshal_custom_field(char *buf, LogField::CustomMarshalFunc plugin_marshal_func);
+
   //
   // unmarshalling routines
   //