Add max inclusion depth support for esi plugin 10.0.x (#12299)

Co-authored-by: Shu Kit Chan <kichan@c40b7d47.home-kichan.bf2.ows.oath.cloud>

Author: shukitchan

doc/admin-guide/plugins/esi.en.rst

@@ -76,7 +76,7 @@ Enabling ESI
 
     esi.so
 
-2. There are four optional arguments that can be passed to the above ``esi.so`` entry:
+2. There are optional arguments that can be passed to the above ``esi.so`` entry:
 
 - ``--private-response`` will add private cache control and expires headers to the processed ESI document.
 - ``--packed-node-support`` will enable the support for using the packed node feature, which will improve the
@@ -86,10 +86,12 @@ Enabling ESI
 - ``--first-byte-flush`` will enable the first byte flush feature, which will flush content to users as soon as the entire
   ESI document is received and parsed without all ESI includes fetched. The flushing will stop at the ESI include markup
   till that include is fetched.
-- ``--max-doc-size <number-of-bytes>`` gives the maximum size of the document, in bytes.  The number of bytes must be
-  be must an unsigned decimal integer, and can be followed (with no white space) by a K, to indicate the given number is
-  multiplied by 1024, or by M, to indicate the given number is multiplied by 1024 * 1024.  Example values: 500,
-  5K, 2M.  If this option is omitted, the maximum document size defaults to 1M.
+- ``--max-doc-size <number-of-bytes>`` (or ``--max-doc-size=<number-of-bytes>``) gives the maximum size of the document,
+  in bytes.  The number of bytes must be an unsigned decimal integer, and can be followed (with no white space) by
+  a K, to indicate the given number is multiplied by 1024, or by M, to indicate the given number is multiplied by
+  1024 * 1024.  Example values: 500, 5K, 2M.  If this option is omitted, the maximum document size defaults to 1M.
+- ``--max-inclusion-depth <max-depth>`` controls the maximum depth of recursive ESI inclusion allowed (between 0 and 9).
+  Default is 3.
 
 3. ``HTTP_COOKIE`` variable support is turned off by default. It can be turned on with ``-f <handler_config>`` or
    ``-handler <handler_config>``. For example:

plugins/esi/esi.cc

@@ -58,6 +58,7 @@ struct OptionInfo {
   bool     disable_gzip_output{false};
   bool     first_byte_flush{false};
   unsigned max_doc_size{1024 * 1024};
+  unsigned max_inclusion_depth{3};
 };
 
 static HandlerManager        *gHandlerManager = nullptr;
@@ -69,6 +70,9 @@ static Utils::HeaderValueList gAllowlistCookies;
 #define MIME_FIELD_XESI     "X-Esi"
 #define MIME_FIELD_XESI_LEN 5
 
+#define MIME_FIELD_XESIDEPTH     "X-Esi-Depth"
+#define MIME_FIELD_XESIDEPTH_LEN 11
+
 #define HTTP_VALUE_PRIVATE_EXPIRES "-1"
 #define HTTP_VALUE_PRIVATE_CC      "max-age=0, private"
 
@@ -298,46 +302,66 @@ ContData::getClientState()
       }
       TSHandleMLocRelease(bufp, req_hdr_loc, url_loc);
     }
-    TSMLoc field_loc = TSMimeHdrFieldGet(req_bufp, req_hdr_loc, 0);
+
+    TSMLoc field_loc   = TSMimeHdrFieldGet(req_bufp, req_hdr_loc, 0);
+    bool   depth_field = false;
     while (field_loc) {
       TSMLoc      next_field_loc;
       const char *name;
       int         name_len;
 
       name = TSMimeHdrFieldNameGet(req_bufp, req_hdr_loc, field_loc, &name_len);
       if (name) {
-        int n_values;
-        n_values = TSMimeHdrFieldValuesCount(req_bufp, req_hdr_loc, field_loc);
-        if (n_values && (n_values != TS_ERROR)) {
-          const char *value     = nullptr;
-          int         value_len = 0;
-          if (n_values == 1) {
-            value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, 0, &value_len);
-
-            if (nullptr != value && value_len) {
-              if (Utils::areEqual(name, name_len, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING) &&
-                  Utils::areEqual(value, value_len, TS_HTTP_VALUE_GZIP, TS_HTTP_LEN_GZIP)) {
-                gzip_output = true;
-              }
-            }
+        if (Utils::areEqual(name, name_len, MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN)) {
+          unsigned d = TSMimeHdrFieldValueUintGet(req_bufp, req_hdr_loc, field_loc, -1);
+          d          = (d + 1) % 10;
+          char      dstr[2];
+          int const len = snprintf(dstr, sizeof(dstr), "%u", d);
+
+          HttpHeader header;
+          if (len != 1) {
+            header = HttpHeader(MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN, "1", 1);
           } else {
-            for (int i = 0; i < n_values; ++i) {
-              value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, i, &value_len);
+            header = HttpHeader(MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN, dstr, 1);
+          }
+          data_fetcher->useHeader(header);
+          esi_vars->populate(header);
+          depth_field = true;
+
+        } else {
+          int n_values;
+          n_values = TSMimeHdrFieldValuesCount(req_bufp, req_hdr_loc, field_loc);
+          if (n_values && (n_values != TS_ERROR)) {
+            const char *value     = nullptr;
+            int         value_len = 0;
+            if (n_values == 1) {
+              value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, 0, &value_len);
+
               if (nullptr != value && value_len) {
                 if (Utils::areEqual(name, name_len, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING) &&
                     Utils::areEqual(value, value_len, TS_HTTP_VALUE_GZIP, TS_HTTP_LEN_GZIP)) {
                   gzip_output = true;
                 }
               }
-            }
+            } else {
+              for (int i = 0; i < n_values; ++i) {
+                value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, i, &value_len);
+                if (nullptr != value && value_len) {
+                  if (Utils::areEqual(name, name_len, TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING) &&
+                      Utils::areEqual(value, value_len, TS_HTTP_VALUE_GZIP, TS_HTTP_LEN_GZIP)) {
+                    gzip_output = true;
+                  }
+                }
+              }
 
-            value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, -1, &value_len);
-          }
+              value = TSMimeHdrFieldValueStringGet(req_bufp, req_hdr_loc, field_loc, -1, &value_len);
+            }
 
-          if (value != nullptr) {
-            HttpHeader header(name, name_len, value, value_len);
-            data_fetcher->useHeader(header);
-            esi_vars->populate(header);
+            if (value != nullptr) {
+              HttpHeader header(name, name_len, value, value_len);
+              data_fetcher->useHeader(header);
+              esi_vars->populate(header);
+            }
           }
         }
       }
@@ -346,6 +370,12 @@ ContData::getClientState()
       TSHandleMLocRelease(req_bufp, req_hdr_loc, field_loc);
       field_loc = next_field_loc;
     }
+
+    if (depth_field == false) {
+      HttpHeader header(MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN, "1", 1);
+      data_fetcher->useHeader(header);
+      esi_vars->populate(header);
+    }
   }
 
   if (gzip_output) {
@@ -1229,7 +1259,7 @@ maskOsCacheHeaders(TSHttpTxn txnp)
 }
 
 static bool
-isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool *intercept_header, bool *head_only)
+isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, const OptionInfo *pOptionInfo, bool *intercept_header, bool *head_only)
 {
   //  We are only interested in transforming "200 OK" responses with a
   //  Content-Type: text/ header and with X-Esi header
@@ -1244,6 +1274,21 @@ isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool *intercept_header, bo
     return false;
   }
 
+  TSMLoc   loc;
+  unsigned d;
+
+  d   = 0;
+  loc = TSMimeHdrFieldFind(bufp, hdr_loc, MIME_FIELD_XESIDEPTH, MIME_FIELD_XESIDEPTH_LEN);
+  if (loc != TS_NULL_MLOC) {
+    d = TSMimeHdrFieldValueUintGet(bufp, hdr_loc, loc, -1);
+  }
+  TSHandleMLocRelease(bufp, hdr_loc, loc);
+  if (d >= pOptionInfo->max_inclusion_depth) {
+    TSError("[esi][%s] The current esi inclusion depth (%u) is larger than or equal to the max (%u)", __FUNCTION__, d,
+            pOptionInfo->max_inclusion_depth);
+    return false;
+  }
+
   int         method_len;
   const char *method;
   method = TSHttpHdrMethodGet(bufp, hdr_loc, &method_len);
@@ -1318,7 +1363,7 @@ isTxnTransformable(TSHttpTxn txnp, bool is_cache_txn, bool *intercept_header, bo
 }
 
 static bool
-isCacheObjTransformable(TSHttpTxn txnp, bool *intercept_header, bool *head_only)
+isCacheObjTransformable(TSHttpTxn txnp, const OptionInfo *pOptionInfo, bool *intercept_header, bool *head_only)
 {
   int obj_status;
   if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) {
@@ -1327,7 +1372,7 @@ isCacheObjTransformable(TSHttpTxn txnp, bool *intercept_header, bool *head_only)
   }
   if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) {
     Dbg(dbg_ctl_local, "[%s] doc found in cache, will add transformation", __FUNCTION__);
-    return isTxnTransformable(txnp, true, intercept_header, head_only);
+    return isTxnTransformable(txnp, true, pOptionInfo, intercept_header, head_only);
   }
   Dbg(dbg_ctl_local, "[%s] cache object's status is %d; not transformable", __FUNCTION__, obj_status);
   return false;
@@ -1499,7 +1544,7 @@ globalHookHandler(TSCont contp, TSEvent event, void *edata)
       if (event == TS_EVENT_HTTP_READ_RESPONSE_HDR) {
         bool mask_cache_headers = false;
         Dbg(dbg_ctl_local, "[%s] handling read response header event", __FUNCTION__);
-        if (isTxnTransformable(txnp, false, &intercept_header, &head_only)) {
+        if (isTxnTransformable(txnp, false, pOptionInfo, &intercept_header, &head_only)) {
           addTransform(txnp, true, intercept_header, head_only, pOptionInfo);
           Stats::increment(Stats::N_OS_DOCS);
           mask_cache_headers = true;
@@ -1512,7 +1557,7 @@ globalHookHandler(TSCont contp, TSEvent event, void *edata)
         }
       } else {
         Dbg(dbg_ctl_local, "[%s] handling cache lookup complete event", __FUNCTION__);
-        if (isCacheObjTransformable(txnp, &intercept_header, &head_only)) {
+        if (isCacheObjTransformable(txnp, pOptionInfo, &intercept_header, &head_only)) {
           // we make the assumption above that a transformable cache
           // object would already have a transformation. We should revisit
           // that assumption in case we change the statement below
@@ -1575,11 +1620,12 @@ esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo)
       {const_cast<char *>("first-byte-flush"),    no_argument,       nullptr, 'b'},
       {const_cast<char *>("handler-filename"),    required_argument, nullptr, 'f'},
       {const_cast<char *>("max-doc-size"),        required_argument, nullptr, 'd'},
+      {const_cast<char *>("max-inclusion-depth"), required_argument, nullptr, 'i'},
       {nullptr,                                   0,                 nullptr, 0  },
     };
 
     int longindex = 0;
-    while ((c = getopt_long(argc, const_cast<char *const *>(argv), "npzbf:d:", longopts, &longindex)) != -1) {
+    while ((c = getopt_long(argc, const_cast<char *const *>(argv), "npzbf:d:i:", longopts, &longindex)) != -1) {
       switch (c) {
       case 'n':
         pOptionInfo->packed_node_support = true;
@@ -1621,6 +1667,18 @@ esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo)
         pOptionInfo->max_doc_size = max * coeff;
         break;
       }
+      case 'i': {
+        unsigned max;
+        auto     num = std::sscanf(optarg, "%u", &max);
+        if (num != 1) {
+          TSEmergency("[esi][%s] value for maximum inclusion depth (%s) is not unsigned integer", __FUNCTION__, optarg);
+        }
+        if (max > 9) {
+          TSEmergency("[esi][%s] maximum inclusion depth (%s) large than 9", __FUNCTION__, optarg);
+        }
+        pOptionInfo->max_inclusion_depth = max;
+        break;
+      }
       default:
         TSEmergency("[esi][%s] bad option", __FUNCTION__);
         return -1;
@@ -1630,9 +1688,10 @@ esiPluginInit(int argc, const char *argv[], OptionInfo *pOptionInfo)
 
   Dbg(dbg_ctl_local,
       "[%s] Plugin started, "
-      "packed-node-support: %d, private-response: %d, disable-gzip-output: %d, first-byte-flush: %d, max-doc-size %u ",
+      "packed-node-support: %d, private-response: %d, disable-gzip-output: %d, first-byte-flush: %d, max-doc-size %u, "
+      "max-inclusion-depth %u ",
       __FUNCTION__, pOptionInfo->packed_node_support, pOptionInfo->private_response, pOptionInfo->disable_gzip_output,
-      pOptionInfo->first_byte_flush, pOptionInfo->max_doc_size);
+      pOptionInfo->first_byte_flush, pOptionInfo->max_doc_size, pOptionInfo->max_inclusion_depth);
 
   return 0;
 }

tests/gold_tests/pluginTest/esi/esi_nested_include.test.py

@@ -0,0 +1,137 @@
+'''
+Test nested include for the ESI plugin.
+'''
+#  Licensed to the Apache Software Foundation (ASF) under one
+#  or more contributor license agreements.  See the NOTICE file
+#  distributed with this work for additional information
+#  regarding copyright ownership.  The ASF licenses this file
+#  to you under the Apache License, Version 2.0 (the
+#  "License"); you may not use this file except in compliance
+#  with the License.  You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+import os
+
+Test.Summary = '''
+Test nested include for the ESI plugin.
+'''
+
+Test.SkipUnless(Condition.PluginExists('esi.so'),)
+
+
+class EsiTest():
+    """
+    A class that encapsulates the configuration and execution of a set of ESI
+    test cases.
+    """
+    """ static: The same server Process is used across all tests. """
+    _server = None
+    """ static: A counter to keep the ATS process names unique across tests. """
+    _ts_counter = 0
+    """ static: A counter to keep any output file names unique across tests. """
+    _output_counter = 0
+    """ The ATS process for this set of test cases. """
+    _ts = None
+
+    def __init__(self, plugin_config):
+        """
+        Args:
+            plugin_config (str): The config line to place in plugin.config for
+                the ATS process.
+        """
+        if EsiTest._server is None:
+            EsiTest._server = EsiTest._create_server()
+
+        self._ts = EsiTest._create_ats(self, plugin_config)
+
+    @staticmethod
+    def _create_server():
+        """
+        Create and start a server process.
+        """
+        # Configure our server.
+        server = Test.MakeOriginServer("server", lookup_key="{%uuid}")
+
+        # Generate the set of ESI responses.
+        request_header = {
+            "headers": "GET /esi-nested-include.php HTTP/1.1\r\n" + "Host: www.example.com\r\n" + "Content-Length: 0\r\n\r\n",
+            "timestamp": "1469733493.993",
+            "body": ""
+        }
+        esi_body = r'''<p>
+<esi:include src="http://www.example.com/esi-nested-include.html"/>
+</p>
+'''
+        response_header = {
+            "headers":
+                "HTTP/1.1 200 OK\r\n" + "X-Esi: 1\r\n" + "Cache-Control: private\r\n" + "Content-Type: text/html\r\n" +
+                "Connection: close\r\n" + "Content-Length: {}\r\n".format(len(esi_body)) + "\r\n",
+            "timestamp": "1469733493.993",
+            "body": esi_body
+        }
+        server.addResponse("sessionfile.log", request_header, response_header)
+
+        # Create a run to start the server.
+        tr = Test.AddTestRun("Start the server.")
+        tr.Processes.Default.StartBefore(server)
+        tr.Processes.Default.Command = "echo starting the server"
+        tr.Processes.Default.ReturnCode = 0
+        tr.StillRunningAfter = server
+
+        return server
+
+    @staticmethod
+    def _create_ats(self, plugin_config):
+        """
+        Create and start an ATS process.
+        """
+        EsiTest._ts_counter += 1
+
+        # Configure ATS with a vanilla ESI plugin configuration.
+        ts = Test.MakeATSProcess("ts{}".format(EsiTest._ts_counter))
+        ts.Disk.records_config.update({
+            'proxy.config.diags.debug.enabled': 1,
+            'proxy.config.diags.debug.tags': 'http|plugin_esi',
+        })
+        ts.Disk.remap_config.AddLine(f'map http://www.example.com/ http://127.0.0.1:{EsiTest._server.Variables.Port}')
+        ts.Disk.plugin_config.AddLine(plugin_config)
+
+        ts.Disk.diags_log.Content = Testers.ContainsExpression(
+            r'The current esi inclusion depth \(3\) is larger than or equal to the max \(3\)',
+            'Verify the ESI error concerning the max inclusion depth')
+
+        # Create a run to start the ATS process.
+        tr = Test.AddTestRun("Start the ATS process.")
+        tr.Processes.Default.StartBefore(ts)
+        tr.Processes.Default.Command = "echo starting ATS"
+        tr.Processes.Default.ReturnCode = 0
+        tr.StillRunningAfter = ts
+        return ts
+
+    def run_test(self):
+        # Test 1: Verify basic ESI functionality without processing internal txn.
+        tr = Test.AddTestRun("First request")
+        tr.Processes.Default.Command = \
+            ('curl http://127.0.0.1:{0}/main.php -H"Host: www.example.com" '
+             '-H"Accept: */*" --verbose'.format(
+                 self._ts.Variables.port))
+        tr.Processes.Default.ReturnCode = 0
+        tr.Processes.Default.Streams.stdout = "gold/nested_include_body.gold"
+        tr.StillRunningAfter = self._server
+        tr.StillRunningAfter = self._ts
+
+
+#
+# Configure and run the test cases.
+#
+
+# Run the tests with ESI configured with private response.
+first_test = EsiTest(plugin_config='esi.so')
+first_test.run_test()