ext_authz: document the limitations of filter_enabled_metadata (envoyproxy#42462)

## Description

This PR clearly documents the limitations of using
`filter_enabled_metadata` in the ExtAuthZ filter and that it doesn't
work if the filter itself is disabled in the main filter chain or using
a per-route override. We have also provided examples using
`ExtensionWithMatcher` which should be the recommended way to invoke
filters conditionally.

The [previous attempt](envoyproxy#41937)
which broke some expectations is reverted.

Fix envoyproxy#41501

---

**Commit Message:** ext_authz: document the limitations of
filter_enabled_metadata
**Additional Description:** Add documentation for describing the
limitations around using `filter_enabled_metadata` and provide
workaround examples using `ExtensionWithMatcher`.
**Risk Level:** N/A
**Testing:** CI
**Docs Changes:** Added
**Release Notes:** N/A

---------

Signed-off-by: Rohit Agrawal <rohit.agrawal@databricks.com>

Author: agrawroh

api/envoy/extensions/filters/http/ext_authz/v3/ext_authz.proto

@@ -165,6 +165,24 @@ message ExtAuthz {
 
   // Specifies if the filter is enabled with metadata matcher.
   // If this field is not specified, the filter will be enabled for all requests.
+  //
+  // .. note::
+  //
+  //   This field is only evaluated if the filter is instantiated. If the filter is marked with
+  //   ``disabled: true`` in the :ref:`HttpFilter
+  //   <envoy_v3_api_msg_extensions.filters.network.http_connection_manager.v3.HttpFilter>`
+  //   configuration or in per-route configuration via :ref:`ExtAuthzPerRoute
+  //   <envoy_v3_api_msg_extensions.filters.http.ext_authz.v3.ExtAuthzPerRoute>`,
+  //   the filter will not be instantiated and this field will have no effect.
+  //
+  // .. tip::
+  //
+  //   For dynamic filter activation based on metadata (such as metadata set by a preceding
+  //   filter), consider using :ref:`ExtensionWithMatcher
+  //   <envoy_v3_api_msg_extensions.common.matching.v3.ExtensionWithMatcher>` instead. This
+  //   provides a more flexible matching framework that can evaluate conditions before filter
+  //   instantiation. See the :ref:`ext_authz filter documentation
+  //   <config_http_filters_ext_authz>` for examples.
   type.matcher.v3.MetadataMatcher filter_enabled_metadata = 14;
 
   // Specifies whether to deny the requests when the filter is disabled.

docs/root/configuration/http/http_filters/_include/ext-authz-extension-with-matcher.yaml

@@ -0,0 +1,123 @@
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address:
+        protocol: TCP
+        address: 0.0.0.0
+        port_value: 10000
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress_http
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_service
+              domains: ["*"]
+              routes:
+              - match:
+                  prefix: "/"
+                route:
+                  host_rewrite_literal: upstream.com
+                  cluster: upstream_com
+          http_filters:
+          # Lua filter sets dynamic metadata that controls whether ext_authz runs.
+          - name: envoy.filters.http.lua
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.lua.v3.Lua
+              default_source_code:
+                inline_string: |
+                  function envoy_on_request(request_handle)
+                    -- Set metadata to conditionally enable ext_authz.
+                    -- For example, enable auth for requests to /secure paths.
+                    local path = request_handle:headers():get(":path")
+                    if string.match(path, "^/secure") then
+                      request_handle:streamInfo():dynamicMetadata():set("envoy.filters.http.ext_authz", "require_auth", "true")
+                    else
+                      request_handle:streamInfo():dynamicMetadata():set("envoy.filters.http.ext_authz", "require_auth", "false")
+                    end
+                  end
+          # ExtensionWithMatcher wraps ext_authz and conditionally invokes it based on dynamic metadata.
+          - name: ext-authz-with-matcher
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.common.matching.v3.ExtensionWithMatcher
+              extension_config:
+                name: envoy.filters.http.ext_authz
+                typed_config:
+                  "@type": type.googleapis.com/envoy.extensions.filters.http.ext_authz.v3.ExtAuthz
+                  grpc_service:
+                    envoy_grpc:
+                      cluster_name: ext-authz
+                    timeout: 0.5s
+                  include_peer_certificate: true
+              # The xds matcher evaluates dynamic metadata to decide whether to invoke ext_authz.
+              # We use matcher_list with custom_match because DynamicMetadataInput returns a custom
+              # MetadataMatchData type that requires a custom matcher and not exact_match_map.
+              xds_matcher:
+                matcher_list:
+                  matchers:
+                  - predicate:
+                      single_predicate:
+                        input:
+                          name: envoy.matching.inputs.dynamic_metadata
+                          typed_config:
+                            "@type": type.googleapis.com/envoy.extensions.matching.common_inputs.network.v3.DynamicMetadataInput
+                            filter: envoy.filters.http.ext_authz
+                            path:
+                            - key: require_auth
+                        custom_match:
+                          name: envoy.matching.matchers.metadata_matcher
+                          typed_config:
+                            "@type": type.googleapis.com/envoy.extensions.matching.input_matchers.metadata.v3.Metadata
+                            value:
+                              string_match:
+                                exact: "false"
+                    # When require_auth is "false", skip ext_authz.
+                    on_match:
+                      action:
+                        name: skip
+                        typed_config:
+                          "@type": type.googleapis.com/envoy.extensions.filters.common.matcher.action.v3.SkipFilter
+          - name: envoy.filters.http.router
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+
+  clusters:
+  - name: ext-authz
+    type: STATIC
+    typed_extension_protocol_options:
+      envoy.extensions.upstreams.http.v3.HttpProtocolOptions:
+        "@type": type.googleapis.com/envoy.extensions.upstreams.http.v3.HttpProtocolOptions
+        explicit_http_config:
+          http2_protocol_options: {}
+    load_assignment:
+      cluster_name: ext-authz
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: 127.0.0.1
+                port_value: 10003
+  - name: upstream_com
+    type: LOGICAL_DNS
+    # Comment out the following line to test on v6 networks.
+    dns_lookup_family: V4_ONLY
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: service_upstream_com
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: upstream.com
+                port_value: 443
+    transport_socket:
+      name: envoy.transport_sockets.tls
+      typed_config:
+        "@type": type.googleapis.com/envoy.extensions.transport_sockets.tls.v3.UpstreamTlsContext
+        sni: upstream.com

docs/root/configuration/http/http_filters/ext_authz_filter.rst

@@ -134,6 +134,48 @@ Per-Route Configuration
 A sample virtual host and route filter configuration.
 In this example, we add additional context on the virtual host and disable the filter for ``/static``-prefixed routes.
 
+Conditional Filter Activation with Dynamic Metadata
+----------------------------------------------------
+
+When you need to conditionally invoke the ext_authz filter based on dynamic metadata set by a preceding
+filter (such as a Lua filter), it is recommended to use :ref:`ExtensionWithMatcher
+<envoy_v3_api_msg_extensions.common.matching.v3.ExtensionWithMatcher>` rather than the
+:ref:`filter_enabled_metadata <envoy_v3_api_field_extensions.filters.http.ext_authz.v3.ExtAuthz.filter_enabled_metadata>`
+field.
+
+The key differences are:
+
+* **ExtensionWithMatcher**: Evaluates matching conditions before filter instantiation. The filter is only
+  created and invoked when the matcher determines it should run. This is the recommended approach for
+  metadata-based conditional invocation.
+
+* **filter_enabled_metadata**: Only evaluated after the filter is instantiated. If the filter is marked with
+  ``disabled: true`` in the HttpFilter configuration, it will not be instantiated and ``filter_enabled_metadata``
+  will have no effect.
+
+The following example demonstrates using ExtensionWithMatcher to conditionally invoke ext_authz based on
+dynamic metadata set by a Lua filter:
+
+.. literalinclude:: _include/ext-authz-extension-with-matcher.yaml
+    :language: yaml
+    :lines: 26-83
+    :lineno-start: 26
+    :linenos:
+    :caption: :download:`ext-authz-extension-with-matcher.yaml <_include/ext-authz-extension-with-matcher.yaml>`
+
+In this configuration:
+
+* The Lua filter examines the request path and sets dynamic metadata (``envoy.filters.http.ext_authz.require_auth``)
+  to indicate whether authorization is required.
+* The ExtensionWithMatcher uses :ref:`DynamicMetadataInput
+  <envoy_v3_api_msg_extensions.matching.common_inputs.network.v3.DynamicMetadataInput>` to read this metadata.
+* When ``require_auth`` is ``true``, the ext_authz filter is invoked.
+* When ``require_auth`` is ``false``, the :ref:`SkipFilter
+  <envoy_v3_api_msg_extensions.filters.common.matcher.action.v3.SkipFilter>` action causes the filter to be skipped.
+
+This pattern provides clean separation between the decision logic (in the Lua filter) and the authorization
+enforcement (in ext_authz), while ensuring the ext_authz filter is only instantiated and invoked when needed.
+
 Statistics
 ----------
 .. _config_http_filters_ext_authz_stats:

test/extensions/filters/http/ext_authz/BUILD

@@ -81,7 +81,11 @@ envoy_extension_cc_test(
         ":logging_test_filter_lib",
         "//source/extensions/clusters/dns:dns_cluster_lib",
         "//source/extensions/filters/http/ext_authz:config",
+        "//source/extensions/filters/http/lua:config",
+        "//source/extensions/filters/http/match_delegate:config",
         "//source/extensions/listener_managers/validation_listener_manager:validation_listener_manager_lib",
+        "//source/extensions/matching/http/metadata_input:metadata_input_lib",
+        "//source/extensions/matching/input_matchers/metadata:config",
         "//source/extensions/network/dns_resolver/getaddrinfo:config",
         "//source/server/config_validation:server_lib",
         "//test/integration:http_integration_lib",
@@ -91,7 +95,12 @@ envoy_extension_cc_test(
         "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/core/v3:pkg_cc_proto",
         "@envoy_api//envoy/config/listener/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/common/matching/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/filters/common/matcher/action/v3:pkg_cc_proto",
         "@envoy_api//envoy/extensions/filters/http/ext_authz/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/matching/common_inputs/network/v3:pkg_cc_proto",
+        "@envoy_api//envoy/extensions/matching/input_matchers/metadata/v3:pkg_cc_proto",
         "@envoy_api//envoy/service/auth/v3:pkg_cc_proto",
     ],
 )