feat(ip_tagging): file-based IP tag loading with async reload

api/envoy/extensions/filters/http/ip_tagging/v3/ip_tagging.proto

@@ -3,6 +3,9 @@ syntax = "proto3";
 package envoy.extensions.filters.http.ip_tagging.v3;
 
 import "envoy/config/core/v3/address.proto";
+import "envoy/config/core/v3/base.proto";
+
+import "google/protobuf/duration.proto";
 
 import "udpa/annotations/status.proto";
 import "udpa/annotations/versioning.proto";
@@ -18,7 +21,7 @@ option (udpa.annotations.file_status).package_version_status = ACTIVE;
 // IP tagging :ref:`configuration overview <config_http_filters_ip_tagging>`.
 // [#extension: envoy.filters.http.ip_tagging]
 
-// [#next-free-field: 6]
+// [#next-free-field: 7]
 message IPTagging {
   option (udpa.annotations.versioning).previous_message_type =
       "envoy.config.filter.http.ip_tagging.v2.IPTagging";
@@ -53,6 +56,12 @@ message IPTagging {
     repeated config.core.v3.CidrRange ip_list = 2;
   }
 
+  // Specifies the content of the IP tag file.
+  // Allow the file to be created with no IP tags.
+  message IPTags {
+    repeated IPTag ip_tags = 1;
+  }
+
   // Specify to which header the tags will be written.
   message IpTagHeader {
     // Describes how to apply the tags to the headers.
@@ -85,16 +94,33 @@ message IPTagging {
     HeaderAction action = 2;
   }
 
+  // Common configuration for file based ip tags.
+  message IpTagsFileProvider {
+    // Data source from which to retrieve ip tags.
+    // Only filename based data source is currently supported for IP tags.
+    // When using this data source, if a ``watched_directory`` is provided, the IP tags file will be re-read when a file move is detected.
+    // See :ref:`watched_directory <envoy_v3_api_msg_config.core.v3.DataSource>` for more information about the ``watched_directory`` field.
+    config.core.v3.DataSource ip_tags_datasource = 1;
+
+    // When :ref:`ip_tags <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_datasource>` is configured
+    // the ip tags will be reloaded from file every ``ip_tags_refresh_rate`` milliseconds.
+    // Defaults to 0, in this case no refresh will be attempted.
+    google.protobuf.Duration ip_tags_refresh_rate = 2 [(validate.rules).duration = {gt {}}];
+  }
+
   // The type of request the filter should apply to.
   RequestType request_type = 1 [(validate.rules).enum = {defined_only: true}];
 
-  // [#comment:TODO(ccaraman): Extend functionality to load IP tags from file system.
-  // Tracked by issue https://github.com/envoyproxy/envoy/issues/2695]
   // The set of IP tags for the filter.
-  repeated IPTag ip_tags = 4 [(validate.rules).repeated = {min_items: 1}];
+  // Only one of :ref:`ip_tags <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.ip_tags>`
+  // or :ref:`ip_tags_datasource <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_datasource>`
+  // can be set for the IP Tagging filter.
+  repeated IPTag ip_tags = 4;
 
   // Specify to which header the tags will be written.
   //
   // If left unspecified, the tags will be appended to the ``x-envoy-ip-tags`` header.
   IpTagHeader ip_tag_header = 5;
+
+  IpTagsFileProvider ip_tags_file_provider = 6;
 }

docs/BUILD

@@ -35,6 +35,7 @@ filegroup(
             "root/configuration/listeners/network_filters/_include/generic_proxy_filter.yaml",
             "root/configuration/overview/_include/xds_api/oauth-sds-example.yaml",
             "root/configuration/security/_include/sds-source-example.yaml",
+            "root/configuration/http/http_filters/_include/ip-tagging-filter.yaml",
         ],
     ) + select({
         "@envoy//bazel:windows_x86_64": [],

docs/root/configuration/http/http_filters/_include/ip-tagging-filter.yaml

@@ -0,0 +1,115 @@
+static_resources:
+  listeners:
+  - address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 8000
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress_http
+          http_filters:
+          - name: ip_tagging
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging
+              request_type: both
+              ip_tags:
+              - ip_tag_name: external_request
+                ip_list:
+                - {address_prefix: 1.2.3.4, prefix_len: 32}
+          - name: envoy.filters.http.router
+            typed_config:
+              '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - domains:
+              - '*'
+              name: local_service
+              routes:
+              - match: {prefix: "/"}
+                route: {cluster: default_service}
+  - address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 9000
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress1_http
+          http_filters:
+          - name: ip_tagging
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging
+              request_type: both
+              ip_tags_file_provider:
+                ip_tags_refresh_rate: 5s
+                ip_tags_datasource:
+                  filename: "/geoip/ip-tags.yaml"
+                  watched_directory:
+                    path: "/geoip/"
+          - name: envoy.filters.http.router
+            typed_config:
+              '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - domains:
+              - '*'
+              name: local_service
+              routes:
+              - match: {prefix: "/"}
+                route: {cluster: default_service}
+  - address:
+      socket_address:
+        address: 0.0.0.0
+        port_value: 7000
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          '@type': type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress2_http
+          http_filters:
+          - name: ip_tagging
+            typed_config:
+              "@type": type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging
+              request_type: both
+              ip_tags_file_provider:
+                ip_tags_refresh_rate: 5s
+                ip_tags_datasource:
+                  filename: "/geoip/ip-tags.json"
+                  watched_directory:
+                    path: "/geoip/"
+          - name: envoy.filters.http.router
+            typed_config:
+              '@type': type.googleapis.com/envoy.extensions.filters.http.router.v3.Router
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - domains:
+              - '*'
+              name: local_service
+              routes:
+              - match: {prefix: "/"}
+                route: {cluster: default_service}
+  clusters:
+  - name: default_service
+    load_assignment:
+      cluster_name: default_service
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: 127.0.0.1
+                port_value: 10001
+admin:
+  address:
+    socket_address:
+      address: 0.0.0.0
+      port_value: 9901

docs/root/configuration/http/http_filters/ip_tagging_filter.rst

@@ -22,12 +22,83 @@ described in the paper `IP-address lookup using
 LC-tries <https://www.csc.kth.se/~snilsson/publications/IP-address-lookup-using-LC-tries/text.pdf>`_ by S. Nilsson and
 G. Karlsson.
 
+IP tags can either be provided directly using the :ref:`ip_tags <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.ip_tags>` API field or
+can be loaded from file if :ref:`ip_tags_datasource <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_datasource>` API field is configured.
+For file based IP tags YAML and JSON file formats are supported.
+IP tags will be dynamically reloaded if ``watched_directory`` is configured for :ref:`ip_tags_datasource <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_datasource>`
+and :ref:`ip_tags_refresh_rate <envoy_v3_api_field_extensions.filters.http.ip_tagging.v3.IPTagging.IpTagsFileProvider.ip_tags_refresh_rate>` is set to value greater than zero.
 
 Configuration
 -------------
 * This filter should be configured with the type URL ``type.googleapis.com/envoy.extensions.filters.http.ip_tagging.v3.IPTagging``.
 * :ref:`v3 API reference <envoy_v3_api_msg_extensions.filters.http.ip_tagging.v3.IPTagging>`
 
+An example configuration of the filter with inline ip tags may look like the following:
+
+.. literalinclude:: _include/ip-tagging-filter.yaml
+    :language: yaml
+    :lines: 13-21
+    :lineno-start: 13
+    :linenos:
+    :caption: :download:`ip-tagging-filter.yaml <_include/ip-tagging-filter.yaml>`
+
+Below is an example configuration of the filter with the file based ip tags in yaml format:
+
+.. literalinclude:: _include/ip-tagging-filter.yaml
+    :language: yaml
+    :lines: 44-54
+    :lineno-start: 44
+    :linenos:
+    :caption: :download:`ip-tagging-filter.yaml <_include/ip-tagging-filter.yaml>`
+
+Where the *ip-tags.yaml* file would have the following content:
+
+.. code-block:: yaml
+
+    ip_tags:
+    - ip_tag_name: external_request
+      ip_list:
+      - {address_prefix: 1.2.3.4, prefix_len: 32}
+    - ip_tag_name: internal_request
+      ip_list:
+      - {address_prefix: 1.2.3.5, prefix_len: 32}
+
+And here is an example configuration of the filter with the file based IP tags in JSON format:
+
+.. literalinclude:: _include/ip-tagging-filter.yaml
+    :language: yaml
+    :lines: 77-87
+    :lineno-start: 77
+    :linenos:
+    :caption: :download:`ip-tagging-filter.yaml <_include/ip-tagging-filter.yaml>`
+
+Where the ``ip-tags.json`` file would have the following content:
+
+.. code-block:: json
+
+  {
+  "ip_tags": [
+      {
+      "ip_tag_name": "external_request",
+      "ip_list": [
+          {
+          "address_prefix": "1.2.3.4",
+          "prefix_len": 32
+          }
+      ]
+      },
+      {
+          "ip_tag_name": "internal_request",
+          "ip_list": [
+              {
+              "address_prefix": "1.2.3.5",
+              "prefix_len": 32
+              }
+          ]
+          }
+  ]
+  }
+
 Statistics
 ----------
 
@@ -41,6 +112,8 @@ the owning HTTP connection manager.
         <tag_name>.hit, Counter, Total number of requests that have the ``<tag_name>`` applied to it
         no_hit, Counter, Total number of requests with no applicable IP tags
         total, Counter, Total number of requests the IP Tagging Filter operated on
+        reload_success, Counter, Total number of successful reloads of IP tags file
+        reload_error, Counter, Total number of failed reloads of IP tags file
 
 Runtime
 -------

source/common/common/logger.h

@@ -104,7 +104,8 @@ const static bool should_log = true;
   FUNCTION(websocket)                                                                              \
   FUNCTION(golang)                                                                                 \
   FUNCTION(stats_sinks)                                                                            \
-  FUNCTION(dynamic_modules)
+  FUNCTION(dynamic_modules)                                                                        \
+  FUNCTION(ip_tagging)
 
 // clang-format off
 enum class Id {

source/common/protobuf/utility.h

@@ -272,6 +272,8 @@ class MessageUtil {
   static void loadFromJson(absl::string_view json, Protobuf::Struct& message);
   static void loadFromYaml(const std::string& yaml, Protobuf::Message& message,
                            ProtobufMessage::ValidationVisitor& validation_visitor);
+  static absl::Status loadFromYamlNoThrow(const std::string& yaml, Protobuf::Message& message,
+                                          ProtobufMessage::ValidationVisitor& validation_visitor);
 #endif
 
   // This function attempts to load Envoy configuration from the specified file

source/common/protobuf/yaml_utility.cc

@@ -113,6 +113,20 @@ void jsonConvertInternal(const Protobuf::Message& source,
   MessageUtil::loadFromJson(json, dest, validation_visitor);
 }
 
+absl::Status jsonConvertInternalNoThrow(const Protobuf::Message& source,
+                                        ProtobufMessage::ValidationVisitor& validation_visitor,
+                                        Protobuf::Message& dest) {
+  absl::Status conversion_status = absl::OkStatus();
+  Protobuf::util::JsonPrintOptions json_options;
+  json_options.preserve_proto_field_names = true;
+  std::string json;
+  conversion_status = Protobuf::util::MessageToJsonString(source, &json, json_options);
+  if (conversion_status.ok()) {
+    MessageUtil::loadFromJson(json, dest, validation_visitor);
+  }
+  return conversion_status;
+}
+
 } // namespace
 
 void MessageUtil::loadFromJson(absl::string_view json, Protobuf::Message& message,
@@ -186,6 +200,38 @@ void MessageUtil::loadFromYaml(const std::string& yaml, Protobuf::Message& messa
   throw EnvoyException("Unable to convert YAML as JSON: " + yaml);
 }
 
+absl::Status
+MessageUtil::loadFromYamlNoThrow(const std::string& yaml, Protobuf::Message& message,
+                                 ProtobufMessage::ValidationVisitor& validation_visitor) {
+  absl::Status load_status = absl::OkStatus();
+  Protobuf::Value value;
+  TRY_NEEDS_AUDIT { value = parseYamlNode(YAML::Load(yaml)); }
+  END_TRY
+  catch (YAML::ParserException& e) {
+    load_status = absl::InvalidArgumentError(fmt::format("Failed to parse yaml: {}", e.what()));
+  }
+  catch (YAML::BadConversion& e) {
+    load_status =
+        absl::InvalidArgumentError(fmt::format("Failed to convert to yaml: {}", e.what()));
+  }
+  catch (std::exception& e) {
+    // There is a potentially wide space of exceptions thrown by the YAML parser,
+    // and enumerating them all may be difficult. Envoy doesn't work well with
+    // unhandled exceptions, so we capture them and record the exception name in
+    // the status.
+    load_status =
+        absl::InvalidArgumentError(fmt::format("Unexpected YAML exception: {}", e.what()));
+  }
+  if (value.kind_case() == Protobuf::Value::kStructValue ||
+      value.kind_case() == Protobuf::Value::kListValue) {
+    load_status = jsonConvertInternalNoThrow(value, validation_visitor, message);
+  } else {
+    load_status =
+        absl::InvalidArgumentError(fmt::format("Unable to convert YAML as JSON: {}", yaml));
+  }
+  return load_status;
+}
+
 std::string MessageUtil::getYamlStringFromMessage(const Protobuf::Message& message,
                                                   const bool block_print,
                                                   const bool always_print_primitive_fields) {

source/extensions/filters/http/ip_tagging/BUILD

@@ -19,7 +19,10 @@ envoy_cc_library(
     deps = [
         "//envoy/http:filter_interface",
         "//envoy/runtime:runtime_interface",
+        "//envoy/thread_local:thread_local_interface",
         "//source/common/common:assert_lib",
+        "//source/common/common:thread_synchronizer_lib",
+        "//source/common/config:datasource_lib",
         "//source/common/http:header_map_lib",
         "//source/common/http:headers_lib",
         "//source/common/network:lc_trie_lib",
@@ -35,6 +38,7 @@ envoy_cc_extension(
     hdrs = ["config.h"],
     deps = [
         "//envoy/registry",
+        "//envoy/thread_local:thread_local_interface",
         "//source/common/protobuf:utility_lib",
         "//source/extensions/filters/http/common:factory_base_lib",
         "//source/extensions/filters/http/ip_tagging:ip_tagging_filter_lib",