From 253168809c9871c8fe233e564ab3762322622759 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Mon, 25 May 2026 20:11:44 +0200 Subject: [PATCH 1/4] tests: Add missing endpoint IDs Make sure each test case has endpoint ID field in the NetworkPolicy so that we can validate for it. Signed-off-by: Jarno Rajahalme --- tests/cilium_tcp_integration.cc | 1 + tests/cilium_tcp_integration_test.cc | 2 ++ tests/cilium_tls_http_integration_test.cc | 1 + tests/cilium_tls_tcp_integration_test.cc | 1 + tests/cilium_websocket_codec_integration_test.cc | 1 + tests/cilium_websocket_decap_integration_test.cc | 1 + tests/cilium_websocket_policy_integration_test.cc | 1 + 7 files changed, 8 insertions(+) diff --git a/tests/cilium_tcp_integration.cc b/tests/cilium_tcp_integration.cc index 40ea6f1a1..049a6e352 100644 --- a/tests/cilium_tcp_integration.cc +++ b/tests/cilium_tcp_integration.cc @@ -26,6 +26,7 @@ const std::string TCP_POLICY_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_tcp_integration_test.cc b/tests/cilium_tcp_integration_test.cc index 24ca3817c..b7be8f2b2 100644 --- a/tests/cilium_tcp_integration_test.cc +++ b/tests/cilium_tcp_integration_test.cc @@ -343,6 +343,7 @@ const std::string TCP_POLICY_LINEPARSER_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: 1 @@ -573,6 +574,7 @@ const std::string TCP_POLICY_BLOCKPARSER_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: 1 diff --git a/tests/cilium_tls_http_integration_test.cc b/tests/cilium_tls_http_integration_test.cc index 040ada472..9d277c945 100644 --- a/tests/cilium_tls_http_integration_test.cc +++ b/tests/cilium_tls_http_integration_test.cc @@ -184,6 +184,7 @@ const std::string BASIC_TLS_POLICY_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_tls_tcp_integration_test.cc b/tests/cilium_tls_tcp_integration_test.cc index 05999a877..d3c42bd41 100644 --- a/tests/cilium_tls_tcp_integration_test.cc +++ b/tests/cilium_tls_tcp_integration_test.cc @@ -273,6 +273,7 @@ const std::string TCP_POLICY_UPSTREAM_TLS_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_websocket_codec_integration_test.cc b/tests/cilium_websocket_codec_integration_test.cc index 05eb64d13..c4d1f70bf 100644 --- a/tests/cilium_websocket_codec_integration_test.cc +++ b/tests/cilium_websocket_codec_integration_test.cc @@ -98,6 +98,7 @@ class CiliumWebSocketIntegrationTest : public CiliumTcpIntegrationTest { - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_websocket_decap_integration_test.cc b/tests/cilium_websocket_decap_integration_test.cc index d7a7055f4..46f9411fe 100644 --- a/tests/cilium_websocket_decap_integration_test.cc +++ b/tests/cilium_websocket_decap_integration_test.cc @@ -101,6 +101,7 @@ class CiliumWebSocketIntegrationTest : public CiliumHttpIntegrationTest { - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_websocket_policy_integration_test.cc b/tests/cilium_websocket_policy_integration_test.cc index c456a9312..17b682c5a 100644 --- a/tests/cilium_websocket_policy_integration_test.cc +++ b/tests/cilium_websocket_policy_integration_test.cc @@ -231,6 +231,7 @@ class CiliumWebSocketIntegrationTest : public CiliumTcpIntegrationTest { - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} From a75a2fa3cf3df4698ff903791eab1b3958e247dc Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Thu, 2 Apr 2026 17:02:22 +0200 Subject: [PATCH 2/4] policy: detect new streams for delta and file-based substriptions Replace Config::GrpcMuxImpl wrapper with stream event callback patch on upstream so that new stream detection works on all the needed Mux types for SotW, Delta, and ADS. New stream detection is the means by which we detect Cilium Agent restarts, which generally requires the ipcache bpf map to be reopened. Delta updates also depend on this detection to force synchronization as the restarted agent may not know which resources to remove. Signed-off-by: Jarno Rajahalme --- WORKSPACE | 1 + cilium/grpc_subscription.cc | 144 ++++-- cilium/grpc_subscription.h | 33 +- cilium/network_policy.cc | 132 ++++-- cilium/network_policy.h | 16 +- ...g-add-grpc-mux-stream-event-callback.patch | 448 ++++++++++++++++++ tests/bpf_metadata_integration_test.cc | 299 +++++++++--- 7 files changed, 900 insertions(+), 173 deletions(-) create mode 100644 patches/0007-config-add-grpc-mux-stream-event-callback.patch diff --git a/WORKSPACE b/WORKSPACE index 15332cd5e..7f0120e3e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -43,6 +43,7 @@ git_repository( "@//patches:0004-thread_local-reset-slot-in-worker-threads-first.patch", "@//patches:0005-http-header-expose-attribute.patch", "@//patches:0006-test-integration-Defer-fake-upstream-read-enable-un.patch", + "@//patches:0007-config-add-grpc-mux-stream-event-callback.patch", "@//patches:0008-repo-Make-yq-dependency-optional-for-CI-config-parsi.patch", ], # // clang-format off: Envoy's format check: Only repository_locations.bzl may contains URL references diff --git a/cilium/grpc_subscription.cc b/cilium/grpc_subscription.cc index 1bfdd9a9e..f16932007 100644 --- a/cilium/grpc_subscription.cc +++ b/cilium/grpc_subscription.cc @@ -2,16 +2,18 @@ #include -#include +#include #include #include #include #include #include "envoy/annotations/resource.pb.h" +#include "envoy/common/callback.h" #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/config/custom_config_validators.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/config/subscription_factory.h" #include "envoy/grpc/async_client.h" @@ -25,9 +27,12 @@ #include "source/common/grpc/common.h" #include "source/common/protobuf/protobuf.h" // IWYU pragma: keep #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" #include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/string_view.h" @@ -38,6 +43,31 @@ namespace Cilium { namespace { +class StreamEventSubscription : public Config::Subscription { +public: + StreamEventSubscription(std::unique_ptr subscription, + Common::CallbackHandlePtr stream_event_handle) + : subscription_(std::move(subscription)), + stream_event_handle_(std::move(stream_event_handle)) {} + + void start(const absl::flat_hash_set& resource_names) override { + subscription_->start(resource_names); + } + + void + updateResourceInterest(const absl::flat_hash_set& update_to_these_names) override { + subscription_->updateResourceInterest(update_to_these_names); + } + + void requestOnDemandUpdate(const absl::flat_hash_set& add_these_names) override { + subscription_->requestOnDemandUpdate(add_these_names); + } + +private: + std::unique_ptr subscription_; + Common::CallbackHandlePtr stream_event_handle_; +}; + // service RPC method fully qualified names. struct Service { std::string sotw_grpc_method_; @@ -124,49 +154,81 @@ subscribe(const absl::string_view type_url, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, - std::chrono::milliseconds init_fetch_timeout) { - auto& api_config_source = config_source.api_config_source(); - THROW_IF_NOT_OK(Config::Utility::checkApiConfigSourceSubscriptionBackingCluster( - context.clusterManager().primaryClusters(), api_config_source)); - + Config::GrpcMuxStreamEventCallback on_stream_event) { + auto initial_fetch_timeout = Config::Utility::configSourceInitialFetchTimeout(config_source); Config::SubscriptionStats stats = Config::Utility::generateStats(scope); Envoy::Config::SubscriptionOptions options; - // No-op custom validators - Envoy::Config::CustomConfigValidatorsPtr nop_config_validators = - std::make_unique(); - auto factory_or_error = Config::Utility::factoryForGrpcApiConfigSource( - context.clusterManager().grpcAsyncClientManager(), api_config_source, scope, true, 0, false); - THROW_IF_NOT_OK_REF(factory_or_error.status()); - - absl::StatusOr rate_limit_settings_or_error = - Config::Utility::parseRateLimitSettings(api_config_source); - THROW_IF_NOT_OK_REF(rate_limit_settings_or_error.status()); - - Config::GrpcMuxContext grpc_mux_context{ - /*async_client_=*/THROW_OR_RETURN_VALUE( - factory_or_error.value()->createUncachedRawAsyncClient(), Grpc::RawAsyncClientPtr), - /*failover_async_client_=*/nullptr, - /*dispatcher_=*/context.mainThreadDispatcher(), - /*service_method_=*/sotwGrpcMethod(type_url), - /*local_info_=*/context.localInfo(), - /*rate_limit_settings_=*/rate_limit_settings_or_error.value(), - /*scope_=*/scope, - /*config_validators_=*/std::move(nop_config_validators), - /*xds_resources_delegate_=*/absl::nullopt, - /*xds_config_tracker_=*/absl::nullopt, - /*backoff_strategy_=*/ - std::make_unique( - Config::SubscriptionFactory::RetryInitialDelayMs, - Config::SubscriptionFactory::RetryMaxDelayMs, context.api().randomGenerator()), - /*target_xds_authority_=*/"", - /*eds_resources_cache_=*/nullptr, // EDS cache is only used for ADS. - /*skip_subsequent_node_=*/api_config_source.set_node_on_first_message_only(), - }; - - return std::make_unique( - std::make_shared(grpc_mux_context), callbacks, resource_decoder, stats, type_url, - context.mainThreadDispatcher(), init_fetch_timeout, /*is_aggregated*/ false, options); + std::shared_ptr grpc_mux; + bool is_aggregated = + config_source.config_source_specifier_case() == envoy::config::core::v3::ConfigSource::kAds; + if (is_aggregated) { + grpc_mux = std::static_pointer_cast(context.xdsManager().adsMux()); + } else { + auto& api_config_source = config_source.api_config_source(); + THROW_IF_NOT_OK(Config::Utility::checkApiConfigSourceSubscriptionBackingCluster( + context.clusterManager().primaryClusters(), api_config_source)); + + // No-op custom validators + Envoy::Config::CustomConfigValidatorsPtr nop_config_validators = + std::make_unique(); + auto factory_or_error = Config::Utility::factoryForGrpcApiConfigSource( + context.clusterManager().grpcAsyncClientManager(), api_config_source, scope, true, 0, + false); + THROW_IF_NOT_OK_REF(factory_or_error.status()); + + absl::StatusOr rate_limit_settings_or_error = + Config::Utility::parseRateLimitSettings(api_config_source); + THROW_IF_NOT_OK_REF(rate_limit_settings_or_error.status()); + + const auto& api_type = api_config_source.api_type(); + bool use_delta = api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; + const auto& service_method = use_delta ? deltaGrpcMethod(type_url) : sotwGrpcMethod(type_url); + + Config::GrpcMuxContext grpc_mux_context{ + THROW_OR_RETURN_VALUE(factory_or_error.value()->createUncachedRawAsyncClient(), + Grpc::RawAsyncClientPtr), + /*failover_async_client_=*/nullptr, + context.mainThreadDispatcher(), + service_method, + context.localInfo(), + rate_limit_settings_or_error.value(), + scope, + std::move(nop_config_validators), + /*xds_resources_delegate_=*/absl::nullopt, + /*xds_config_tracker_=*/absl::nullopt, + std::make_unique( + Config::SubscriptionFactory::RetryInitialDelayMs, + Config::SubscriptionFactory::RetryMaxDelayMs, context.api().randomGenerator()), + /*target_xds_authority_=*/"", + /*eds_resources_cache_=*/nullptr, // EDS cache is only used for ADS. + /*skip_subsequent_node_=*/api_config_source.set_node_on_first_message_only(), + }; + + grpc_mux = use_delta ? std::static_pointer_cast( + std::make_shared(grpc_mux_context)) + : std::static_pointer_cast( + std::make_shared(grpc_mux_context)); + } + + Common::CallbackHandlePtr stream_event_handle; + if (on_stream_event) { + auto stream_event_callback = std::move(on_stream_event); + stream_event_handle = grpc_mux->addStreamEventCallback(stream_event_callback); + if (grpc_mux->grpcStreamConnected()) { + stream_event_callback(Config::GrpcMuxStreamEvent::Established); + } + } + + auto subscription = std::make_unique( + grpc_mux, callbacks, resource_decoder, stats, type_url, context.mainThreadDispatcher(), + initial_fetch_timeout, is_aggregated, options); + if (stream_event_handle) { + return std::make_unique(std::move(subscription), + std::move(stream_event_handle)); + } + return subscription; } } // namespace Cilium diff --git a/cilium/grpc_subscription.h b/cilium/grpc_subscription.h index 10d5b5155..1e8e85f08 100644 --- a/cilium/grpc_subscription.h +++ b/cilium/grpc_subscription.h @@ -1,51 +1,24 @@ #pragma once -#include #include #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" -#include "envoy/server/factory_context.h" +#include "envoy/ssl/context_manager.h" #include "envoy/stats/scope.h" -#include "source/extensions/config_subscription/grpc/grpc_mux_context.h" -#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" - #include "absl/strings/string_view.h" namespace Envoy { namespace Cilium { -// GrpcMux wrapper to get access to control plane identifier -class GrpcMuxImpl : public Config::GrpcMuxImpl { -public: - GrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context) : Config::GrpcMuxImpl(grpc_mux_context) {} - - ~GrpcMuxImpl() override = default; - - void onStreamEstablished() override { - new_stream_ = true; - Config::GrpcMuxImpl::onStreamEstablished(); - } - - // isNewStream returns true for the first call after a new stream has been established - bool isNewStream() { - bool new_stream = new_stream_; - new_stream_ = false; - return new_stream; - } - -private: - bool new_stream_ = true; -}; - std::unique_ptr subscribe(const absl::string_view type_url, const envoy::config::core::v3::ConfigSource& config_source, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, - std::chrono::milliseconds init_fetch_timeout = std::chrono::milliseconds(0)); - + Config::GrpcMuxStreamEventCallback on_stream_event = {}); } // namespace Cilium } // namespace Envoy diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index cf5d2141a..05a6b5456 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -22,6 +22,7 @@ #include "envoy/config/core/v3/address.pb.h" #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/event/dispatcher_thread_deletable.h" #include "envoy/http/header_map.h" @@ -49,7 +50,6 @@ #include "source/common/network/utility.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/server/transport_socket_config_impl.h" #include "absl/container/btree_map.h" @@ -127,32 +127,44 @@ class PolicyInstanceImpl; using PolicyMapSnapshot = absl::flat_hash_map>; +// PolicyStreamState is shared by all policies created from one accepted NPDS stream generation. +class PolicyStreamState { +public: + explicit PolicyStreamState(uint64_t stream_generation) : stream_generation_(stream_generation) {} + + uint64_t streamGeneration() const { return stream_generation_; } + +private: + const uint64_t stream_generation_; +}; +using PolicyStreamStateSharedPtr = std::shared_ptr; +using PolicyStreamStateConstSharedPtr = std::shared_ptr; + class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, public Envoy::Event::DispatcherThreadDeletable, - public Logger::Loggable { + public Logger::Loggable, + public std::enable_shared_from_this { public: NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, const envoy::config::core::v3::ConfigSource& config_source); ~NetworkPolicyMapImpl() override; void startSubscription(const envoy::config::core::v3::ConfigSource& config_source) { - if (config_source.config_source_specifier_case() == - envoy::config::core::v3::ConfigSource::kAds) { - auto ads_mux = context_.xdsManager().adsMux(); - subscription_ = THROW_OR_RETURN_VALUE( - context_.clusterManager().subscriptionFactory().subscriptionOverAdsGrpcMux( - ads_mux, config_source, NetworkPolicyTypeUrl, *npds_stats_scope_, *this, - std::make_shared(), {}), - Config::SubscriptionPtr); - } else { - subscription_ = subscribe(NetworkPolicyTypeUrl, config_source, context_, *npds_stats_scope_, - *this, std::make_shared()); - } + const uint64_t subscription_id = ++subscription_id_; + subscription_ = subscribe( + NetworkPolicyTypeUrl, config_source, context_, *npds_stats_scope_, *this, + std::make_shared(), + [weak_this = weak_from_this(), subscription_id](Config::GrpcMuxStreamEvent event) { + if (auto shared_this = weak_this.lock()) { + shared_this->onSubscriptionStreamEvent(subscription_id, event); + } + }); } // This is used for testing with a file-based subscription void startSubscription(std::unique_ptr&& subscription) { subscription_ = std::move(subscription); + subscription_connected_ = false; // XXX } const envoy::config::core::v3::ConfigSource& getConfigSource() const { return config_source_; } @@ -181,19 +193,8 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, void tlsWrapperMissingPolicyInc() const { stats_.tls_wrapper_missing_policy_.inc(); } protected: - bool isNewStream() const { - auto sub = dynamic_cast(subscription_.get()); - if (!sub) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcSubscriptionImpl"); - return false; - } - auto mux = dynamic_cast(sub->grpcMux().get()); - if (!mux) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcMuxImpl"); - return false; - } - return mux->isNewStream(); - } + uint64_t streamGeneration() const { return subscription_stream_generation_; } + void resetStreamForTest() { subscription_stream_generation_++; } // run the given function after all the threads have scheduled void runAfterAllThreads(std::function cb) const { @@ -214,9 +215,43 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, createOrReusePolicy(const cilium::NetworkPolicy& config, const PolicyMapSnapshot& old_policy_map); void installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, - Init::ManagerImpl& version_init_manager, std::string&& version_name); + Init::ManagerImpl& version_init_manager, std::string&& version_name, + const PolicyStreamStateSharedPtr& policy_stream_state); private: + void onSubscriptionStreamEvent(uint64_t subscription_id, Config::GrpcMuxStreamEvent event) { + // skip stale notifications for earlier subscriptions + if (subscription_id != subscription_id_) { + return; + } + switch (event) { + case Config::GrpcMuxStreamEvent::Established: + ++subscription_stream_generation_; + subscription_connected_ = true; + break; + case Config::GrpcMuxStreamEvent::Closed: + if (!subscription_connected_) { + return; + } + subscription_connected_ = false; + + // The close callback runs on the subscription object's own stack, so defer any possible + // recreation until after it unwinds to avoid destroying the current subscription + // mid-callback. + context_.mainThreadDispatcher().post( + [weak_this = weak_from_this(), subscription_id = subscription_id_]() { + if (auto shared_this = weak_this.lock()) { + // skip stale callbacks for earlier subscriptions + if (subscription_id != shared_this->subscription_id_) { + return; + } + // shared_this->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + } + }); + break; + } + } + // Helpers for atomic swap of the policy map pointer. // // store() is only used for the initialization of the map during construction. @@ -260,8 +295,12 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, static uint64_t instance_id_; + bool subscription_connected_{false}; + uint64_t subscription_id_{0}; Server::Configuration::ServerFactoryContext& context_; std::atomic map_ptr_; + // Policies hold a shared per-stream state object. + PolicyStreamStateSharedPtr policy_stream_state_{std::make_shared(0)}; Stats::ScopeSharedPtr npds_stats_scope_; Stats::ScopeSharedPtr policy_stats_scope_; @@ -276,6 +315,7 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, envoy::config::core::v3::ConfigSource config_source_; std::unique_ptr subscription_; + static uint64_t subscription_stream_generation_; ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; @@ -287,6 +327,7 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, }; uint64_t NetworkPolicyMapImpl::instance_id_ = 0; +uint64_t NetworkPolicyMapImpl::subscription_stream_generation_ = 1; IpAddressPair::IpAddressPair(const cilium::NetworkPolicy& proto) { for (const auto& ip_addr : proto.endpoint_ips()) { @@ -1691,8 +1732,7 @@ NetworkPolicyMap::~NetworkPolicyMap() { ENVOY_LOG(debug, "Cilium L7 NetworkPolicyMap: posting NetworkPolicyMapImpl deletion to main thread"); - context_.mainThreadDispatcher().deleteInDispatcherThread( - Event::DispatcherThreadDeletableConstPtr(impl_.release())); + context_.mainThreadDispatcher().post([impl = std::move(impl_)]() mutable { impl.reset(); }); } bool NetworkPolicyMap::exists(const std::string& endpoint_policy_name) const { @@ -1728,7 +1768,8 @@ NetworkPolicyMapImpl::NetworkPolicyMapImpl( context_.messageValidationContext().dynamicValidationVisitor())), parked_init_manager_(std::make_unique("Cilium NetworkPolicyMap parked")), config_source_(config_source), - stats_{ALL_CILIUM_POLICY_STATS(POOL_COUNTER(*policy_stats_scope_))} { + stats_{ALL_CILIUM_POLICY_COUNTERS(POOL_COUNTER(*policy_stats_scope_)) + ALL_CILIUM_POLICY_GAUGES(POOL_GAUGE(*policy_stats_scope_))} { // Use listener init manager for subscription initialization context.initManager().add(init_target_); transport_factory_context_->setInitManager(*parked_init_manager_); @@ -1783,14 +1824,19 @@ NetworkPolicyMapImpl::createOrReusePolicy(const cilium::NetworkPolicy& config, return std::make_shared(*this, new_hash, config); } -void NetworkPolicyMapImpl::installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, - Init::ManagerImpl& version_init_manager, - std::string&& version_name) { +void NetworkPolicyMapImpl::installNewPolicyMap( + PolicyMapSnapshot&& new_policy_map, Init::ManagerImpl& version_init_manager, + std::string&& version_name, const PolicyStreamStateSharedPtr& policy_stream_state) { // Initialize SDS secrets. We do not wait for the completion. version_init_manager.initialize(Init::WatcherImpl(std::move(version_name), []() {})); const auto* old_policy_map = exchange(new PolicyMapSnapshot(std::move(new_policy_map))); + // Record stream state only after a successful install. The reserved value 0 + // keeps the initial accepted update on any stream source classified as new. + policy_stream_state_ = policy_stream_state; + stats_.policy_stream_generation_.set(policy_stream_state_->streamGeneration()); + // Delete the old map once all worker threads have entered their event queues, as this // is proof that they no longer refer to the old map. runAfterAllThreads([old_policy_map]() { @@ -1846,15 +1892,20 @@ void NetworkPolicyMapImpl::removeInitManager() { absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::vector& resources, const std::string& version_info) { - ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}", - instance_id_, resources.size(), version_info); + subscription_connected_ = true; + auto stream_generation = streamGeneration(); + // policy_stream_state_ gets updated on first successful update, + // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. + const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); + ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}, stream {}", + instance_id_, resources.size(), version_info, stream_generation); stats_.updates_total_.inc(); // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, // and that is also when the old stream terminates and a new one is created. // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, // so open it before the workers get a chance to enforce policy on the new IDs. - if (isNewStream()) { + if (is_new_stream) { ENVOY_LOG(info, "New NetworkPolicy stream"); reopenIpcache(); @@ -1867,6 +1918,8 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // SDS secrets will use this! transport_factory_context_->setInitManager(version_init_manager); + const auto policy_stream_state = + is_new_stream ? std::make_shared(stream_generation) : policy_stream_state_; const auto* old_policy_map = load(); PolicyMapSnapshot new_policy_map; try { @@ -1895,8 +1948,9 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( } stats_.update_success_.inc(); - removeInitManager(); - installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name)); + + installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name), + policy_stream_state); return absl::OkStatus(); } diff --git a/cilium/network_policy.h b/cilium/network_policy.h index ab5621c22..03d376506 100644 --- a/cilium/network_policy.h +++ b/cilium/network_policy.h @@ -171,18 +171,22 @@ class NetworkPolicyDecoder : public Envoy::Config::OpaqueResourceDecoder { * All Cilium L7 filter stats. @see stats_macros.h */ // clang-format off -#define ALL_CILIUM_POLICY_STATS(COUNTER) \ - COUNTER(updates_total) \ - COUNTER(updates_rejected) \ - COUNTER(tls_wrapper_missing_policy) \ +#define ALL_CILIUM_POLICY_COUNTERS(COUNTER) \ + COUNTER(updates_total) \ + COUNTER(updates_rejected) \ + COUNTER(tls_wrapper_missing_policy) \ COUNTER(update_success) + +#define ALL_CILIUM_POLICY_GAUGES(GAUGE) \ + GAUGE(policy_stream_generation, NeverImport) // clang-format on /** * Struct definition for all policy stats. @see stats_macros.h */ struct PolicyStats { - ALL_CILIUM_POLICY_STATS(GENERATE_COUNTER_STRUCT) + ALL_CILIUM_POLICY_COUNTERS(GENERATE_COUNTER_STRUCT) + ALL_CILIUM_POLICY_GAUGES(GENERATE_GAUGE_STRUCT) }; class NetworkPolicyMapImpl; @@ -211,7 +215,7 @@ class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable impl_; + std::shared_ptr impl_; }; using NetworkPolicyMapSharedPtr = std::shared_ptr; diff --git a/patches/0007-config-add-grpc-mux-stream-event-callback.patch b/patches/0007-config-add-grpc-mux-stream-event-callback.patch new file mode 100644 index 000000000..e92fff4dd --- /dev/null +++ b/patches/0007-config-add-grpc-mux-stream-event-callback.patch @@ -0,0 +1,448 @@ +diff --git a/envoy/config/BUILD b/envoy/config/BUILD +--- a/envoy/config/BUILD ++++ b/envoy/config/BUILD +@@ -66,6 +66,7 @@ envoy_cc_library( + name = "grpc_mux_interface", + hdrs = ["grpc_mux.h"], + deps = [ ++ "//envoy/common:callback", + ":eds_resources_cache_interface", + ":subscription_interface", + "//envoy/stats:stats_macros", +diff --git a/envoy/config/grpc_mux.h b/envoy/config/grpc_mux.h +--- a/envoy/config/grpc_mux.h ++++ b/envoy/config/grpc_mux.h +@@ -1,8 +1,10 @@ + #pragma once + ++#include + #include + + #include "envoy/common/backoff_strategy.h" ++#include "envoy/common/callback.h" + #include "envoy/common/exception.h" + #include "envoy/common/pure.h" + #include "envoy/config/custom_config_validators.h" +@@ -17,6 +19,11 @@ namespace Envoy { + namespace Config { + + using ScopedResume = std::unique_ptr; ++ ++enum class GrpcMuxStreamEvent { Established, Closed }; ++ ++using GrpcMuxStreamEventCallback = std::function; ++ + /** + * All control plane related stats. @see stats_macros.h + */ +@@ -61,6 +67,18 @@ public: + */ + virtual void start() PURE; + ++ /** ++ * Add a callback to observe gRPC mux stream lifecycle events. ++ * @return a handle that unregisters the callback when destroyed. ++ */ ++ virtual Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) PURE; ++ ++ /** ++ * @return true if the mux currently has an established gRPC stream. ++ */ ++ virtual bool grpcStreamConnected() const PURE; ++ + /** + * Pause discovery requests for a given API type. This is useful when we're processing an update + * for LDS or CDS and don't want a flood of updates for RDS or EDS respectively. Discovery +diff --git a/source/common/config/BUILD b/source/common/config/BUILD +--- a/source/common/config/BUILD ++++ b/source/common/config/BUILD +@@ -86,10 +86,19 @@ envoy_cc_library( + ], + ) +- ++envoy_cc_library( ++ name = "grpc_mux_stream_event_tracker_lib", ++ hdrs = ["grpc_mux_stream_event_tracker.h"], ++ deps = [ ++ "//envoy/config:grpc_mux_interface", ++ "//source/common/common:callback_impl_lib", ++ ], ++) ++ + envoy_cc_library( + name = "null_grpc_mux_lib", + hdrs = ["null_grpc_mux_impl.h"], + deps = [ ++ ":grpc_mux_stream_event_tracker_lib", + "//envoy/config:grpc_mux_interface", + ], + ) +diff --git a/source/common/config/null_grpc_mux_impl.h b/source/common/config/null_grpc_mux_impl.h +--- a/source/common/config/null_grpc_mux_impl.h ++++ b/source/common/config/null_grpc_mux_impl.h +@@ -1,6 +1,7 @@ + #pragma once + + #include "envoy/config/grpc_mux.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + + namespace Envoy { + namespace Config { +@@ -11,5 +12,11 @@ class NullGrpcMuxImpl : public GrpcMux, + public: + void start() override {} ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return false; } + ScopedResume pause(const std::string&) override { + return std::make_unique([] {}); + } +@@ -40,6 +48,9 @@ public: + void onEstablishmentFailure(bool) override {} + void onDiscoveryResponse(std::unique_ptr&&, + ControlPlaneStats&) override {} ++ ++private: ++ GrpcMuxStreamEventTracker stream_event_tracker_; + }; + + } // namespace Config +diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD +--- a/source/extensions/config_subscription/grpc/BUILD ++++ b/source/extensions/config_subscription/grpc/BUILD +@@ -33,6 +33,7 @@ envoy_cc_extension( + ":eds_resources_cache_lib", + ":grpc_mux_context_lib", + ":grpc_mux_failover_lib", ++ "//source/common/config:grpc_mux_stream_event_tracker_lib", + ":grpc_stream_lib", + ":xds_source_id_lib", + "//envoy/config:custom_config_validators_interface", +@@ -65,6 +66,7 @@ envoy_cc_library( + ":grpc_mux_context_lib", + ":grpc_mux_failover_lib", + ":grpc_stream_lib", ++ "//source/common/config:grpc_mux_stream_event_tracker_lib", + ":pausable_ack_queue_lib", + ":watch_map_lib", + "//envoy/config:custom_config_validators_interface", +diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +--- a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc ++++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +@@ -569,6 +569,7 @@ void GrpcMuxImpl::onStreamEstablished() { + for (const auto& type_url : subscriptions_) { + queueDiscoveryRequest(type_url); + } ++ stream_event_tracker_.onStreamEstablished(); + } + + void GrpcMuxImpl::onEstablishmentFailure(bool) { +@@ -589,6 +590,7 @@ void GrpcMuxImpl::onEstablishmentFailure(bool) { + api_state.second->previously_fetched_data_ = true; + } + } ++ stream_event_tracker_.onStreamClosed(); + } + + void GrpcMuxImpl::queueDiscoveryRequest(absl::string_view queue_item) { +diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/grpc_mux_impl.h +--- a/source/extensions/config_subscription/grpc/grpc_mux_impl.h ++++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.h +@@ -3,6 +3,7 @@ + #include + #include + #include ++#include + + #include "envoy/common/random_generator.h" + #include "envoy/common/time.h" +@@ -29,5 +30,6 @@ + #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + + #include "absl/container/node_hash_map.h" + #include "xds/core/v3/resource_name.pb.h" +@@ -57,6 +59,13 @@ public: + + void start() override; + ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + // GrpcMux + ScopedResume pause(const std::string& type_url) override; + ScopedResume pause(const std::vector type_urls) override; +@@ -290,6 +299,7 @@ private: + const bool skip_subsequent_node_; + CustomConfigValidatorsPtr config_validators_; + XdsConfigTrackerOptRef xds_config_tracker_; ++ GrpcMuxStreamEventTracker stream_event_tracker_; + XdsResourcesDelegateOptRef xds_resources_delegate_; + EdsResourcesCachePtr eds_resources_cache_; + const std::string target_xds_authority_; +diff --git a/source/common/config/grpc_mux_stream_event_tracker.h b/source/common/config/grpc_mux_stream_event_tracker.h +new file mode 100644 +--- /dev/null ++++ b/source/common/config/grpc_mux_stream_event_tracker.h +@@ -0,0 +1,36 @@ ++#pragma once ++ ++#include ++ ++#include "envoy/config/grpc_mux.h" ++ ++#include "source/common/common/callback_impl.h" ++ ++namespace Envoy { ++namespace Config { ++ ++class GrpcMuxStreamEventTracker { ++public: ++ Common::CallbackHandlePtr addStreamEventCallback(GrpcMuxStreamEventCallback callback) { ++ return callbacks_.add(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const { return grpc_stream_connected_; } ++ ++ void onStreamEstablished() { ++ grpc_stream_connected_ = true; ++ callbacks_.runCallbacks(GrpcMuxStreamEvent::Established); ++ } ++ ++ void onStreamClosed() { ++ grpc_stream_connected_ = false; ++ callbacks_.runCallbacks(GrpcMuxStreamEvent::Closed); ++ } ++ ++private: ++ bool grpc_stream_connected_{false}; ++ Common::CallbackManager callbacks_; ++}; ++ ++} // namespace Config ++} // namespace Envoy +diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc +--- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc ++++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc +@@ -194,6 +194,7 @@ void NewGrpcMuxImpl::onStreamEstablished() { + } + pausable_ack_queue_.clear(); + trySendDiscoveryRequests(); ++ stream_event_tracker_.onStreamEstablished(); + } + + void NewGrpcMuxImpl::onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) { +@@ -215,6 +216,7 @@ void NewGrpcMuxImpl::onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) { + } + } while (all_subscribed.size() != subscriptions_.size()); + should_send_initial_resource_versions_ = next_attempt_may_send_initial_resource_version; ++ stream_event_tracker_.onStreamClosed(); + } + + void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } +diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +--- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h ++++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +@@ -1,6 +1,7 @@ + #pragma once + + #include ++#include + + #include "envoy/common/random_generator.h" + #include "envoy/common/token_bucket.h" +@@ -20,6 +21,7 @@ + #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" + #include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + #include "source/extensions/config_subscription/grpc/watch_map.h" + + namespace Envoy { +@@ -50,6 +52,13 @@ public: + static void shutdownAll(); + + void shutdown() { shutdown_ = true; } ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + + GrpcMuxWatchPtr addWatch(const std::string& type_url, + const absl::flat_hash_set& resources, +@@ -217,6 +226,7 @@ private: + + const LocalInfo::LocalInfo& local_info_; + CustomConfigValidatorsPtr config_validators_; ++ GrpcMuxStreamEventTracker stream_event_tracker_; + Common::CallbackHandlePtr dynamic_update_callback_handle_; + XdsConfigTrackerOptRef xds_config_tracker_; + EdsResourcesCachePtr eds_resources_cache_; +diff --git a/source/extensions/config_subscription/grpc/xds_mux/BUILD b/source/extensions/config_subscription/grpc/xds_mux/BUILD +--- a/source/extensions/config_subscription/grpc/xds_mux/BUILD ++++ b/source/extensions/config_subscription/grpc/xds_mux/BUILD +@@ -72,6 +72,7 @@ envoy_cc_extension( + "//source/extensions/config_subscription/grpc:eds_resources_cache_lib", + "//source/extensions/config_subscription/grpc:grpc_mux_context_lib", + "//source/extensions/config_subscription/grpc:grpc_mux_failover_lib", ++ "//source/common/config:grpc_mux_stream_event_tracker_lib", + "//source/extensions/config_subscription/grpc:grpc_stream_lib", + "//source/extensions/config_subscription/grpc:pausable_ack_queue_lib", + "//source/extensions/config_subscription/grpc:watch_map_lib", +diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +--- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc ++++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +@@ -322,6 +322,7 @@ void GrpcMuxImpl::handleEstablishedStream() { + maybeUpdateQueueSizeStat(0); + pausable_ack_queue_.clear(); + trySendDiscoveryRequests(); ++ stream_event_tracker_.onStreamEstablished(); + } + + template +@@ -346,6 +347,7 @@ void GrpcMuxImpl::handleStreamEstablishmentFailure( + } + } while (all_subscribed.size() != subscriptions_.size()); + should_send_initial_resource_versions_ = next_attempt_may_send_initial_resource_version; ++ stream_event_tracker_.onStreamClosed(); + } + + template +diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +--- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h ++++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +@@ -3,6 +3,7 @@ + #include + #include + #include ++#include + + #include "envoy/common/random_generator.h" + #include "envoy/common/time.h" +@@ -23,6 +24,7 @@ + #include "source/common/grpc/common.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + #include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" + #include "source/extensions/config_subscription/grpc/watch_map.h" + #include "source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h" +@@ -82,6 +84,13 @@ public: + SubscriptionCallbacks& callbacks, + OpaqueResourceDecoderSharedPtr resource_decoder, + const SubscriptionOptions& options) override; ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + void updateWatch(const std::string& type_url, Watch* watch, + const absl::flat_hash_set& resources, + const SubscriptionOptions& options); +@@ -239,6 +248,7 @@ private: + // this one is up to GrpcMux. + const LocalInfo::LocalInfo& local_info_; + Common::CallbackHandlePtr dynamic_update_callback_handle_; ++ GrpcMuxStreamEventTracker stream_event_tracker_; + CustomConfigValidatorsPtr config_validators_; + XdsConfigTrackerOptRef xds_config_tracker_; + XdsResourcesDelegateOptRef xds_resources_delegate_; +@@ -291,6 +301,13 @@ private: + class NullGrpcMuxImpl : public GrpcMux { + public: + void start() override {} ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + + ScopedResume pause(const std::string&) override { + return std::make_unique([]() {}); +@@ -313,6 +330,9 @@ public: + void requestOnDemandUpdate(const std::string&, const absl::flat_hash_set&) override { + ENVOY_BUG(false, "unexpected request for on demand update"); + } + + EdsResourcesCacheOptRef edsResourcesCache() override { return {}; } ++ ++private: ++ GrpcMuxStreamEventTracker stream_event_tracker_; + }; +diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc +--- a/test/common/config/grpc_subscription_impl_test.cc ++++ b/test/common/config/grpc_subscription_impl_test.cc +@@ -64,6 +64,42 @@ TEST_P(GrpcSubscriptionImplTest, RemoteStreamClose) { + timer_->invokeCallback(); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0, 0, "")); + } + ++TEST_P(GrpcSubscriptionImplTest, StreamEventCallbacks) { ++ uint32_t established = 0; ++ uint32_t closed = 0; ++ auto callback_handle = mux_->addStreamEventCallback([&](GrpcMuxStreamEvent event) { ++ if (event == GrpcMuxStreamEvent::Established) { ++ ++established; ++ } else { ++ ++closed; ++ } ++ }); ++ ++ EXPECT_FALSE(mux_->grpcStreamConnected()); ++ startSubscription({"cluster0", "cluster1"}); ++ EXPECT_TRUE(mux_->grpcStreamConnected()); ++ EXPECT_EQ(1, established); ++ EXPECT_EQ(0, closed); ++ ++ EXPECT_CALL(callbacks_, ++ onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)) ++ .Times(0); ++ EXPECT_CALL(*timer_, enableTimer(_, _)); ++ EXPECT_CALL(random_, random()); ++ onRemoteClose(); ++ EXPECT_FALSE(mux_->grpcStreamConnected()); ++ EXPECT_EQ(1, established); ++ EXPECT_EQ(1, closed); ++ ++ callback_handle.reset(); ++ EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); ++ expectSendMessage({"cluster0", "cluster1"}, "", true); ++ timer_->invokeCallback(); ++ EXPECT_TRUE(mux_->grpcStreamConnected()); ++ EXPECT_EQ(1, established); ++ EXPECT_EQ(1, closed); ++} ++ + // Validate that When the management server gets multiple requests for the same version, it can + // ignore later ones. This allows the nonce to be used. + TEST_P(GrpcSubscriptionImplTest, RepeatedNonce) { +diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h +--- a/test/mocks/config/mocks.h ++++ b/test/mocks/config/mocks.h +@@ -116,6 +116,10 @@ public: + MOCK_METHOD(ScopedResume, pause, (const std::string& type_url), (override)); + MOCK_METHOD(ScopedResume, pause, (const std::vector type_urls), (override)); + ++ MOCK_METHOD(Common::CallbackHandlePtr, addStreamEventCallback, ++ (GrpcMuxStreamEventCallback callback), (override)); ++ MOCK_METHOD(bool, grpcStreamConnected, (), (const, override)); ++ + MOCK_METHOD(void, addSubscription, + (const absl::flat_hash_set& resources, const std::string& type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats& stats, diff --git a/tests/bpf_metadata_integration_test.cc b/tests/bpf_metadata_integration_test.cc index 271ba7123..af7dda646 100644 --- a/tests/bpf_metadata_integration_test.cc +++ b/tests/bpf_metadata_integration_test.cc @@ -1,6 +1,9 @@ #include +#include +#include #include +#include #include #include "envoy/config/bootstrap/v3/bootstrap.pb.h" @@ -9,10 +12,11 @@ #include "envoy/config/listener/v3/listener.pb.h" #include "envoy/grpc/status.h" #include "envoy/http/codec.h" -#include "envoy/network/address.h" #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/common/assert.h" +#include "source/common/common/base_logger.h" +#include "source/common/common/logger.h" #include "source/common/protobuf/utility.h" #include "test/common/grpc/grpc_client_integration.h" @@ -56,6 +60,10 @@ const std::string policy2 = R"EOF( - remote_policies: [ 111 ] )EOF"; +const std::string invalid_policy = R"EOF( + endpoint_id: 8192 +)EOF"; + const std::string policy_host1 = R"EOF( policy: 111 host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] @@ -80,30 +88,64 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, cluster: cluster_0 )EOF") { skip_tag_extraction_rule_check_ = true; + use_lds_ = false; // skip built in listener setup, we do it explicitly via xDS + setUpstreamCount(1); // same as default + defer_listener_finalization_ = true; + + for (Logger::Logger& logger : Logger::Registry::loggers()) { + logger.setLevel(spdlog::level::trace); + } } ~BpfMetadataIntegrationTest() override { resetConnections(); } - void setGrpcServiceHelper(envoy::config::core::v3::GrpcService& grpc_service, - const std::string& cluster_name, - Network::Address::InstanceConstSharedPtr address) { - setGrpcService(grpc_service, cluster_name, address); + void setUpGrpcLds(bool use_ads = true) { + config_helper_.addConfigModifier( + [this, use_ads](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + listener_config_.Swap(bootstrap.mutable_static_resources()->mutable_listeners(0)); + listener_config_.set_name(listener_name_); + bootstrap.mutable_static_resources()->mutable_listeners()->Clear(); + + auto* lds_config_source = bootstrap.mutable_dynamic_resources()->mutable_lds_config(); + lds_config_source->Clear(); + lds_config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + if (use_ads) { + lds_config_source->mutable_ads(); + } else { + setGrpcApiConfigSource(*lds_config_source); + } + }); } - void setUpGrpcLds() { - config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - listener_config_.Swap(bootstrap.mutable_static_resources()->mutable_listeners(0)); - listener_config_.set_name(listener_name_); - bootstrap.mutable_static_resources()->mutable_listeners()->Clear(); + void setGrpcApiConfigSource(envoy::config::core::v3::ConfigSource& config_source, + envoy::config::core::v3::ApiConfigSource::ApiType api_type = + envoy::config::core::v3::ApiConfigSource::GRPC) { + config_source.set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + auto* api_config_source = config_source.mutable_api_config_source(); + api_config_source->set_set_node_on_first_message_only(true); + api_config_source->set_api_type(api_type); + api_config_source->set_transport_api_version(envoy::config::core::v3::ApiVersion::V3); + api_config_source->add_grpc_services()->mutable_envoy_grpc()->set_cluster_name( + "xds-grpc-cilium"); + } - auto* lds_config_source = bootstrap.mutable_dynamic_resources()->mutable_lds_config(); - lds_config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); - lds_config_source->mutable_ads(); - }); + void setBpfMetadataNpdsConfig(::cilium::BpfMetadata& bpf_config, bool use_ads, + envoy::config::core::v3::ApiConfigSource::ApiType api_type = + envoy::config::core::v3::ApiConfigSource::GRPC) { + auto* config_source = bpf_config.mutable_cilium_config_source(); + config_source->Clear(); + if (use_ads) { + config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + config_source->mutable_ads(); + } else { + setGrpcApiConfigSource(*config_source, api_type); + } } // Inject the cilium.bpf_metadata listener filter with config_source into the listener. - void addBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, bool) { + void addBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, bool use_ads, + envoy::config::core::v3::ApiConfigSource::ApiType api_type = + envoy::config::core::v3::ApiConfigSource::GRPC) { auto* listener_filter = listener.add_listener_filters(); listener_filter->set_name("cilium.bpf_metadata"); @@ -111,18 +153,12 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, bpf_config.set_is_ingress(false); bpf_config.set_use_nphds(true); - auto* config_source = bpf_config.mutable_cilium_config_source(); - config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); - config_source->mutable_ads(); + setBpfMetadataNpdsConfig(bpf_config, use_ads, api_type); listener_filter->mutable_typed_config()->PackFrom(bpf_config); } - void initialize() override { - use_lds_ = false; - setUpstreamCount(1); - defer_listener_finalization_ = true; - + void initializeAds() { config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { // Add the ADS gRPC cluster. auto* ads_cluster = bootstrap.mutable_static_resources()->add_clusters(); @@ -149,44 +185,107 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, BaseIntegrationTest::initialize(); } + void initializeSotw() { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* xds_cluster = bootstrap.mutable_static_resources()->add_clusters(); + xds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + xds_cluster->set_name("xds-grpc-cilium"); + ConfigHelper::setHttp2(*xds_cluster); + + auto* cds_config = bootstrap.mutable_dynamic_resources()->mutable_cds_config(); + setGrpcApiConfigSource(*cds_config); + }); + + // Must be last modifier — it removes static listeners. + setUpGrpcLds(/*use_ads=*/false); + + BaseIntegrationTest::initialize(); + } + void createUpstreams() override { BaseIntegrationTest::createUpstreams(); - // ADS upstream (fake_upstreams_[1]). + // ADS or SotW upstream (fake_upstreams_[1]). addFakeUpstream(Envoy::Http::CodecType::HTTP2); } - FakeUpstream& getAdsFakeUpstream() const { return *fake_upstreams_[1]; } + FakeUpstream& getFakeUpstream() const { return *fake_upstreams_[1]; } - void createAdsStream() { - AssertionResult result = - getAdsFakeUpstream().waitForHttpConnection(*dispatcher_, ads_connection_); + void createXdsConnection() { + if (xds_connection_ == nullptr) { + AssertionResult result = + getFakeUpstream().waitForHttpConnection(*dispatcher_, xds_connection_); + RELEASE_ASSERT(result, result.message()); + } + } + + void createXdsStream(FakeStreamPtr& stream) { + auto result = xds_connection_->waitForNewStream(*dispatcher_, stream); + RELEASE_ASSERT(result, result.message()); + result = stream->waitForHeadersComplete(); RELEASE_ASSERT(result, result.message()); - auto result2 = ads_connection_->waitForNewStream(*dispatcher_, ads_stream_); - RELEASE_ASSERT(result2, result2.message()); - ads_stream_->startGrpcStream(); + stream->startGrpcStream(); + } + + void createAdsStream() { + createXdsConnection(); + createXdsStream(ads_stream_); } - void sendCdsResponse(const std::string& version) { + void createSotWStreams(const std::string& response_version) { + createXdsConnection(); + + for (int i = 0; i < 4; i++) { + FakeStreamPtr stream; + createXdsStream(stream); + + envoy::service::discovery::v3::DiscoveryRequest request; + auto result = stream->waitForGrpcMessage(*dispatcher_, request); + RELEASE_ASSERT(result, result.message()); + if (request.type_url() == NetworkPolicyTypeUrl) { + ENVOY_LOG_MISC(info, "GOT NPDS STREAM"); + npds_stream_ = std::move(stream); + return; + } else if (request.type_url() == Envoy::Config::TestTypeUrl::get().Listener) { + ENVOY_LOG_MISC(info, "GOT LDS STREAM"); + lds_stream_ = std::move(stream); + sendLdsResponse(*lds_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, + response_version); + } else if (request.type_url() == Envoy::Config::TestTypeUrl::get().Cluster) { + ENVOY_LOG_MISC(info, "GOT CDS STREAM"); + cds_stream_ = std::move(stream); + sendCdsResponse(*cds_stream_, response_version); + } else if (request.type_url() == NetworkPolicyHostsTypeUrl) { + ENVOY_LOG_MISC(info, "GOT NPHDS STREAM"); + nphds_stream_ = std::move(stream); + sendNphdsResponse(*nphds_stream_, response_version); + } + } + + RELEASE_ASSERT(npds_stream_ != nullptr, "NPDS stream was not established"); + } + + void sendCdsResponse(FakeStream& stream, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(Envoy::Config::TestTypeUrl::get().Cluster); - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); } - void sendLdsResponse(const std::vector& listener_configs, + void sendLdsResponse(FakeStream& stream, + const std::vector& listener_configs, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(Envoy::Config::TestTypeUrl::get().Listener); for (const auto& listener_config : listener_configs) { response.add_resources()->PackFrom(listener_config); } - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); } - void sendLdsResponse(const std::vector& listener_configs, + void sendLdsResponse(FakeStream& stream, const std::vector& listener_configs, const std::string& version) { std::vector proto_configs; proto_configs.reserve(listener_configs.size()); @@ -194,26 +293,30 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, proto_configs.emplace_back( TestUtility::parseYaml(listener_blob)); } - sendLdsResponse(proto_configs, version); + sendLdsResponse(stream, proto_configs, version); } - void sendNpdsResponse(const std::string& version) { + void sendNpdsResponse(FakeStream& stream, const std::string& version, + const std::vector& policy_configs = {policy1, policy2}) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(NetworkPolicyTypeUrl); std::vector proto_configs; - proto_configs.emplace_back(TestUtility::parseYaml(policy1)); - proto_configs.emplace_back(TestUtility::parseYaml(policy2)); + proto_configs.reserve(policy_configs.size()); + for (const auto& policy_config : policy_configs) { + proto_configs.emplace_back(TestUtility::parseYaml(policy_config)); + } for (const auto& policy_config : proto_configs) { response.add_resources()->PackFrom(policy_config); } - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); } - void sendNphdsResponse(const std::string& version) { + void sendNphdsResponse(FakeStream& stream, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(NetworkPolicyHostsTypeUrl); std::vector proto_configs; proto_configs.emplace_back(TestUtility::parseYaml(policy_host1)); @@ -221,24 +324,43 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, for (const auto& policy_host_config : proto_configs) { response.add_resources()->PackFrom(policy_host_config); } - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); } void resetConnections() { - if (ads_connection_ != nullptr) { - AssertionResult result = ads_connection_->close(); + if (xds_connection_ != nullptr) { + AssertionResult result = xds_connection_->close(); RELEASE_ASSERT(result, result.message()); - result = ads_connection_->waitForDisconnect(); + result = xds_connection_->waitForDisconnect(); RELEASE_ASSERT(result, result.message()); - ads_connection_.reset(); + xds_connection_.reset(); } + ads_stream_.reset(); + lds_stream_.reset(); + cds_stream_.reset(); + npds_stream_.reset(); + nphds_stream_.reset(); + } + + uint64_t policyStreamGeneration() const { + return test_server_->gauge("cilium.policy.policy_stream_generation")->value(); + } + + uint64_t waitForPolicyStreamGenerationAfter(uint64_t previous_generation) { + test_server_->waitForGaugeGe("cilium.policy.policy_stream_generation", previous_generation + 1); + const uint64_t generation = policyStreamGeneration(); + EXPECT_GT(generation, previous_generation); + return generation; } envoy::config::listener::v3::Listener listener_config_; std::string listener_name_{"testing-listener-0"}; - FakeHttpConnectionPtr ads_connection_; + FakeHttpConnectionPtr xds_connection_; FakeStreamPtr ads_stream_; + FakeStreamPtr lds_stream_; + FakeStreamPtr cds_stream_; + FakeStreamPtr npds_stream_; + FakeStreamPtr nphds_stream_; }; INSTANTIATE_TEST_SUITE_P(IpVersionsAndGrpcTypes, BpfMetadataIntegrationTest, @@ -251,21 +373,84 @@ TEST_P(BpfMetadataIntegrationTest, BpfMetadataWithNpdsAndNpdhsViaAds) { EXPECT_TRUE(compareDiscoveryRequest( Config::TestTypeUrl::get().Cluster, "", {}, {}, {}, /*expect_node=*/true, Envoy::Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); - sendCdsResponse("1"); + sendCdsResponse(*ads_stream_, "1"); EXPECT_TRUE(compareDiscoveryRequest( Config::TestTypeUrl::get().Listener, "", {}, {}, {}, /*expect_node=*/false, Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); - sendLdsResponse({MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); + sendLdsResponse(*ads_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); }; - initialize(); + initializeAds(); test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().listenerManager().listeners().size(), 1); - sendNpdsResponse("1"); + sendNpdsResponse(*ads_stream_, "1"); test_server_->waitForCounterGe("cilium.policy.update_success", 1); - sendNphdsResponse("1"); + sendNphdsResponse(*ads_stream_, "1"); test_server_->waitForCounterGe("cilium.hostmap.update_success", 1); } +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedAdsGrpcStreams) { + on_server_init_function_ = [&]() { + createAdsStream(); + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/true); + EXPECT_TRUE(compareDiscoveryRequest( + Config::TestTypeUrl::get().Cluster, "", {}, {}, {}, + /*expect_node=*/true, Envoy::Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); + sendCdsResponse(*ads_stream_, "1"); + EXPECT_TRUE(compareDiscoveryRequest( + Config::TestTypeUrl::get().Listener, "", {}, {}, {}, /*expect_node=*/false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); + sendLdsResponse(*ads_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); + }; + initializeAds(); + + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + EXPECT_EQ(policyStreamGeneration(), 0); + + sendNpdsResponse(*ads_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(0); + + sendNpdsResponse(*ads_stream_, "2"); + test_server_->waitForCounterGe("cilium.policy.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + resetConnections(); + EXPECT_EQ(policyStreamGeneration(), first_generation); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedSotwGrpcStreams) { + on_server_init_function_ = [&]() { + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createSotWStreams("1"); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + EXPECT_EQ(policyStreamGeneration(), 0); + + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(0); + + sendNpdsResponse(*npds_stream_, "2"); + test_server_->waitForCounterGe("cilium.policy.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + resetConnections(); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + createSotWStreams("2"); + sendNpdsResponse(*npds_stream_, "3", {invalid_policy}); + // The invalid policy is rejected by the real gRPC subscription decoder/validator before + // NetworkPolicyMapImpl::onConfigUpdate() runs, so this increments NPDS subscription stats + // rather than cilium.policy.updates_rejected. + test_server_->waitForCounterGe("cilium.npds.update_rejected", 1); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + sendNpdsResponse(*npds_stream_, "4"); + test_server_->waitForCounterGe("cilium.policy.update_success", 3); + waitForPolicyStreamGenerationAfter(first_generation); +} + } // namespace } // namespace Envoy From 75409e7b710c66e3ab6343a4b19a2731b259c5d4 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Thu, 7 May 2026 11:36:22 +0200 Subject: [PATCH 3/4] api: Add Delta NPDS and NPHDS Add Delta rpc to the APIs so that we can run NPDS and NPHDS also via Delta xDS. Signed-off-by: Jarno Rajahalme --- cilium/api/npds.proto | 4 + cilium/api/nphds.proto | 4 + cilium/bpf_metadata.cc | 30 +- cilium/bpf_metadata.h | 4 +- cilium/host_map.cc | 194 +++++- cilium/host_map.h | 26 +- cilium/network_policy.cc | 786 +++++++++++++++++++++---- cilium/network_policy.h | 17 + go/cilium/api/npds.pb.go | 61 +- go/cilium/api/npds_grpc.pb.go | 34 +- go/cilium/api/nphds.pb.go | 23 +- go/cilium/api/nphds_grpc.pb.go | 32 + tests/bpf_metadata.cc | 12 +- tests/bpf_metadata.h | 8 +- tests/bpf_metadata_integration_test.cc | 360 ++++++++++- 15 files changed, 1381 insertions(+), 214 deletions(-) diff --git a/cilium/api/npds.proto b/cilium/api/npds.proto index 1a43692e9..2d28992ad 100644 --- a/cilium/api/npds.proto +++ b/cilium/api/npds.proto @@ -20,6 +20,10 @@ import "validate/validate.proto"; service NetworkPolicyDiscoveryService { option (envoy.annotations.resource).type = "cilium.NetworkPolicy"; + rpc DeltaNetworkPolicies(stream envoy.service.discovery.v3.DeltaDiscoveryRequest) + returns (stream envoy.service.discovery.v3.DeltaDiscoveryResponse) { + } + rpc StreamNetworkPolicies(stream envoy.service.discovery.v3.DiscoveryRequest) returns (stream envoy.service.discovery.v3.DiscoveryResponse) { } diff --git a/cilium/api/nphds.proto b/cilium/api/nphds.proto index 82a6dbb2a..bb5628510 100644 --- a/cilium/api/nphds.proto +++ b/cilium/api/nphds.proto @@ -26,6 +26,10 @@ service NetworkPolicyHostsDiscoveryService { body: "*" }; } + + rpc DeltaNetworkPolicyHosts(stream envoy.service.discovery.v3.DeltaDiscoveryRequest) + returns (stream envoy.service.discovery.v3.DeltaDiscoveryResponse) { + } } // The mapping of a network policy identifier to the IP addresses of all the diff --git a/cilium/bpf_metadata.cc b/cilium/bpf_metadata.cc index f386e94f4..cc20ecccd 100644 --- a/cilium/bpf_metadata.cc +++ b/cilium/bpf_metadata.cc @@ -235,14 +235,15 @@ Config::Config(const ::cilium::BpfMetadata& config, config.ipv6_source_address())); } if (config.use_nphds()) { - hosts_ = - context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(cilium_host_map), - [&context, config_source = config_source_] { - auto map = std::make_shared(context.serverFactoryContext()); - map->startSubscription(context.serverFactoryContext(), config_source); - return map; - }); + hosts_ = context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(cilium_host_map), + [&context, config_source = config_source_] { + auto map = std::make_shared(context.serverFactoryContext()); + map->startSubscription(context.serverFactoryContext(), config_source); + return map; + }); + // update desired config source on the map + hosts_->setConfigSource(config_source_); } // Note: all instances use the bpf root of the first filter with non-empty @@ -279,12 +280,13 @@ Config::Config(const ::cilium::BpfMetadata& config, // instances! // Only created if either ipcache_ or hosts_ map exists if (ipcache_ || hosts_) { - npmap_ = - context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), - [&context, config_source = config_source_] { - return std::make_shared(context, config_source, true); - }); + npmap_ = context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), + [&context, config_source = config_source_] { + return std::make_shared(context, config_source, true); + }); + // update desired config source on the map + npmap_->setConfigSource(config_source_); } } diff --git a/cilium/bpf_metadata.h b/cilium/bpf_metadata.h index 753578a5c..8ba3c9e5b 100644 --- a/cilium/bpf_metadata.h +++ b/cilium/bpf_metadata.h @@ -175,10 +175,10 @@ class Config : public Cilium::PolicyResolver, Random::RandomGenerator& random_; envoy::config::core::v3::ConfigSource config_source_; - std::shared_ptr npmap_; + std::shared_ptr npmap_; Cilium::CtMapSharedPtr ct_maps_; Cilium::IpCacheSharedPtr ipcache_; - std::shared_ptr hosts_; + std::shared_ptr hosts_; private: uint32_t resolveSourceIdentity(const Network::Address::Ip* sip, const Network::Address::Ip* dip, diff --git a/cilium/host_map.cc b/cilium/host_map.cc index ddf67da2f..6c0c912c4 100644 --- a/cilium/host_map.cc +++ b/cilium/host_map.cc @@ -4,15 +4,18 @@ #include #include +#include #include #include #include #include +#include #include #include #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/event/dispatcher.h" #include "envoy/server/factory_context.h" @@ -21,9 +24,11 @@ #include "envoy/thread_local/thread_local.h" #include "envoy/thread_local/thread_local_object.h" +#include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "source/common/common/macros.h" +#include "absl/container/flat_hash_set.h" #include "absl/numeric/int128.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -58,9 +63,18 @@ unsigned int checkPrefix(T addr, bool have_prefix, unsigned int plen, absl::stri } // namespace struct ThreadLocalHostMapInitializer : public PolicyHostMap::ThreadLocalHostMap { -protected: +public: friend class PolicyHostMap; // PolicyHostMap can insert(); + ThreadLocalHostMapInitializer() = default; + + explicit ThreadLocalHostMapInitializer(const PolicyHostMap::ThreadLocalHostMap* host_map) { + if (host_map != nullptr) { + static_cast(*this) = *host_map; + } + } + +protected: // find the map of the given prefix length, insert in the decreasing order if // it does not exist template @@ -159,6 +173,29 @@ struct ThreadLocalHostMapInitializer : public PolicyHostMap::ThreadLocalHostMap fmt::format("NetworkPolicyHosts: Invalid host entry \'{}\' for policy {}", host, policy)); } } + + template + void prunePolicyMapVec(MapVec& maps, const absl::flat_hash_set& nids) { + for (auto vec_it = maps.begin(); vec_it != maps.end();) { + auto& map = vec_it->second; + for (auto map_it = map.begin(); map_it != map.end();) { + auto it = map_it++; + if (nids.contains(it->second)) { + map.erase(it); + } + } + if (map.empty()) { + vec_it = maps.erase(vec_it); + } else { + ++vec_it; + } + } + } + + void remove(const absl::flat_hash_set& removed_nids) { + prunePolicyMapVec(ipv4_to_policy_, removed_nids); + prunePolicyMapVec(ipv6_to_policy_, removed_nids); + } }; uint64_t PolicyHostMap::instance_id_ = 0; @@ -196,22 +233,155 @@ PolicyHostMap::PolicyHostMap(Server::Configuration::CommonFactoryContext& contex } void PolicyHostMap::startSubscription(Server::Configuration::CommonFactoryContext& context, - const envoy::config::core::v3::ConfigSource& config_source) { - if (config_source.config_source_specifier_case() == envoy::config::core::v3::ConfigSource::kAds) { - auto ads_mux = context.xdsManager().adsMux(); - subscription_ = THROW_OR_RETURN_VALUE( - context.clusterManager().subscriptionFactory().subscriptionOverAdsGrpcMux( - ads_mux, config_source, NetworkPolicyHostsTypeUrl, *scope_, *this, - std::make_shared(), {}), - Config::SubscriptionPtr); - } else { - subscription_ = subscribe(NetworkPolicyHostsTypeUrl, config_source, context, *scope_, *this, - std::make_shared()); + const envoy::config::core::v3::ConfigSource& npds_config) { + context_ = &context; + desired_config_source_ = npds_config; + subscribe(); +} + +void PolicyHostMap::setConfigSource(const envoy::config::core::v3::ConfigSource& config_source) { + desired_config_source_ = config_source; + if (context_ != nullptr) { + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/false); + } +} + +bool PolicyHostMap::subscriptionUseDeltaXds() const { + if (!config_source_.has_api_config_source()) { + return false; } + const auto& api_type = config_source_.api_config_source().api_type(); + return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; +} +void PolicyHostMap::subscribe() { + ASSERT(context_ != nullptr); + subscription_connected_ = false; + config_source_ = desired_config_source_; + ++subscription_id_; + + auto on_stream_event = [weak_this = weak_from_this(), + id = subscription_id_](Config::GrpcMuxStreamEvent event) { + if (auto shared_this = weak_this.lock()) { + shared_this->onSubscriptionStreamEvent(id, event); + } + }; + + subscription_ = + Cilium::subscribe(NetworkPolicyHostsTypeUrl, config_source_, *context_, *scope_, *this, + std::make_shared(), std::move(on_stream_event)); subscription_->start({}); } +void PolicyHostMap::onSubscriptionStreamEvent(uint64_t subscription_id, + Config::GrpcMuxStreamEvent event) { + if (subscription_id != subscription_id_) { + return; + } + + switch (event) { + case Config::GrpcMuxStreamEvent::Established: + subscription_connected_ = true; + break; + case Config::GrpcMuxStreamEvent::Closed: + if (!subscription_connected_) { + return; + } + subscription_connected_ = false; + + if (context_ == nullptr) { + return; + } + + context_->mainThreadDispatcher().post( + [weak_this = weak_from_this(), subscription_id = subscription_id_]() { + if (auto shared_this = weak_this.lock()) { + if (subscription_id != shared_this->subscription_id_) { + return; + } + shared_this->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + } + }); + break; + } +} + +void PolicyHostMap::maybeRecreateSubscriptionInDesiredMode(bool transport_closed) { + if (subscription_ && (subscription_connected_ || !transport_closed)) { + if (subscription_connected_ && subscriptionUseDeltaXds()) { + return; + } + if (Protobuf::util::MessageDifferencer::Equals(config_source_, desired_config_source_)) { + return; + } + } + + subscribe(); +} + +absl::Status +PolicyHostMap::onConfigUpdate(const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + const bool is_new_stream = subscription_id_ != accepted_subscription_id_; + ENVOY_LOG( + debug, + "PolicyHostMap::onConfigUpdate({}), {} added_resources, {} removed_resources, version: {}, " + "subscription_id: {}, accepted_subscription_id: {}, is_new_stream: {}", + name_, added_resources.size(), removed_resources.size(), system_version_info, + subscription_id_, accepted_subscription_id_, is_new_stream); + + auto newmap = + std::make_shared(is_new_stream ? nullptr : getHostMap()); + + absl::flat_hash_set to_remove; + to_remove.reserve(added_resources.size() + removed_resources.size()); + + for (const auto& name : removed_resources) { + uint64_t nid = 0; + auto [ptr, ec] = std::from_chars(name.data(), name.data() + name.size(), nid); + if (ec != std::errc{} || ptr != name.data() + name.size()) { + throw EnvoyException(fmt::format("Invalid removed resource name '{}'", name)); + } + ENVOY_LOG(trace, + "Removing NetworkPolicyHosts for policy {} in delta onConfigUpdate() version {}", nid, + system_version_info); + to_remove.insert(nid); + } + for (const auto& resource : added_resources) { + const auto& config = dynamic_cast(resource.get().resource()); + to_remove.insert(config.policy()); + } + newmap->remove(to_remove); + + for (const auto& resource : added_resources) { + const auto& config = dynamic_cast(resource.get().resource()); + ENVOY_LOG(trace, + "Received NetworkPolicyHosts for policy {} in delta onConfigUpdate() version {}", + config.policy(), system_version_info); + newmap->insert(config); + } + + // Force 'this' to be not deleted for as long as the lambda stays + // alive. Note that generally capturing a shared pointer is + // dangerous as it may happen that there is a circular reference + // from 'this' to itself via the lambda capture, leading to 'this' + // never being released. It should happen in this case, though. + std::shared_ptr shared_this = shared_from_this(); + + // Assign the new map to all threads. + tls_->set([shared_this, newmap](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + UNREFERENCED_PARAMETER(shared_this); + ENVOY_LOG(trace, "PolicyHostMap: Assigning new map"); + return newmap; + }); + logmaps("delta onConfigUpdate"); + accepted_subscription_id_ = subscription_id_; + stats_.update_success_.inc(); + return absl::OkStatus(); +} + absl::Status PolicyHostMap::onConfigUpdate(const std::vector& resources, const std::string& version_info) { diff --git a/cilium/host_map.h b/cilium/host_map.h index af07eec7a..280369121 100644 --- a/cilium/host_map.h +++ b/cilium/host_map.h @@ -15,6 +15,7 @@ #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/network/address.h" #include "envoy/protobuf/message_validator.h" @@ -26,7 +27,6 @@ #include "envoy/thread_local/thread_local_object.h" #include "source/common/common/logger.h" -#include "source/common/common/macros.h" #include "source/common/network/utility.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" @@ -118,6 +118,8 @@ class PolicyHostMap : public Singleton::Instance, void startSubscription(Server::Configuration::CommonFactoryContext& context, const envoy::config::core::v3::ConfigSource& config_source); + void setConfigSource(const envoy::config::core::v3::ConfigSource& config_source); + // This is used for testing with a file-based subscription void startSubscription(std::unique_ptr&& subscription) { subscription_ = std::move(subscription); @@ -229,22 +231,30 @@ class PolicyHostMap : public Singleton::Instance, const std::string& version_info) override; absl::Status onConfigUpdate(const std::vector& added_resources, const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override { - // NOT IMPLEMENTED YET. - UNREFERENCED_PARAMETER(added_resources); - UNREFERENCED_PARAMETER(removed_resources); - UNREFERENCED_PARAMETER(system_version_info); - return absl::OkStatus(); - } + const std::string& system_version_info) override; void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) override; private: + bool subscriptionUseDeltaXds() const; + void subscribe(); + void onSubscriptionStreamEvent(uint64_t subscription_id, Config::GrpcMuxStreamEvent event); + void maybeRecreateSubscriptionInDesiredMode(bool transport_closed); + ThreadLocal::SlotPtr tls_; std::string name_; Stats::ScopeSharedPtr scope_; Stats::ScopeSharedPtr stats_scope_; std::unique_ptr subscription_; + Server::Configuration::CommonFactoryContext* context_{nullptr}; + // We need a separate desired_config_source_ as it may be set to a pessimistic value via explicit + // BpfMetadata config in CiliumEnvoyConfig CRD, and we should not change to a "worse" (e.g., SotW) + // ConfigSource if "better" (e.g., Delta) is already up-and-running (in config_source_). + envoy::config::core::v3::ConfigSource desired_config_source_; + envoy::config::core::v3::ConfigSource config_source_; + uint64_t subscription_id_{0}; + uint64_t accepted_subscription_id_{0}; + bool subscription_connected_{false}; static uint64_t instance_id_; PolicyHostsStats stats_; }; diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index 05a6b5456..631385b6d 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -40,7 +40,6 @@ #include "source/common/common/assert.h" #include "source/common/common/logger.h" -#include "source/common/common/macros.h" #include "source/common/common/matchers.h" #include "source/common/common/thread.h" #include "source/common/http/header_utility.h" @@ -61,6 +60,7 @@ #include "absl/strings/match.h" #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" +#include "absl/types/variant.h" #include "cilium/accesslog.h" #include "cilium/api/npds.pb.h" #include "cilium/grpc_subscription.h" @@ -122,12 +122,9 @@ template <> struct formatter { namespace Envoy { namespace Cilium { -class PolicyInstanceImpl; - -using PolicyMapSnapshot = - absl::flat_hash_map>; - // PolicyStreamState is shared by all policies created from one accepted NPDS stream generation. +// When the NPDS stream restarts, new policies get a fresh state object while old policies keep the +// old one until the old policy map has quiesced and been retired. class PolicyStreamState { public: explicit PolicyStreamState(uint64_t stream_generation) : stream_generation_(stream_generation) {} @@ -140,31 +137,183 @@ class PolicyStreamState { using PolicyStreamStateSharedPtr = std::shared_ptr; using PolicyStreamStateConstSharedPtr = std::shared_ptr; +class PolicyInstanceImpl; + +// Stable policy map, usually keyed with endpoint IP (IPv4 and IPv6). +using PolicyMapSnapshot = + absl::flat_hash_map>; + +// variant wrapper for supported resource map keys for delta policy updates +// Delta xDS refers to removed resources by resource name, so we must have a map to +// locate the policy/selector to be removed. +class ResourceKey { +public: + struct PolicyResourceEntry { + std::shared_ptr policy; + }; + + // reference to a policy entry by name + struct PolicyEndpointIpEntry { + std::string policy_name; + }; + + static ResourceKey policyResource(const std::shared_ptr& policy) { + return ResourceKey(PolicyResourceEntry{policy}); + } + + static ResourceKey policyEndpointIp(const std::string& name) { + return ResourceKey(PolicyEndpointIpEntry{name}); + } + + const PolicyResourceEntry* policyResourceEntry() const { + return absl::get_if(&value_); + } + + const PolicyEndpointIpEntry* policyEndpointIpEntry() const { + return absl::get_if(&value_); + } + + bool isPolicyEndpointIpEntry() const { + return absl::holds_alternative(value_); + } + +private: + explicit ResourceKey(const PolicyResourceEntry& value) : value_(value) {} + explicit ResourceKey(const PolicyEndpointIpEntry& value) : value_(value) {} + + absl::variant value_; +}; + +// Map of Delta xDS resources for name collision and duplicate name detection. +class ResourceMap : public absl::flat_hash_map { +public: + using absl::flat_hash_map::flat_hash_map; + + const ResourceKey* findEntry(const std::string& key) const { + auto it = find(key); + return it != end() ? &it->second : nullptr; + } +}; + +// ResourceMapOverlay lets delta updates stage tentative resource-map removals and insertions on top +// of the current ResourceMap while validation is still in progress. This preserves transactional +// behavior without copying the full map: failed updates can be discarded cheaply, and successful +// ones are applied to the real map only after the whole update has been accepted. +class ResourceMapOverlay { +public: + ResourceMapOverlay() = default; + explicit ResourceMapOverlay(const ResourceMap& base) : base_(&base) {} + + const ResourceKey* findEntry(const std::string& key) const { + auto upsert_it = upserts_.find(key); + if (upsert_it != upserts_.end()) { + return &upsert_it->second; + } + if (removed_.contains(key)) { + return nullptr; + } + return base_ ? base_->findEntry(key) : nullptr; + } + + std::string describeExistingResourceKey(const std::string& key) const; + + bool emplace(const std::string& key, ResourceKey&& value) { + if (findEntry(key)) { + return false; + } + removed_.erase(key); + return upserts_.emplace(key, std::move(value)).second; + } + + void erase(const std::string& key) { + upserts_.erase(key); + if (base_ && base_->find(key) != base_->end()) { + removed_.insert(key); + } else { + removed_.erase(key); + } + } + + bool erasePolicyResourceIfPresent(const std::string& resource_name) { + const auto* entry = findEntry(resource_name); + if (entry == nullptr) { + return false; + } + const auto* policy_entry = entry->policyResourceEntry(); + if (policy_entry == nullptr) { + return false; + } + erasePolicyResource(resource_name, policy_entry->policy); + return true; + } + + void erasePolicyResource(const std::string& resource_name, + const std::shared_ptr& policy); + + PolicyMapSnapshot toPolicyMapSnapshot() const; + + void applyTo(ResourceMap& map) && { + if (!upserts_.empty()) { + map.reserve(map.size() + upserts_.size()); + } + for (const auto& key : removed_) { + map.erase(key); + } + for (auto& [key, value] : upserts_) { + map.insert_or_assign(std::move(key), std::move(value)); + } + } + +private: + template void forEachEntry(Visitor visitor) const { + if (base_) { + for (const auto& [key, value] : *base_) { + if (!removed_.contains(key) && !upserts_.contains(key)) { + visitor(key, value); + } + } + } + for (const auto& [key, value] : upserts_) { + visitor(key, value); + } + } + + std::pair> + findPolicyByEndpointIp(const std::string& endpoint_ip) const; + + const ResourceMap* base_{}; + absl::flat_hash_set removed_; + absl::flat_hash_map upserts_; +}; + +// helper for validating resource names. +void validateResourceName(const std::string& resource_name, absl::string_view subject) { + if (resource_name.empty()) { + throw EnvoyException(fmt::format("{} must not be empty", subject)); + } + if (std::ranges::any_of(resource_name, [](unsigned char c) { return absl::ascii_isspace(c); })) { + throw EnvoyException( + fmt::format("{} '{}' must not contain whitespace", subject, resource_name)); + } +} + class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, public Envoy::Event::DispatcherThreadDeletable, public Logger::Loggable, public std::enable_shared_from_this { public: + friend class PortNetworkPolicyRule; NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, const envoy::config::core::v3::ConfigSource& config_source); ~NetworkPolicyMapImpl() override; - void startSubscription(const envoy::config::core::v3::ConfigSource& config_source) { - const uint64_t subscription_id = ++subscription_id_; - subscription_ = subscribe( - NetworkPolicyTypeUrl, config_source, context_, *npds_stats_scope_, *this, - std::make_shared(), - [weak_this = weak_from_this(), subscription_id](Config::GrpcMuxStreamEvent event) { - if (auto shared_this = weak_this.lock()) { - shared_this->onSubscriptionStreamEvent(subscription_id, event); - } - }); - } + void subscribe(); // This is used for testing with a file-based subscription - void startSubscription(std::unique_ptr&& subscription) { + void subscribe(std::unique_ptr&& subscription) { subscription_ = std::move(subscription); - subscription_connected_ = false; // XXX + config_source_ = desired_config_source_; + subscription_connected_ = false; } const envoy::config::core::v3::ConfigSource& getConfigSource() const { return config_source_; } @@ -174,13 +323,7 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, const std::string& version_info) override; absl::Status onConfigUpdate(const std::vector& added_resources, const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override { - // NOT IMPLEMENTED YET. - UNREFERENCED_PARAMETER(added_resources); - UNREFERENCED_PARAMETER(removed_resources); - UNREFERENCED_PARAMETER(system_version_info); - return absl::OkStatus(); - } + const std::string& system_version_info) override; void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) override; @@ -196,6 +339,29 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, uint64_t streamGeneration() const { return subscription_stream_generation_; } void resetStreamForTest() { subscription_stream_generation_++; } + bool useDeltaXds() const { + if (!desired_config_source_.has_api_config_source()) { + return false; + } + const auto& api_type = desired_config_source_.api_config_source().api_type(); + return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; + } + + bool subscriptionUseDeltaXds() const { + if (!config_source_.has_api_config_source()) { + return false; + } + const auto& api_type = config_source_.api_config_source().api_type(); + return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; + } + + void setConfigSource(const envoy::config::core::v3::ConfigSource& config_source) { + desired_config_source_ = config_source; + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/false); + } + // run the given function after all the threads have scheduled void runAfterAllThreads(std::function cb) const { // We can guarantee the callback 'cb' runs in the main thread after all worker threads have @@ -205,16 +371,16 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, context_.threadLocal().runOnAllWorkerThreads([]() {}, cb); } - std::string resourceName(const cilium::NetworkPolicy& config) { - return fmt::format("{}", config.endpoint_id()); - } - void reopenIpcache(); - std::shared_ptr - createOrReusePolicy(const cilium::NetworkPolicy& config, const PolicyMapSnapshot& old_policy_map); + void validatePolicy(const std::string& resource_name, const cilium::NetworkPolicy& config, + const std::string& version_info); - void installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, + std::pair, bool> + createOrReusePolicy(const std::string& resource_name, const cilium::NetworkPolicy& config, + bool is_new_stream, const ResourceMap& resource_map); + + void installNewPolicyMap(ResourceMapOverlay&& pending_resource_map, Init::ManagerImpl& version_init_manager, std::string&& version_name, const PolicyStreamStateSharedPtr& policy_stream_state); @@ -235,6 +401,12 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, } subscription_connected_ = false; + // Test code executes synchronously + if (subscription_factory_for_test_) { + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + return; + } + // The close callback runs on the subscription object's own stack, so defer any possible // recreation until after it unwinds to avoid destroying the current subscription // mid-callback. @@ -245,13 +417,36 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, if (subscription_id != shared_this->subscription_id_) { return; } - // shared_this->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + shared_this->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); } }); break; } } + void startSubscription() { + ASSERT(subscription_ != nullptr); + subscription_->start({}); + } + + void maybeRecreateSubscriptionInDesiredMode(bool transport_closed) { + // only ever skip subscribe if we have a subscription already, and it is already connected in + // delta or desired mode, or still connecting in desired mode. + if (subscription_ && (subscription_connected_ || !transport_closed)) { + if (subscription_connected_ && subscriptionUseDeltaXds()) { + // Keep delta on a connected subscription until transport closes. + return; + } + if (Protobuf::util::MessageDifferencer::Equals(config_source_, desired_config_source_)) { + // Let the current subscription keep going when it is already in the desired mode. + return; + } + } + + // Recreate the subscription in the latest desired mode. + subscribe(); + } + // Helpers for atomic swap of the policy map pointer. // // store() is only used for the initialization of the map during construction. @@ -291,19 +486,50 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, } const PolicyInstance* getPolicyInstanceImpl(const std::string& endpoint_policy_name) const; + PolicyInstanceConstSharedPtr + getPolicyInstanceSharedImpl(const std::string& endpoint_policy_name) const; void removeInitManager(); static uint64_t instance_id_; bool subscription_connected_{false}; uint64_t subscription_id_{0}; + bool subscription_should_start_{false}; + + void scheduleDeferredDeletion(const PolicyMapSnapshot* old_policy_map); + + void startManagedSubscriptionForTest() { + subscription_should_start_ = true; + subscribe(); + } + void setSubscriptionFactoryForTest(NetworkPolicyMap::SubscriptionFactoryForTest factory) { + subscription_factory_for_test_ = std::move(factory); + } + void onSubscriptionConnectedForTest() { + onSubscriptionStreamEvent(subscription_id_, Config::GrpcMuxStreamEvent::Established); + } + void onSubscriptionTransportCloseForTest() { + onSubscriptionStreamEvent(subscription_id_, Config::GrpcMuxStreamEvent::Closed); + } + bool subscriptionConnectedForTest() const { return subscription_connected_; } + Server::Configuration::ServerFactoryContext& context_; + std::atomic map_ptr_; - // Policies hold a shared per-stream state object. + // Policies hold a shared per-stream state object. A freshly installed stream stores its actual + // gRPC stream generation here. PolicyStreamStateSharedPtr policy_stream_state_{std::make_shared(0)}; + const ResourceMap empty_resource_map_; + ResourceMap resource_map_; Stats::ScopeSharedPtr npds_stats_scope_; Stats::ScopeSharedPtr policy_stats_scope_; + // We need a separate desired_config_source_ as it may be set to a pessimistic value via explicit + // BpfMetadata config in CiliumEnvoyConfig CRD, and we should not change to a "worse" (e.g., SotW) + // ConfigSource if "better" (e.g., Delta) is already up-and-running (in config_source_). + envoy::config::core::v3::ConfigSource desired_config_source_; + envoy::config::core::v3::ConfigSource config_source_; + // init target which starts gRPC subscription Init::TargetImpl init_target_; std::shared_ptr @@ -313,9 +539,9 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, // while parked, log and rotate it out before making it active again. std::unique_ptr parked_init_manager_; - envoy::config::core::v3::ConfigSource config_source_; std::unique_ptr subscription_; static uint64_t subscription_stream_generation_; + NetworkPolicyMap::SubscriptionFactoryForTest subscription_factory_for_test_; ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; @@ -742,13 +968,6 @@ class PortNetworkPolicyRule : public Logger::Loggable { } } - // inheritpassprecedence bumps up the precedence of a rule in a lower tier to the precedence - // range reserved right after the precedence of the given pass rule. - void inheritPassPrecedence(const PortNetworkPolicyRule& pass_rule) { - precedence_ -= pass_rule.tier_last_precedence_; - precedence_ += pass_rule.precedence_; - } - bool isRemoteWildcard() const { return remotes_.empty(); } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id) const { @@ -929,14 +1148,14 @@ class PortNetworkPolicyRule : public Logger::Loggable { bool hasHttpRules() const { return http_rules_ && !http_rules_->empty(); } - std::string name_; + const std::string name_; DownstreamTLSContextSharedPtr server_context_; UpstreamTLSContextSharedPtr client_context_; bool has_headermatches_{false}; const RuleVerdict verdict_; const uint16_t proxy_id_; - uint32_t precedence_; - uint32_t tier_last_precedence_; + const uint32_t precedence_; + const uint32_t tier_last_precedence_; uint32_t pass_index_; absl::btree_set remotes_; @@ -1025,7 +1244,6 @@ class PortNetworkPolicyRules : public Logger::Loggable { // we must add a default allow rule to retain the semantics of the combined rules. void appendRules(const std::vector& rules) { if (initialized_ && rules.empty() != rules_.empty()) { - // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (auto& rule : rules) { @@ -1292,7 +1510,7 @@ struct PortRangeCompare { // PolicySnapshot is keyed by port ranges, and contains a list of PortNetworkPolicyRules's // applicable to this range. A list is needed as rules may come from multiple sources (e.g., -// resulting from use of named ports and numbered ports in Cilium Network Policy at the same time). +// resulting from use of named ports and numbered ports in Cilium NetworkPolicy at the same time). class PolicySnapshot : public absl::btree_map { public: using absl::btree_map::btree_map; @@ -1642,6 +1860,7 @@ class PortNetworkPolicy : public Logger::Loggable { // methods. class PolicyInstanceImpl : public PolicyInstance { public: + friend class NetworkPolicyMapImpl; PolicyInstanceImpl(const NetworkPolicyMapImpl& parent, uint64_t hash, const cilium::NetworkPolicy& proto) : endpoint_id_(proto.endpoint_id()), hash_(hash), policy_proto_(proto), endpoint_ips_(proto), @@ -1701,16 +1920,164 @@ class PolicyInstanceImpl : public PolicyInstance { const PortNetworkPolicy egress_; }; +template std::string endpointIpsForLog(const EndpointIps& endpoint_ips) { + std::string formatted = "["; + bool first = true; + for (const auto& endpoint_ip : endpoint_ips) { + if (!first) { + formatted += ", "; + } + formatted += endpoint_ip; + first = false; + } + formatted += "]"; + return formatted; +} + +std::string describePolicyResourceForLog(const std::string& resource_name, + const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource description requires a policy"); + return fmt::format("policy resource '{}' (endpoint_id {}, endpoint_ips {})", resource_name, + policy->endpoint_id_, endpointIpsForLog(policy->policy_proto_.endpoint_ips())); +} + +std::string describePolicyResourceForLog(const std::string& resource_name, + const cilium::NetworkPolicy& policy) { + return fmt::format("policy resource '{}' (endpoint_id {}, endpoint_ips {})", resource_name, + policy.endpoint_id(), endpointIpsForLog(policy.endpoint_ips())); +} + +std::pair, bool> +NetworkPolicyMapImpl::createOrReusePolicy(const std::string& resource_name, + const cilium::NetworkPolicy& config, bool is_new_stream, + const ResourceMap& resource_map) { + const uint64_t new_hash = MessageUtil::hash(config); + if (!is_new_stream) { + const auto* old_entry = resource_map.findEntry(resource_name); + if (old_entry) { + const auto* old_policy_entry = old_entry->policyResourceEntry(); + if (old_policy_entry) { + const auto& old_policy = old_policy_entry->policy; + if (old_policy && old_policy->hash_ == new_hash && + Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config)) { + return {old_policy, true}; + } + } + } + } + return {std::make_shared(*this, new_hash, config), false}; +} + +template +void insertPolicyResource(ResourceMapOverlay& pending_resource_map, + const cilium::NetworkPolicy& config, const std::string& version_info, + bool is_new_stream, const std::string& resource_name, + const std::shared_ptr& policy, Logger logger) { + if (!pending_resource_map.emplace(resource_name, ResourceKey::policyResource(policy))) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " + "stream: incoming {} collides with existing {}", + version_info, resource_name, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(resource_name))); + } + for (const auto& endpoint_ip : config.endpoint_ips()) { + logger(endpoint_ip); + if (!pending_resource_map.emplace(endpoint_ip, ResourceKey::policyEndpointIp(resource_name))) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " + "stream: incoming {} collides with existing {}", + version_info, endpoint_ip, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(endpoint_ip))); + } + } +} + +std::pair> +ResourceMapOverlay::findPolicyByEndpointIp(const std::string& endpoint_ip) const { + const auto* key = findEntry(endpoint_ip); + if (key) { + const auto* ip_entry = key->policyEndpointIpEntry(); + if (ip_entry) { + key = findEntry(ip_entry->policy_name); + if (key) { + const auto* policy_entry = key->policyResourceEntry(); + if (policy_entry) { + return {ip_entry->policy_name, policy_entry->policy}; + } + } + } + } + static std::string empty_string; + return {empty_string, nullptr}; +} + +std::string ResourceMapOverlay::describeExistingResourceKey(const std::string& key) const { + const auto* entry = findEntry(key); + if (entry == nullptr) { + return fmt::format("resource key '{}'", key); + } + if (const auto* policy_entry = entry->policyResourceEntry(); + policy_entry != nullptr && policy_entry->policy != nullptr) { + return describePolicyResourceForLog(key, policy_entry->policy); + } + if (!entry->isPolicyEndpointIpEntry()) { + return fmt::format("resource key '{}'", key); + } + + auto [resource_name, policy] = findPolicyByEndpointIp(key); + if (policy == nullptr) { + return fmt::format("endpoint IP alias '{}'", key); + } + + if (!resource_name.empty()) { + return fmt::format("endpoint IP alias '{}' owned by {}", key, + describePolicyResourceForLog(resource_name, policy)); + } + + return fmt::format("endpoint IP alias '{}' owned by endpoint_id {} with endpoint_ips {}", key, + policy->endpoint_id_, endpointIpsForLog(policy->policy_proto_.endpoint_ips())); +} + +PolicyMapSnapshot ResourceMapOverlay::toPolicyMapSnapshot() const { + PolicyMapSnapshot policy_map; + forEachEntry([&](const std::string& resource_name, const ResourceKey& resource_key) { + const auto* policy_entry = resource_key.policyResourceEntry(); + if (policy_entry == nullptr || policy_entry->policy == nullptr) { + return; + } + for (const auto& endpoint_ip : policy_entry->policy->policy_proto_.endpoint_ips()) { + const bool inserted = policy_map.emplace(endpoint_ip, policy_entry->policy).second; + RELEASE_ASSERT( + inserted, + fmt::format("duplicate endpoint IP alias '{}' while creating policy map snapshot from {}", + endpoint_ip, + describePolicyResourceForLog(resource_name, policy_entry->policy))); + } + }); + return policy_map; +} + +void ResourceMapOverlay::erasePolicyResource( + const std::string& resource_name, const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource key must carry a policy"); + for (const auto& endpoint_ip : policy->policy_proto_.endpoint_ips()) { + erase(endpoint_ip); + } + erase(resource_name); +} + // Common base constructor // This is used directly for testing with a file-based subscription NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& context, const envoy::config::core::v3::ConfigSource& config_source, bool subscribe) : context_(context.serverFactoryContext()) { - impl_ = std::make_unique(context, config_source); + impl_ = std::make_shared(context, config_source); if (subscribe) { - impl_->startSubscription(config_source); + impl_->subscribe(); } } @@ -1736,12 +2103,40 @@ NetworkPolicyMap::~NetworkPolicyMap() { } bool NetworkPolicyMap::exists(const std::string& endpoint_policy_name) const { - return impl_->getPolicyInstanceImpl(endpoint_policy_name) != nullptr; + return impl_->getPolicyInstanceImpl(endpoint_policy_name); +} + +bool NetworkPolicyMap::useDeltaXds() const { return impl_->useDeltaXds(); } + +void NetworkPolicyMap::setConfigSource(const envoy::config::core::v3::ConfigSource& config_source) { + impl_->setConfigSource(config_source); } void NetworkPolicyMap::startSubscriptionForTest( std::unique_ptr&& subscription) { - impl_->startSubscription(std::move(subscription)); + impl_->subscribe(std::move(subscription)); +} + +void NetworkPolicyMap::startManagedSubscriptionForTest() { + impl_->startManagedSubscriptionForTest(); +} + +void NetworkPolicyMap::setSubscriptionFactoryForTest(SubscriptionFactoryForTest factory) { + impl_->setSubscriptionFactoryForTest(std::move(factory)); +} + +void NetworkPolicyMap::onSubscriptionConnectedForTest() { impl_->onSubscriptionConnectedForTest(); } + +void NetworkPolicyMap::onSubscriptionTransportCloseForTest() { + impl_->onSubscriptionTransportCloseForTest(); +} + +bool NetworkPolicyMap::subscriptionUseDeltaXdsForTest() const { + return impl_->subscriptionUseDeltaXds(); +} + +bool NetworkPolicyMap::subscriptionConnectedForTest() const { + return impl_->subscriptionConnectedForTest(); } Envoy::Config::SubscriptionCallbacks& NetworkPolicyMap::subscriptionCallbacksForTest() const { @@ -1750,15 +2145,25 @@ Envoy::Config::SubscriptionCallbacks& NetworkPolicyMap::subscriptionCallbacksFor PolicyStats& NetworkPolicyMap::statsForTest() const { return impl_->stats_; } +void NetworkPolicyMap::resetStreamForTest() { impl_->resetStreamForTest(); } + +PolicyInstanceConstSharedPtr +NetworkPolicyMap::getPolicyInstanceSharedForTest(const std::string& endpoint_policy_name) const { + return impl_->getPolicyInstanceSharedImpl(endpoint_policy_name); +} + NetworkPolicyMapImpl::NetworkPolicyMapImpl( Server::Configuration::FactoryContext& context, const envoy::config::core::v3::ConfigSource& config_source) : context_(context.serverFactoryContext()), map_ptr_(nullptr), npds_stats_scope_(context_.serverScope().createScope("cilium.npds.")), policy_stats_scope_(context_.serverScope().createScope("cilium.policy.")), - init_target_(fmt::format("Cilium Network Policy subscription start"), + desired_config_source_(config_source), config_source_(config_source), + init_target_(fmt::format("Cilium NetworkPolicy subscription start"), [this]() { - subscription_->start({}); + // Production subscription is allowed to start from now on. + subscription_should_start_ = true; + startSubscription(); // Allow listener init to continue before network policy updates are received init_target_.ready(); }), @@ -1767,7 +2172,6 @@ NetworkPolicyMapImpl::NetworkPolicyMapImpl( context_, *npds_stats_scope_, context_.messageValidationContext().dynamicValidationVisitor())), parked_init_manager_(std::make_unique("Cilium NetworkPolicyMap parked")), - config_source_(config_source), stats_{ALL_CILIUM_POLICY_COUNTERS(POOL_COUNTER(*policy_stats_scope_)) ALL_CILIUM_POLICY_GAUGES(POOL_GAUGE(*policy_stats_scope_))} { // Use listener init manager for subscription initialization @@ -1795,54 +2199,63 @@ NetworkPolicyMapImpl::~NetworkPolicyMapImpl() { delete load(); } +void NetworkPolicyMapImpl::subscribe() { + subscription_connected_ = false; + config_source_ = desired_config_source_; + ++subscription_id_; + + if (subscription_factory_for_test_) { + subscription_ = subscription_factory_for_test_(subscriptionUseDeltaXds()); + if (subscription_should_start_) { + startSubscription(); + } + return; + } + + auto on_stream_event = [weak_this = weak_from_this(), + id = subscription_id_](Config::GrpcMuxStreamEvent event) { + if (auto shared_this = weak_this.lock()) { + shared_this->onSubscriptionStreamEvent(id, event); + } + }; + + subscription_ = + Cilium::subscribe(NetworkPolicyTypeUrl, config_source_, context_, *npds_stats_scope_, *this, + std::make_shared(), std::move(on_stream_event)); + + if (subscription_should_start_) { + startSubscription(); + } +} + void NetworkPolicyMapImpl::reopenIpcache() { // Get ipcache singleton only if it was successfully created previously. // Cilium agent re-creates IP cache on restart, and the first accepted update on // the new stream must reopen it before workers enforce refreshed identities. IpCacheSharedPtr ipcache = IpCache::getIpCache(context_); - if (ipcache != nullptr) { + if (ipcache) { ENVOY_LOG(info, "Reopening ipcache on new stream"); ipcache->open(); } } -std::shared_ptr -NetworkPolicyMapImpl::createOrReusePolicy(const cilium::NetworkPolicy& config, - const PolicyMapSnapshot& old_policy_map) { - const uint64_t new_hash = MessageUtil::hash(config); - auto policy_it = old_policy_map.find(config.endpoint_ips()[0]); - if (policy_it != old_policy_map.cend()) { - const auto& old_policy = policy_it->second; - if (old_policy && old_policy->hash_ == new_hash && - Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config)) { - ENVOY_LOG(trace, "New policy is equal to old one, not updating."); - return old_policy; - } - } - - // May throw - return std::make_shared(*this, new_hash, config); -} - void NetworkPolicyMapImpl::installNewPolicyMap( - PolicyMapSnapshot&& new_policy_map, Init::ManagerImpl& version_init_manager, + ResourceMapOverlay&& pending_resource_map, Init::ManagerImpl& version_init_manager, std::string&& version_name, const PolicyStreamStateSharedPtr& policy_stream_state) { // Initialize SDS secrets. We do not wait for the completion. version_init_manager.initialize(Init::WatcherImpl(std::move(version_name), []() {})); - const auto* old_policy_map = exchange(new PolicyMapSnapshot(std::move(new_policy_map))); - - // Record stream state only after a successful install. The reserved value 0 - // keeps the initial accepted update on any stream source classified as new. policy_stream_state_ = policy_stream_state; stats_.policy_stream_generation_.set(policy_stream_state_->streamGeneration()); - // Delete the old map once all worker threads have entered their event queues, as this - // is proof that they no longer refer to the old map. - runAfterAllThreads([old_policy_map]() { - // Clean-up in the main thread after all threads have scheduled - delete old_policy_map; - }); + // Build the immutable lookup snapshot before applyTo() moves staged entries into resource_map_. + auto new_policy_map = pending_resource_map.toPolicyMapSnapshot(); + std::move(pending_resource_map).applyTo(resource_map_); + + // old version can be GC'd once all worker threads have quiesced. + // Delete the old map once all worker threads have entered their event queues, as this is proof + // that they no longer refer to the old map. + scheduleDeferredDeletion(exchange(new PolicyMapSnapshot(std::move(new_policy_map)))); } // removeInitManager must be called at the end of each policy update @@ -1885,6 +2298,16 @@ void NetworkPolicyMapImpl::removeInitManager() { transport_factory_context_->setInitManager(*parked_init_manager_); } +void NetworkPolicyMapImpl::scheduleDeferredDeletion(const PolicyMapSnapshot* old_policy_map) { + if (old_policy_map == nullptr) { + return; + } + runAfterAllThreads([old_policy_map]() { + // Clean-up in the main thread after all worker threads have scheduled. + delete old_policy_map; + }); +} + // onConfigUpdate parses the new network policy resources, allocates a new policy map and atomically // swaps it in place of the old policy map. Throws if any of the 'resources' can not be // parsed. Otherwise an OK status is returned without pausing NPDS gRPC stream, causing a new @@ -1892,53 +2315,58 @@ void NetworkPolicyMapImpl::removeInitManager() { absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::vector& resources, const std::string& version_info) { + stats_.updates_total_.inc(); subscription_connected_ = true; + auto stream_generation = streamGeneration(); // policy_stream_state_ gets updated on first successful update, // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); - ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}, stream {}", - instance_id_, resources.size(), version_info, stream_generation); - stats_.updates_total_.inc(); - // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, - // and that is also when the old stream terminates and a new one is created. - // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, - // so open it before the workers get a chance to enforce policy on the new IDs. - if (is_new_stream) { - ENVOY_LOG(info, "New NetworkPolicy stream"); + // Start from an empty resource map for a new stream + const auto& resource_map = is_new_stream ? empty_resource_map_ : resource_map_; - reopenIpcache(); - } + ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}, stream {}", + instance_id_, resources.size(), version_info, stream_generation); std::string version_name = fmt::format("NetworkPolicyMap version {}", version_info); Init::ManagerImpl version_init_manager(version_name); - // Set the init manager to use via the transport factory context - // Must be set before the new network policy is parsed, as the parsed - // SDS secrets will use this! - transport_factory_context_->setInitManager(version_init_manager); + transport_factory_context_->setInitManager(version_init_manager); // before parsing policies + ResourceMapOverlay pending_resource_map; const auto policy_stream_state = is_new_stream ? std::make_shared(stream_generation) : policy_stream_state_; - const auto* old_policy_map = load(); - PolicyMapSnapshot new_policy_map; + try { + // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, + // and that is also when the old stream terminates and a new one is created. + // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, + // so open it before the workers get a chance to enforce policy on the new IDs. + if (is_new_stream) { + ENVOY_LOG(info, "New NetworkPolicy stream {}", stream_generation); + reopenIpcache(); + } + for (const auto& resource : resources) { const auto& config = dynamic_cast(resource.get().resource()); - if (config.endpoint_ips().empty()) { - throw EnvoyException("Network Policy has no endpoint ips"); - } + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicy resource name"); + validatePolicy(resource_name, config, version_info); ENVOY_LOG(debug, - "Received Network Policy for endpoint {}, endpoint_ip {} in onConfigUpdate() " + "Received NetworkPolicy for endpoint {}, endpoint_ip {} in onConfigUpdate() " "version {}", config.endpoint_id(), config.endpoint_ips()[0], version_info); - auto policy = createOrReusePolicy(config, *old_policy_map); - for (const auto& endpoint_ip : config.endpoint_ips()) { - ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", endpoint_ip); - // new_policy_map is not exception safe, policy must be computed separately! - new_policy_map.emplace(endpoint_ip, policy); + auto [policy, reused] = + createOrReusePolicy(resource_name, config, is_new_stream, resource_map); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); } + insertPolicyResource( + pending_resource_map, config, version_info, is_new_stream, resource_name, policy, + [](absl::string_view ip) { + ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", ip); + }); } } catch (const EnvoyException& e) { ENVOY_LOG(warn, "NetworkPolicy update for version {} failed: {}", version_info, e.what()); @@ -1948,10 +2376,125 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( } stats_.update_success_.inc(); + removeInitManager(); + + // SotW update always rewrites the whole resource map + resource_map_.clear(); + + installNewPolicyMap(std::move(pending_resource_map), version_init_manager, + std::move(version_name), policy_stream_state); + return absl::OkStatus(); +} + +void NetworkPolicyMapImpl::validatePolicy(const std::string& resource_name, + const cilium::NetworkPolicy& config, + const std::string& version_info) { + if (config.endpoint_ips().empty()) { + throw EnvoyException("NetworkPolicyResource has no endpoint ips"); + } + ENVOY_LOG(debug, + "Received NetworkPolicy {} for endpoint {}, endpoint_ip {} in " + "onConfigUpdate() version {}", + resource_name, config.endpoint_id(), config.endpoint_ips()[0], version_info); + if (config.endpoint_id() == 0) { + throw EnvoyException("NetworkPolicyResource endpoint_id must be non-zero"); + } +} + +absl::Status NetworkPolicyMapImpl::onConfigUpdate( + const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + stats_.updates_total_.inc(); + subscription_connected_ = true; + + auto stream_generation = streamGeneration(); + // policy_stream_state_ gets updated on first successful update, + // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. + const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); + + // Start from an empty resource map for a new stream + const auto& resource_map = is_new_stream ? empty_resource_map_ : resource_map_; + + ENVOY_LOG(debug, + "NetworkPolicyMapImpl::onConfigUpdate({}), {} added resources, {} removed resources, " + "version: {}, stream {}", + instance_id_, added_resources.size(), removed_resources.size(), system_version_info, + stream_generation); - installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name), - policy_stream_state); + std::string version_name = fmt::format("NetworkPolicyMap version {}", system_version_info); + Init::ManagerImpl version_init_manager(version_name); + transport_factory_context_->setInitManager(version_init_manager); // before parsing policies + + ResourceMapOverlay pending_resource_map = + is_new_stream ? ResourceMapOverlay() : ResourceMapOverlay(resource_map); + const auto policy_stream_state = + is_new_stream ? std::make_shared(stream_generation) : policy_stream_state_; + + try { + // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, + // and that is also when the old stream terminates and a new one is created. + // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, + // so open it before the workers get a chance to enforce policy on the new IDs. + if (is_new_stream) { + ENVOY_LOG(info, "New NetworkPolicy stream {}", stream_generation); + reopenIpcache(); + } + + for (const auto& removed_resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource {}", removed_resource); + const auto* resource_entry = pending_resource_map.findEntry(removed_resource); + if (resource_entry == nullptr) { + continue; + } + if (pending_resource_map.erasePolicyResourceIfPresent(removed_resource)) { + continue; + } + throw EnvoyException( + fmt::format("NetworkPolicyResource removed resource '{}' is a policy endpoint IP alias, " + "not a resource name", + removed_resource)); + } + + for (const auto& resource : added_resources) { + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicy added resource name"); + pending_resource_map.erasePolicyResourceIfPresent(resource_name); + + const auto& config = dynamic_cast(resource.get().resource()); + validatePolicy(resource_name, config, system_version_info); + ENVOY_LOG(debug, + "Received NetworkPolicyResource {} for endpoint {}, endpoint_ip {} in " + "onConfigUpdate() version {}", + resource_name, config.endpoint_id(), config.endpoint_ips()[0], system_version_info); + + auto [policy, reused] = + createOrReusePolicy(resource_name, config, is_new_stream, resource_map); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + } + insertPolicyResource(pending_resource_map, config, system_version_info, is_new_stream, + resource_name, policy, [](absl::string_view ip) { + ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", ip); + }); + } + } catch (const EnvoyException& e) { + ENVOY_LOG(warn, "NetworkPolicy update for version {} failed: {}", system_version_info, + e.what()); + stats_.updates_rejected_.inc(); + removeInitManager(); + throw; // re-throw + } + + stats_.update_success_.inc(); + removeInitManager(); + // Do not carry over any resources from an old stream + if (is_new_stream) { + resource_map_.clear(); + } + installNewPolicyMap(std::move(pending_resource_map), version_init_manager, + std::move(version_name), policy_stream_state); return absl::OkStatus(); } @@ -1959,7 +2502,8 @@ void NetworkPolicyMapImpl::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailu const EnvoyException*) { // We need to allow server startup to continue, even if we have a bad // config. - ENVOY_LOG(debug, "Network Policy Update failed, keeping existing policy."); + ENVOY_LOG(debug, "NetworkPolicy update on stream {} failed, keeping existing policy.", + streamGeneration()); } ProtobufTypes::MessagePtr @@ -2084,6 +2628,16 @@ NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) cons return nullptr; } +PolicyInstanceConstSharedPtr +NetworkPolicyMapImpl::getPolicyInstanceSharedImpl(const std::string& endpoint_ip) const { + const auto* map = load(); + auto it = map->find(endpoint_ip); + if (it != map->end()) { + return it->second; + } + return nullptr; +} + // getPolicyInstance return a const reference to a policy in the policy map for the given // 'endpoint_ip'. If there is no policy for the given IP, a default policy is returned, // controlled by the 'default_allow_egress' argument as follows: @@ -2098,9 +2652,7 @@ NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) cons const PolicyInstance& NetworkPolicyMap::getPolicyInstance(const std::string& endpoint_ip, bool default_allow_egress) const { const auto* policy = impl_->getPolicyInstanceImpl(endpoint_ip); - return policy != nullptr ? *policy - : default_allow_egress ? getAllowAllEgressPolicy() - : getDenyAllPolicy(); + return policy ? *policy : default_allow_egress ? getAllowAllEgressPolicy() : getDenyAllPolicy(); } } // namespace Cilium diff --git a/cilium/network_policy.h b/cilium/network_policy.h index 03d376506..449a1fb6f 100644 --- a/cilium/network_policy.h +++ b/cilium/network_policy.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -196,9 +197,13 @@ class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable(bool use_delta_xds)>; + protected: friend class CiliumNetworkPolicyTest; friend struct TestHelper; PolicyStats& statsForTest() const; + void resetStreamForTest(); + PolicyInstanceConstSharedPtr + getPolicyInstanceSharedForTest(const std::string& endpoint_policy_name) const; void startSubscriptionForTest(std::unique_ptr&& subscription); + void startManagedSubscriptionForTest(); + void setSubscriptionFactoryForTest(SubscriptionFactoryForTest factory); + void onSubscriptionConnectedForTest(); + void onSubscriptionTransportCloseForTest(); + bool subscriptionUseDeltaXdsForTest() const; + bool subscriptionConnectedForTest() const; Envoy::Config::SubscriptionCallbacks& subscriptionCallbacksForTest() const; private: diff --git a/go/cilium/api/npds.pb.go b/go/cilium/api/npds.pb.go index 2b09745e8..5fad4195e 100644 --- a/go/cilium/api/npds.pb.go +++ b/go/cilium/api/npds.pb.go @@ -1266,8 +1266,9 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\\\n" + "\x19NetworkPoliciesConfigDump\x12?\n" + - "\x0fnetworkpolicies\x18\x01 \x03(\v2\x15.cilium.NetworkPolicyR\x0fnetworkpolicies2\xda\x02\n" + - "\x1dNetworkPolicyDiscoveryService\x12z\n" + + "\x0fnetworkpolicies\x18\x01 \x03(\v2\x15.cilium.NetworkPolicyR\x0fnetworkpolicies2\xe0\x03\n" + + "\x1dNetworkPolicyDiscoveryService\x12\x83\x01\n" + + "\x14DeltaNetworkPolicies\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x12z\n" + "\x15StreamNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\x9e\x01\n" + "\x14FetchNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\")\x82\xd3\xe4\x93\x02#:\x01*\"\x1e/v3/discovery:network_policies\x1a\x1c\x8a\xa4\x96\xf3\a\x16\n" + "\x14cilium.NetworkPolicyB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" @@ -1287,26 +1288,28 @@ func file_cilium_api_npds_proto_rawDescGZIP() []byte { var file_cilium_api_npds_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_cilium_api_npds_proto_msgTypes = make([]protoimpl.MessageInfo, 13) var file_cilium_api_npds_proto_goTypes = []any{ - (HeaderMatch_MatchAction)(0), // 0: cilium.HeaderMatch.MatchAction - (HeaderMatch_MismatchAction)(0), // 1: cilium.HeaderMatch.MismatchAction - (*NetworkPolicy)(nil), // 2: cilium.NetworkPolicy - (*PortNetworkPolicy)(nil), // 3: cilium.PortNetworkPolicy - (*TLSContext)(nil), // 4: cilium.TLSContext - (*PortNetworkPolicyRule)(nil), // 5: cilium.PortNetworkPolicyRule - (*HttpNetworkPolicyRules)(nil), // 6: cilium.HttpNetworkPolicyRules - (*HeaderMatch)(nil), // 7: cilium.HeaderMatch - (*HttpNetworkPolicyRule)(nil), // 8: cilium.HttpNetworkPolicyRule - (*KafkaNetworkPolicyRules)(nil), // 9: cilium.KafkaNetworkPolicyRules - (*KafkaNetworkPolicyRule)(nil), // 10: cilium.KafkaNetworkPolicyRule - (*L7NetworkPolicyRules)(nil), // 11: cilium.L7NetworkPolicyRules - (*L7NetworkPolicyRule)(nil), // 12: cilium.L7NetworkPolicyRule - (*NetworkPoliciesConfigDump)(nil), // 13: cilium.NetworkPoliciesConfigDump - nil, // 14: cilium.L7NetworkPolicyRule.RuleEntry - (v3.SocketAddress_Protocol)(0), // 15: envoy.config.core.v3.SocketAddress.Protocol - (*v31.HeaderMatcher)(nil), // 16: envoy.config.route.v3.HeaderMatcher - (*v32.MetadataMatcher)(nil), // 17: envoy.type.matcher.v3.MetadataMatcher - (*v33.DiscoveryRequest)(nil), // 18: envoy.service.discovery.v3.DiscoveryRequest - (*v33.DiscoveryResponse)(nil), // 19: envoy.service.discovery.v3.DiscoveryResponse + (HeaderMatch_MatchAction)(0), // 0: cilium.HeaderMatch.MatchAction + (HeaderMatch_MismatchAction)(0), // 1: cilium.HeaderMatch.MismatchAction + (*NetworkPolicy)(nil), // 2: cilium.NetworkPolicy + (*PortNetworkPolicy)(nil), // 3: cilium.PortNetworkPolicy + (*TLSContext)(nil), // 4: cilium.TLSContext + (*PortNetworkPolicyRule)(nil), // 5: cilium.PortNetworkPolicyRule + (*HttpNetworkPolicyRules)(nil), // 6: cilium.HttpNetworkPolicyRules + (*HeaderMatch)(nil), // 7: cilium.HeaderMatch + (*HttpNetworkPolicyRule)(nil), // 8: cilium.HttpNetworkPolicyRule + (*KafkaNetworkPolicyRules)(nil), // 9: cilium.KafkaNetworkPolicyRules + (*KafkaNetworkPolicyRule)(nil), // 10: cilium.KafkaNetworkPolicyRule + (*L7NetworkPolicyRules)(nil), // 11: cilium.L7NetworkPolicyRules + (*L7NetworkPolicyRule)(nil), // 12: cilium.L7NetworkPolicyRule + (*NetworkPoliciesConfigDump)(nil), // 13: cilium.NetworkPoliciesConfigDump + nil, // 14: cilium.L7NetworkPolicyRule.RuleEntry + (v3.SocketAddress_Protocol)(0), // 15: envoy.config.core.v3.SocketAddress.Protocol + (*v31.HeaderMatcher)(nil), // 16: envoy.config.route.v3.HeaderMatcher + (*v32.MetadataMatcher)(nil), // 17: envoy.type.matcher.v3.MetadataMatcher + (*v33.DeltaDiscoveryRequest)(nil), // 18: envoy.service.discovery.v3.DeltaDiscoveryRequest + (*v33.DiscoveryRequest)(nil), // 19: envoy.service.discovery.v3.DiscoveryRequest + (*v33.DeltaDiscoveryResponse)(nil), // 20: envoy.service.discovery.v3.DeltaDiscoveryResponse + (*v33.DiscoveryResponse)(nil), // 21: envoy.service.discovery.v3.DiscoveryResponse } var file_cilium_api_npds_proto_depIdxs = []int32{ 3, // 0: cilium.NetworkPolicy.ingress_per_port_policies:type_name -> cilium.PortNetworkPolicy @@ -1329,12 +1332,14 @@ var file_cilium_api_npds_proto_depIdxs = []int32{ 14, // 17: cilium.L7NetworkPolicyRule.rule:type_name -> cilium.L7NetworkPolicyRule.RuleEntry 17, // 18: cilium.L7NetworkPolicyRule.metadata_rule:type_name -> envoy.type.matcher.v3.MetadataMatcher 2, // 19: cilium.NetworkPoliciesConfigDump.networkpolicies:type_name -> cilium.NetworkPolicy - 18, // 20: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 18, // 21: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 19, // 22: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 19, // 23: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 22, // [22:24] is the sub-list for method output_type - 20, // [20:22] is the sub-list for method input_type + 18, // 20: cilium.NetworkPolicyDiscoveryService.DeltaNetworkPolicies:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 19, // 21: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 19, // 22: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 20, // 23: cilium.NetworkPolicyDiscoveryService.DeltaNetworkPolicies:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 21, // 24: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 21, // 25: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 23, // [23:26] is the sub-list for method output_type + 20, // [20:23] is the sub-list for method input_type 20, // [20:20] is the sub-list for extension type_name 20, // [20:20] is the sub-list for extension extendee 0, // [0:20] is the sub-list for field type_name diff --git a/go/cilium/api/npds_grpc.pb.go b/go/cilium/api/npds_grpc.pb.go index e2a51d9ab..98ee186bf 100644 --- a/go/cilium/api/npds_grpc.pb.go +++ b/go/cilium/api/npds_grpc.pb.go @@ -20,6 +20,7 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( + NetworkPolicyDiscoveryService_DeltaNetworkPolicies_FullMethodName = "/cilium.NetworkPolicyDiscoveryService/DeltaNetworkPolicies" NetworkPolicyDiscoveryService_StreamNetworkPolicies_FullMethodName = "/cilium.NetworkPolicyDiscoveryService/StreamNetworkPolicies" NetworkPolicyDiscoveryService_FetchNetworkPolicies_FullMethodName = "/cilium.NetworkPolicyDiscoveryService/FetchNetworkPolicies" ) @@ -30,6 +31,7 @@ const ( // // Each resource name is a network policy identifier. type NetworkPolicyDiscoveryServiceClient interface { + DeltaNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) StreamNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) FetchNetworkPolicies(ctx context.Context, in *v3.DiscoveryRequest, opts ...grpc.CallOption) (*v3.DiscoveryResponse, error) } @@ -42,9 +44,22 @@ func NewNetworkPolicyDiscoveryServiceClient(cc grpc.ClientConnInterface) Network return &networkPolicyDiscoveryServiceClient{cc} } +func (c *networkPolicyDiscoveryServiceClient) DeltaNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyDiscoveryService_ServiceDesc.Streams[0], NetworkPolicyDiscoveryService_DeltaNetworkPolicies_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyDiscoveryService_DeltaNetworkPoliciesClient = grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + func (c *networkPolicyDiscoveryServiceClient) StreamNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &NetworkPolicyDiscoveryService_ServiceDesc.Streams[0], NetworkPolicyDiscoveryService_StreamNetworkPolicies_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyDiscoveryService_ServiceDesc.Streams[1], NetworkPolicyDiscoveryService_StreamNetworkPolicies_FullMethodName, cOpts...) if err != nil { return nil, err } @@ -71,6 +86,7 @@ func (c *networkPolicyDiscoveryServiceClient) FetchNetworkPolicies(ctx context.C // // Each resource name is a network policy identifier. type NetworkPolicyDiscoveryServiceServer interface { + DeltaNetworkPolicies(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error StreamNetworkPolicies(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error FetchNetworkPolicies(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) mustEmbedUnimplementedNetworkPolicyDiscoveryServiceServer() @@ -83,6 +99,9 @@ type NetworkPolicyDiscoveryServiceServer interface { // pointer dereference when methods are called. type UnimplementedNetworkPolicyDiscoveryServiceServer struct{} +func (UnimplementedNetworkPolicyDiscoveryServiceServer) DeltaNetworkPolicies(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method DeltaNetworkPolicies not implemented") +} func (UnimplementedNetworkPolicyDiscoveryServiceServer) StreamNetworkPolicies(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error { return status.Error(codes.Unimplemented, "method StreamNetworkPolicies not implemented") } @@ -111,6 +130,13 @@ func RegisterNetworkPolicyDiscoveryServiceServer(s grpc.ServiceRegistrar, srv Ne s.RegisterService(&NetworkPolicyDiscoveryService_ServiceDesc, srv) } +func _NetworkPolicyDiscoveryService_DeltaNetworkPolicies_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyDiscoveryServiceServer).DeltaNetworkPolicies(&grpc.GenericServerStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyDiscoveryService_DeltaNetworkPoliciesServer = grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + func _NetworkPolicyDiscoveryService_StreamNetworkPolicies_Handler(srv interface{}, stream grpc.ServerStream) error { return srv.(NetworkPolicyDiscoveryServiceServer).StreamNetworkPolicies(&grpc.GenericServerStream[v3.DiscoveryRequest, v3.DiscoveryResponse]{ServerStream: stream}) } @@ -149,6 +175,12 @@ var NetworkPolicyDiscoveryService_ServiceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{ + { + StreamName: "DeltaNetworkPolicies", + Handler: _NetworkPolicyDiscoveryService_DeltaNetworkPolicies_Handler, + ServerStreams: true, + ClientStreams: true, + }, { StreamName: "StreamNetworkPolicies", Handler: _NetworkPolicyDiscoveryService_StreamNetworkPolicies_Handler, diff --git a/go/cilium/api/nphds.pb.go b/go/cilium/api/nphds.pb.go index f4a2b2dfe..c05898ac0 100644 --- a/go/cilium/api/nphds.pb.go +++ b/go/cilium/api/nphds.pb.go @@ -90,10 +90,11 @@ const file_cilium_api_nphds_proto_rawDesc = "" + "\x16cilium/api/nphds.proto\x12\x06cilium\x1a*envoy/service/discovery/v3/discovery.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"c\n" + "\x12NetworkPolicyHosts\x12\x16\n" + "\x06policy\x18\x01 \x01(\x04R\x06policy\x125\n" + - "\x0ehost_addresses\x18\x02 \x03(\tB\x0e\xfaB\v\x92\x01\b\x18\x01\"\x04r\x02\x10\x01R\rhostAddresses2\xee\x02\n" + + "\x0ehost_addresses\x18\x02 \x03(\tB\x0e\xfaB\v\x92\x01\b\x18\x01\"\x04r\x02\x10\x01R\rhostAddresses2\xf7\x03\n" + "\"NetworkPolicyHostsDiscoveryService\x12}\n" + "\x18StreamNetworkPolicyHosts\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\xa5\x01\n" + - "\x17FetchNetworkPolicyHosts\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"-\x82\xd3\xe4\x93\x02':\x01*\"\"/v2/discovery:network_policy_hosts\x1a!\x8a\xa4\x96\xf3\a\x1b\n" + + "\x17FetchNetworkPolicyHosts\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"-\x82\xd3\xe4\x93\x02':\x01*\"\"/v2/discovery:network_policy_hosts\x12\x86\x01\n" + + "\x17DeltaNetworkPolicyHosts\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x1a!\x8a\xa4\x96\xf3\a\x1b\n" + "\x19cilium.NetworkPolicyHostsB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( @@ -110,17 +111,21 @@ func file_cilium_api_nphds_proto_rawDescGZIP() []byte { var file_cilium_api_nphds_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_cilium_api_nphds_proto_goTypes = []any{ - (*NetworkPolicyHosts)(nil), // 0: cilium.NetworkPolicyHosts - (*v3.DiscoveryRequest)(nil), // 1: envoy.service.discovery.v3.DiscoveryRequest - (*v3.DiscoveryResponse)(nil), // 2: envoy.service.discovery.v3.DiscoveryResponse + (*NetworkPolicyHosts)(nil), // 0: cilium.NetworkPolicyHosts + (*v3.DiscoveryRequest)(nil), // 1: envoy.service.discovery.v3.DiscoveryRequest + (*v3.DeltaDiscoveryRequest)(nil), // 2: envoy.service.discovery.v3.DeltaDiscoveryRequest + (*v3.DiscoveryResponse)(nil), // 3: envoy.service.discovery.v3.DiscoveryResponse + (*v3.DeltaDiscoveryResponse)(nil), // 4: envoy.service.discovery.v3.DeltaDiscoveryResponse } var file_cilium_api_nphds_proto_depIdxs = []int32{ 1, // 0: cilium.NetworkPolicyHostsDiscoveryService.StreamNetworkPolicyHosts:input_type -> envoy.service.discovery.v3.DiscoveryRequest 1, // 1: cilium.NetworkPolicyHostsDiscoveryService.FetchNetworkPolicyHosts:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 2, // 2: cilium.NetworkPolicyHostsDiscoveryService.StreamNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 2, // 3: cilium.NetworkPolicyHostsDiscoveryService.FetchNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 2, // [2:4] is the sub-list for method output_type - 0, // [0:2] is the sub-list for method input_type + 2, // 2: cilium.NetworkPolicyHostsDiscoveryService.DeltaNetworkPolicyHosts:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 3, // 3: cilium.NetworkPolicyHostsDiscoveryService.StreamNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 3, // 4: cilium.NetworkPolicyHostsDiscoveryService.FetchNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 4, // 5: cilium.NetworkPolicyHostsDiscoveryService.DeltaNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 3, // [3:6] is the sub-list for method output_type + 0, // [0:3] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/go/cilium/api/nphds_grpc.pb.go b/go/cilium/api/nphds_grpc.pb.go index 144a3a815..63cd6c3fa 100644 --- a/go/cilium/api/nphds_grpc.pb.go +++ b/go/cilium/api/nphds_grpc.pb.go @@ -22,6 +22,7 @@ const _ = grpc.SupportPackageIsVersion9 const ( NetworkPolicyHostsDiscoveryService_StreamNetworkPolicyHosts_FullMethodName = "/cilium.NetworkPolicyHostsDiscoveryService/StreamNetworkPolicyHosts" NetworkPolicyHostsDiscoveryService_FetchNetworkPolicyHosts_FullMethodName = "/cilium.NetworkPolicyHostsDiscoveryService/FetchNetworkPolicyHosts" + NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_FullMethodName = "/cilium.NetworkPolicyHostsDiscoveryService/DeltaNetworkPolicyHosts" ) // NetworkPolicyHostsDiscoveryServiceClient is the client API for NetworkPolicyHostsDiscoveryService service. @@ -32,6 +33,7 @@ const ( type NetworkPolicyHostsDiscoveryServiceClient interface { StreamNetworkPolicyHosts(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) FetchNetworkPolicyHosts(ctx context.Context, in *v3.DiscoveryRequest, opts ...grpc.CallOption) (*v3.DiscoveryResponse, error) + DeltaNetworkPolicyHosts(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) } type networkPolicyHostsDiscoveryServiceClient struct { @@ -65,6 +67,19 @@ func (c *networkPolicyHostsDiscoveryServiceClient) FetchNetworkPolicyHosts(ctx c return out, nil } +func (c *networkPolicyHostsDiscoveryServiceClient) DeltaNetworkPolicyHosts(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyHostsDiscoveryService_ServiceDesc.Streams[1], NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHostsClient = grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + // NetworkPolicyHostsDiscoveryServiceServer is the server API for NetworkPolicyHostsDiscoveryService service. // All implementations must embed UnimplementedNetworkPolicyHostsDiscoveryServiceServer // for forward compatibility. @@ -73,6 +88,7 @@ func (c *networkPolicyHostsDiscoveryServiceClient) FetchNetworkPolicyHosts(ctx c type NetworkPolicyHostsDiscoveryServiceServer interface { StreamNetworkPolicyHosts(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error FetchNetworkPolicyHosts(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) + DeltaNetworkPolicyHosts(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error mustEmbedUnimplementedNetworkPolicyHostsDiscoveryServiceServer() } @@ -89,6 +105,9 @@ func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) StreamNetworkPolicy func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) FetchNetworkPolicyHosts(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { return nil, status.Error(codes.Unimplemented, "method FetchNetworkPolicyHosts not implemented") } +func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) DeltaNetworkPolicyHosts(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method DeltaNetworkPolicyHosts not implemented") +} func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) mustEmbedUnimplementedNetworkPolicyHostsDiscoveryServiceServer() { } func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) testEmbeddedByValue() {} @@ -136,6 +155,13 @@ func _NetworkPolicyHostsDiscoveryService_FetchNetworkPolicyHosts_Handler(srv int return interceptor(ctx, in, info, handler) } +func _NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyHostsDiscoveryServiceServer).DeltaNetworkPolicyHosts(&grpc.GenericServerStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHostsServer = grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + // NetworkPolicyHostsDiscoveryService_ServiceDesc is the grpc.ServiceDesc for NetworkPolicyHostsDiscoveryService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -155,6 +181,12 @@ var NetworkPolicyHostsDiscoveryService_ServiceDesc = grpc.ServiceDesc{ ServerStreams: true, ClientStreams: true, }, + { + StreamName: "DeltaNetworkPolicyHosts", + Handler: _NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_Handler, + ServerStreams: true, + ClientStreams: true, + }, }, Metadata: "cilium/api/nphds.proto", } diff --git a/tests/bpf_metadata.cc b/tests/bpf_metadata.cc index 84bd5e87a..c7eda8ada 100644 --- a/tests/bpf_metadata.cc +++ b/tests/bpf_metadata.cc @@ -39,10 +39,10 @@ namespace Envoy { std::string host_map_config = "version_info: \"0\""; -std::shared_ptr hostmap{nullptr}; // Keep reference to singleton +std::shared_ptr hostmap{nullptr}; // Keep reference to singleton Network::Address::InstanceConstSharedPtr original_dst_address; -std::shared_ptr npmap{nullptr}; // Keep reference to singleton +std::shared_ptr npmap{nullptr}; // Keep reference to singleton std::string policy_config = "version_info: \"0\""; std::string policy_path = ""; @@ -51,10 +51,10 @@ std::vector> sds_configs{}; namespace Cilium { -std::shared_ptr +std::shared_ptr TestHelper::createHostMap(const std::string& config, Server::Configuration::ListenerFactoryContext& context) { - return context.serverFactoryContext().singletonManager().getTyped( + return context.serverFactoryContext().singletonManager().getTyped( "cilium_host_map_singleton", [&config, &context] { std::string path = TestEnvironment::writeStringToFileForTest("host_map.yaml", config); ENVOY_LOG_MISC(debug, "Loading Cilium Host Map from file \'{}\' instead of using gRPC", @@ -75,11 +75,11 @@ TestHelper::createHostMap(const std::string& config, }); } -std::shared_ptr +std::shared_ptr TestHelper::createPolicyMap(const std::string& config, const std::vector>& secret_configs, Server::Configuration::FactoryContext& context) { - return context.serverFactoryContext().singletonManager().getTyped( + return context.serverFactoryContext().singletonManager().getTyped( "cilium_network_policy_singleton", [&config, &secret_configs, &context] { if (!secret_configs.empty()) { for (const auto& sds_pair : secret_configs) { diff --git a/tests/bpf_metadata.h b/tests/bpf_metadata.h index ecdf80e22..995a805ee 100644 --- a/tests/bpf_metadata.h +++ b/tests/bpf_metadata.h @@ -18,10 +18,10 @@ namespace Envoy { extern std::string host_map_config; -extern std::shared_ptr hostmap; +extern std::shared_ptr hostmap; extern Network::Address::InstanceConstSharedPtr original_dst_address; -extern std::shared_ptr npmap; +extern std::shared_ptr npmap; extern std::string policy_config; extern std::string policy_path; @@ -30,9 +30,9 @@ extern std::vector> sds_configs; namespace Cilium { struct TestHelper { - static std::shared_ptr + static std::shared_ptr createHostMap(const std::string& config, Server::Configuration::ListenerFactoryContext&); - static std::shared_ptr + static std::shared_ptr createPolicyMap(const std::string& config, const std::vector>& secret_configs, Server::Configuration::FactoryContext&); diff --git a/tests/bpf_metadata_integration_test.cc b/tests/bpf_metadata_integration_test.cc index af7dda646..67dd74184 100644 --- a/tests/bpf_metadata_integration_test.cc +++ b/tests/bpf_metadata_integration_test.cc @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -15,8 +16,8 @@ #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/common/assert.h" -#include "source/common/common/base_logger.h" #include "source/common/common/logger.h" +#include "source/common/network/utility.h" #include "source/common/protobuf/utility.h" #include "test/common/grpc/grpc_client_integration.h" @@ -26,18 +27,28 @@ #include "test/test_common/resources.h" #include "test/test_common/utility.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" #include "cilium/api/bpf_metadata.pb.h" #include "cilium/api/npds.pb.h" #include "cilium/api/nphds.pb.h" +#include "cilium/host_map.h" +#include "cilium/network_policy.h" +#include "cilium/policy_id.h" #include "gtest/gtest.h" namespace Envoy { namespace { const std::string NetworkPolicyTypeUrl = "type.googleapis.com/cilium.NetworkPolicy"; - const std::string NetworkPolicyHostsTypeUrl = "type.googleapis.com/cilium.NetworkPolicyHosts"; +struct NetworkPolicyResourceConfig { + std::string name; + std::string version; + std::string yaml; +}; + const std::string policy1 = R"EOF( endpoint_ips: - '10.1.1.1' @@ -74,6 +85,51 @@ const std::string policy_host2 = R"EOF( host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] )EOF"; +const NetworkPolicyResourceConfig policy_host1_resource = {"111", "1", R"EOF( + policy: 111 + host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy_host2_resource = {"222", "1", R"EOF( + policy: 222 + host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy_host1_new_stream_resource = {"111", "2", R"EOF( + policy: 111 + host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy42_resource = {"policy-42", "1", R"EOF( + endpoint_ips: + - '10.1.2.3' + endpoint_id: 42 + egress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 222 ] +)EOF"}; + +const NetworkPolicyResourceConfig policy43_resource = {"policy-43", "1", R"EOF( + endpoint_ips: + - '10.2.3.4' + endpoint_id: 43 + ingress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 111 ] +)EOF"}; + +const NetworkPolicyResourceConfig policy42_new_stream_resource = {"policy-42", "2", R"EOF( + endpoint_ips: + - '10.1.2.3' + endpoint_id: 42 + egress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 222 ] +)EOF"}; + class BpfMetadataIntegrationTest : public BaseIntegrationTest, public Grpc::GrpcClientIntegrationParamTest { public: @@ -92,9 +148,11 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, setUpstreamCount(1); // same as default defer_listener_finalization_ = true; +#if 0 for (Logger::Logger& logger : Logger::Registry::loggers()) { logger.setLevel(spdlog::level::trace); } +#endif } ~BpfMetadataIntegrationTest() override { resetConnections(); } @@ -158,6 +216,23 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, listener_filter->mutable_typed_config()->PackFrom(bpf_config); } + void updateBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, + envoy::config::core::v3::ApiConfigSource::ApiType api_type) { + for (auto& listener_filter : *listener.mutable_listener_filters()) { + if (listener_filter.name() != "cilium.bpf_metadata") { + continue; + } + + ::cilium::BpfMetadata bpf_config; + RELEASE_ASSERT(listener_filter.typed_config().UnpackTo(&bpf_config), + "failed to unpack cilium.bpf_metadata listener filter"); + setBpfMetadataNpdsConfig(bpf_config, /*use_ads=*/false, api_type); + listener_filter.mutable_typed_config()->PackFrom(bpf_config); + return; + } + RELEASE_ASSERT(false, "cilium.bpf_metadata listener filter not found"); + } + void initializeAds() { config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { // Add the ADS gRPC cluster. @@ -231,37 +306,64 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, createXdsStream(ads_stream_); } - void createSotWStreams(const std::string& response_version) { + void createStreamsUntil(const std::string& response_version, absl::string_view type_url, + bool expect_delta = false) { createXdsConnection(); for (int i = 0; i < 4; i++) { FakeStreamPtr stream; createXdsStream(stream); - envoy::service::discovery::v3::DiscoveryRequest request; - auto result = stream->waitForGrpcMessage(*dispatcher_, request); - RELEASE_ASSERT(result, result.message()); - if (request.type_url() == NetworkPolicyTypeUrl) { + std::string request_type_url; + const bool is_delta = + stream->headers().getPathValue().find("/Delta") != absl::string_view::npos; + if (is_delta) { + envoy::service::discovery::v3::DeltaDiscoveryRequest request; + auto result = stream->waitForGrpcMessage(*dispatcher_, request); + RELEASE_ASSERT(result, result.message()); + request_type_url = request.type_url(); + } else { + envoy::service::discovery::v3::DiscoveryRequest request; + auto result = stream->waitForGrpcMessage(*dispatcher_, request); + RELEASE_ASSERT(result, result.message()); + request_type_url = request.type_url(); + } + + if (request_type_url == NetworkPolicyTypeUrl) { ENVOY_LOG_MISC(info, "GOT NPDS STREAM"); npds_stream_ = std::move(stream); - return; - } else if (request.type_url() == Envoy::Config::TestTypeUrl::get().Listener) { + if (type_url == NetworkPolicyTypeUrl && is_delta == expect_delta) { + return; + } + } else if (request_type_url == Envoy::Config::TestTypeUrl::get().Listener) { ENVOY_LOG_MISC(info, "GOT LDS STREAM"); lds_stream_ = std::move(stream); sendLdsResponse(*lds_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, response_version); - } else if (request.type_url() == Envoy::Config::TestTypeUrl::get().Cluster) { + } else if (request_type_url == Envoy::Config::TestTypeUrl::get().Cluster) { ENVOY_LOG_MISC(info, "GOT CDS STREAM"); cds_stream_ = std::move(stream); sendCdsResponse(*cds_stream_, response_version); - } else if (request.type_url() == NetworkPolicyHostsTypeUrl) { + } else if (request_type_url == NetworkPolicyHostsTypeUrl) { ENVOY_LOG_MISC(info, "GOT NPHDS STREAM"); nphds_stream_ = std::move(stream); - sendNphdsResponse(*nphds_stream_, response_version); + if (!is_delta) { + sendNphdsResponse(*nphds_stream_, response_version); + } + if (type_url == NetworkPolicyHostsTypeUrl && is_delta == expect_delta) { + return; + } } } - RELEASE_ASSERT(npds_stream_ != nullptr, "NPDS stream was not established"); + RELEASE_ASSERT(false, fmt::format("{} stream was not established", type_url)); + } + + void createSotWStreams(const std::string& response_version) { + createStreamsUntil(response_version, NetworkPolicyTypeUrl); + if (nphds_stream_ == nullptr) { + createStreamsUntil(response_version, NetworkPolicyHostsTypeUrl); + } } void sendCdsResponse(FakeStream& stream, const std::string& version) { @@ -327,6 +429,75 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, stream.sendGrpcMessage(response); } + void sendNpdsDeltaResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs, + const std::vector& removed_resources = {}) { + envoy::service::discovery::v3::DeltaDiscoveryResponse response; + response.set_system_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource* resource = response.add_resources(); + resource->set_name(resource_config.name); + resource->set_version(resource_config.version); + resource->mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + } + for (const auto& removed_resource : removed_resources) { + response.add_removed_resources(removed_resource); + } + stream.sendGrpcMessage(response); + } + + void sendNphdsDeltaResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs, + const std::vector& removed_resources = {}) { + envoy::service::discovery::v3::DeltaDiscoveryResponse response; + response.set_system_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyHostsTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource* resource = response.add_resources(); + resource->set_name(resource_config.name); + resource->set_version(resource_config.version); + resource->mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + } + for (const auto& removed_resource : removed_resources) { + response.add_removed_resources(removed_resource); + } + stream.sendGrpcMessage(response); + } + + AssertionResult compareNpdsAck() { + return compareDeltaDiscoveryRequest(NetworkPolicyTypeUrl, {}, {}, npds_stream_.get(), + Grpc::Status::WellKnownGrpcStatus::Ok, "", + /*expect_node=*/false); + } + + AssertionResult compareNphdsAck() { + return compareDeltaDiscoveryRequest(NetworkPolicyHostsTypeUrl, {}, {}, nphds_stream_.get(), + Grpc::Status::WellKnownGrpcStatus::Ok, "", + /*expect_node=*/false); + } + + void retireStream(FakeStreamPtr& stream) { + if (stream != nullptr) { + retired_streams_.push_back(std::move(stream)); + } + } + + void resetGrpcStream(FakeStreamPtr& stream) { + stream->encodeResetStream(); + AssertionResult result = stream->waitForReset(*dispatcher_); + RELEASE_ASSERT(result, result.message()); + retireStream(stream); + } + + void resetNpdsStream() { resetGrpcStream(npds_stream_); } + + void resetNphdsStream() { resetGrpcStream(nphds_stream_); } + void resetConnections() { if (xds_connection_ != nullptr) { AssertionResult result = xds_connection_->close(); @@ -353,6 +524,41 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, return generation; } + std::shared_ptr networkPolicyMap() const { + auto map = test_server_->server().singletonManager().getTyped( + "cilium_network_policy_singleton"); + RELEASE_ASSERT(map != nullptr, "Cilium NetworkPolicyMap singleton was not created"); + return map; + } + + uint64_t resolveHostPolicyId(const std::string& address) const { + auto parsed_address = Network::Utility::parseInternetAddressNoThrow(address); + RELEASE_ASSERT(parsed_address != nullptr, + fmt::format("failed to parse host address {}", address)); + auto map = test_server_->server().singletonManager().getTyped( + "cilium_host_map_singleton"); + RELEASE_ASSERT(map != nullptr, "Cilium PolicyHostMap singleton was not created"); + + absl::Mutex lock; + bool resolved = false; + uint64_t policy_id = Cilium::ID::UNKNOWN; + + // PolicyHostMap lookups must run on an Envoy thread that has thread-local storage registered. + // The gtest/integration thread calling this helper is not such a thread, so dereferencing the + // TLS-backed host map directly here is unsafe. Posting the resolve to the server dispatcher + // keeps the lookup on a valid Envoy TLS thread while still letting the test read the result. + test_server_->server().dispatcher().post([&, map, parsed_address]() { + policy_id = map->resolve(parsed_address->ip()); + lock.Lock(); + resolved = true; + lock.Unlock(); + }); + + lock.LockWhen(absl::Condition(&resolved)); + lock.Unlock(); + return policy_id; + } + envoy::config::listener::v3::Listener listener_config_; std::string listener_name_{"testing-listener-0"}; FakeHttpConnectionPtr xds_connection_; @@ -361,6 +567,7 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, FakeStreamPtr cds_stream_; FakeStreamPtr npds_stream_; FakeStreamPtr nphds_stream_; + std::vector retired_streams_; }; INSTANTIATE_TEST_SUITE_P(IpVersionsAndGrpcTypes, BpfMetadataIntegrationTest, @@ -452,5 +659,132 @@ TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedSotwGrpcS waitForPolicyStreamGenerationAfter(first_generation); } +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNpdsStreams) { + on_server_init_function_ = [&]() { + // Step 1: establish the initial SotW LDS, CDS, NPHDS, and NPDS streams. + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createSotWStreams("1"); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + + // Step 2: accept a real SotW NPDS response so the starting mode has installed policy. + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t sotw_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.1.1")); + EXPECT_TRUE(policy_map->exists("10.2.2.2")); + + // Step 3: update the BpfMetadata config source; this is evidence that Delta NPDS is available. + updateBpfMetadataListenerFilter(listener_config_, + envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); + sendLdsResponse(*lds_stream_, {listener_config_}, "2"); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 2); + + // Step 4: observe the immediate switch to Delta NPDS without advancing accepted policy state. + createStreamsUntil("2", NetworkPolicyTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + + // Step 5: accept the first Delta NPDS update and retire the prior SotW policy resources. + sendNpdsDeltaResponse(*npds_stream_, "1", {policy42_resource, policy43_resource}); + EXPECT_TRUE(compareNpdsAck()); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(sotw_generation); + EXPECT_FALSE(policy_map->exists("10.1.1.1")); + EXPECT_FALSE(policy_map->exists("10.2.2.2")); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 6: accept a same-stream Delta update; stream generation and omitted resources stay put. + sendNpdsDeltaResponse(*npds_stream_, "2", {policy42_new_stream_resource}); + EXPECT_TRUE(compareNpdsAck()); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 7: reset the Delta NPDS stream. + resetNpdsStream(); + + // Step 8: open the replacement Delta stream; reconnect alone must not advance policy state. + createStreamsUntil("3", NetworkPolicyTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 9: accept the first update on the new stream and retire resources from the old stream. + sendNpdsDeltaResponse(*npds_stream_, "3", {policy42_new_stream_resource}); + waitForPolicyStreamGenerationAfter(first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_FALSE(policy_map->exists("10.2.3.4")); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNphdsStreams) { + on_server_init_function_ = [&]() { + // Step 1: establish the initial SotW LDS, CDS, NPHDS, and NPDS streams. + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createSotWStreams("1"); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 2: accept a real SotW NPDS response so the starting mode has installed policy. + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t sotw_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.1.1")); + EXPECT_TRUE(policy_map->exists("10.2.2.2")); + + // Step 3: update the BpfMetadata config source; this is evidence that Delta NPHDS is available. + updateBpfMetadataListenerFilter(listener_config_, + envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); + sendLdsResponse(*lds_stream_, {listener_config_}, "2"); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 2); + + // Step 4: observe the immediate switch to Delta NPHDS without advancing accepted policy state. + createStreamsUntil("2", NetworkPolicyHostsTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 5: accept the first Delta NPHDS update. This should not change policy stream generation. + sendNphdsDeltaResponse(*nphds_stream_, "1", {policy_host1_resource, policy_host2_resource}); + EXPECT_TRUE(compareNphdsAck()); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 6: accept a same-stream Delta update; omitted resources stay present on the same stream. + sendNphdsDeltaResponse(*nphds_stream_, "2", {policy_host1_new_stream_resource}); + EXPECT_TRUE(compareNphdsAck()); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 3); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 7: reset the Delta NPHDS stream. + resetNphdsStream(); + + // Step 8: open the replacement Delta stream; reconnect alone must not advance policy state. + createStreamsUntil("3", NetworkPolicyHostsTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 9: accept the first update on the new stream and retire resources from the old stream. + sendNphdsDeltaResponse(*nphds_stream_, "3", {policy_host1_new_stream_resource}); + EXPECT_TRUE(compareNphdsAck()); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 4); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), Cilium::ID::UNKNOWN); +} + } // namespace } // namespace Envoy From c1f458a4d44a0c03b1562fe36b8731a8c36e6416 Mon Sep 17 00:00:00 2001 From: Jarno Rajahalme Date: Mon, 4 May 2026 21:35:48 +0200 Subject: [PATCH 4/4] policy: Add NetworkPolicyResourcesDiscoveryService Add new cilium/versioned.h generic container for transactional selector updates. Add a new NetworkPolicyResourceDiscoveryService that implements delta (and SotW) updates for policies and selectors, and where policies refer to selectors by their resource name. NPRDS adds a top-level oneof wrapper that wraps either a Selector or a NetworkPolicy. NetworkPolicy definition is shared with NPDS, but PortNetworkPolicyRule adds a new selectors field that is only used with NPRDS. Add 'policy_type' enum to BpfMetadata config to control whether NPDS (default) or NPRDS is used. Store the latest desired ConfigSource in the policy map and use it for: - initial policy map subscription - re-subscription when connection under current subscription is terminated - a healthy network policy stream is not disrupted, unless the desired config is for delta xDS and the current one is not This means that we switch to NPRDS (Delta) mode eagerly when we have evidence that the agent is capable, but we switch to NPDS (SotW) mode only when xDS stream transport had failed to connect or closes. This should work for Cilium Agent upgrades and downgrades, as the agent expresses the desired mode, and listens for both. Clear the resource map on a first update on a new stream. This fixes NACK cases where further updates on the stream would have IP collisions with resources that were kept from the previous stream. Signed-off-by: Jarno Rajahalme --- cilium/BUILD | 12 + cilium/api/bpf_metadata.proto | 7 + cilium/api/npds.proto | 37 + cilium/bpf_metadata.cc | 10 +- cilium/grpc_subscription.cc | 1 + cilium/network_policy.cc | 949 ++++++++-- cilium/network_policy.h | 42 +- cilium/versioned.h | 531 ++++++ go/cilium/api/bpf_metadata.pb.go | 84 +- go/cilium/api/bpf_metadata.pb.validate.go | 2 + go/cilium/api/npds.pb.go | 362 +++- go/cilium/api/npds.pb.validate.go | 288 +++ go/cilium/api/npds_grpc.pb.go | 135 ++ tests/BUILD | 9 + tests/bpf_metadata.cc | 4 +- tests/bpf_metadata_integration_test.cc | 279 ++- tests/cilium_network_policy_test.cc | 1948 ++++++++++++++++++++- tests/versioned_test.cc | 1165 ++++++++++++ 18 files changed, 5535 insertions(+), 330 deletions(-) create mode 100644 cilium/versioned.h create mode 100644 tests/versioned_test.cc diff --git a/cilium/BUILD b/cilium/BUILD index ca9e1269e..1af2056e8 100644 --- a/cilium/BUILD +++ b/cilium/BUILD @@ -28,6 +28,17 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "versioned_lib", + hdrs = ["versioned.h"], + repository = "@envoy", + deps = [ + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@envoy//source/common/common:assert_lib", + ], +) + envoy_cc_library( name = "network_policy_lib", srcs = [ @@ -45,6 +56,7 @@ envoy_cc_library( "//cilium:conntrack_lib", "//cilium:grpc_subscription_lib", "//cilium:ipcache_lib", + "//cilium:versioned_lib", "//cilium/api:npds_cc_proto", "@envoy//envoy/singleton:manager_interface", "@envoy//source/common/common:logger_lib", diff --git a/cilium/api/bpf_metadata.proto b/cilium/api/bpf_metadata.proto index 4f29224b9..ee857812a 100644 --- a/cilium/api/bpf_metadata.proto +++ b/cilium/api/bpf_metadata.proto @@ -87,4 +87,11 @@ message BpfMetadata { // Configuration for the source of Cilium xDS updates. // Used for all cilium-specific xDS protocol, e.g., NPHDS, NPDS, and Secrets (SDS) therein. envoy.config.core.v3.ConfigSource cilium_config_source = 16; + + // Policy type to use + enum PolicyType { + NPDS = 0; // Legacy NPDS (default) + NPRDS = 1; // New NetworkPolicyResource (NPRDS) + } + bool policy_type = 17; } diff --git a/cilium/api/npds.proto b/cilium/api/npds.proto index 2d28992ad..f66937635 100644 --- a/cilium/api/npds.proto +++ b/cilium/api/npds.proto @@ -17,6 +17,7 @@ import "validate/validate.proto"; // [#protodoc-title: Network policy management and NPDS] // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. service NetworkPolicyDiscoveryService { option (envoy.annotations.resource).type = "cilium.NetworkPolicy"; @@ -37,6 +38,36 @@ service NetworkPolicyDiscoveryService { } } +// Policy and selector resource names are exact-match identifiers in NPRDS. +service NetworkPolicyResourceDiscoveryService { + option (envoy.annotations.resource).type = "cilium.NetworkPolicyResource"; + + rpc StreamNetworkPolicyResources(stream envoy.service.discovery.v3.DiscoveryRequest) + returns (stream envoy.service.discovery.v3.DiscoveryResponse) { + } + + rpc DeltaNetworkPolicyResources(stream envoy.service.discovery.v3.DeltaDiscoveryRequest) + returns (stream envoy.service.discovery.v3.DeltaDiscoveryResponse) { + } +} + +// An NPRDS resource that carries either an endpoint policy or a shared selector. +message NetworkPolicyResource { + oneof resource { + NetworkPolicy policy = 1; + Selector selector = 2; + } +} + +// A shared set of remote identities referenced by selector resource name. +// Unlike the old state-of-the-world remote identity lists, an empty selector +// matches nothing. +message Selector { + // The set of numeric remote security IDs selected by this selector. + // If empty, this selector selects no remote identities. + repeated uint32 remote_identities = 1; +} + // A network policy that is enforced by a filter on the network flows to/from // associated hosts. message NetworkPolicy { @@ -157,6 +188,12 @@ message PortNetworkPolicyRule { // Optional. If not specified, any remote host is matched by this predicate. repeated uint32 remote_policies = 7; + // Optional selector resource names that can be resolved to shared remote + // policy sets in delta NPDS. + // Selector references are matched by exact selector resource name. + // Optional. If not specified, any remote host is matched by this predicate. + repeated string selectors = 11; + // Optional downstream TLS context. If present, the incoming connection must // be a TLS connection. TLSContext downstream_tls_context = 3; diff --git a/cilium/bpf_metadata.cc b/cilium/bpf_metadata.cc index cc20ecccd..405a24400 100644 --- a/cilium/bpf_metadata.cc +++ b/cilium/bpf_metadata.cc @@ -280,13 +280,15 @@ Config::Config(const ::cilium::BpfMetadata& config, // instances! // Only created if either ipcache_ or hosts_ map exists if (ipcache_ || hosts_) { + bool use_nprds = config.policy_type() == cilium::BpfMetadata::NPRDS; npmap_ = context.serverFactoryContext().singletonManager().getTyped( SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), - [&context, config_source = config_source_] { - return std::make_shared(context, config_source, true); + [&context, use_nprds, config_source = config_source_] { + return std::make_shared(context, use_nprds, config_source, + true); }); - // update desired config source on the map - npmap_->setConfigSource(config_source_); + // update desired config on the map + npmap_->setConfig(use_nprds, config_source_); } } diff --git a/cilium/grpc_subscription.cc b/cilium/grpc_subscription.cc index f16932007..57013a4de 100644 --- a/cilium/grpc_subscription.cc +++ b/cilium/grpc_subscription.cc @@ -87,6 +87,7 @@ TypeUrlToServiceMap* buildTypeUrlToServiceMap() { // https://www.mail-archive.com/protobuf@googlegroups.com/msg04540.html. for (absl::string_view name : { "cilium.NetworkPolicyDiscoveryService", + "cilium.NetworkPolicyResourceDiscoveryService", "cilium.NetworkPolicyHostsDiscoveryService", }) { const auto* service_desc = diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index 631385b6d..bdfececfe 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -7,11 +7,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include @@ -66,11 +68,23 @@ #include "cilium/grpc_subscription.h" #include "cilium/ipcache.h" #include "cilium/secret_watcher.h" +#include "cilium/versioned.h" namespace { static constexpr absl::string_view NetworkPolicyTypeUrl = "type.googleapis.com/cilium.NetworkPolicy"; +static constexpr absl::string_view NetworkPolicyResourceTypeUrl = + "type.googleapis.com/cilium.NetworkPolicyResource"; + +bool configSourceIsDelta(const envoy::config::core::v3::ConfigSource& config_source) { + if (!config_source.has_api_config_source()) { + return false; + } + const auto& api_type = config_source.api_config_source().api_type(); + return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; +} } // namespace @@ -123,20 +137,62 @@ namespace Envoy { namespace Cilium { // PolicyStreamState is shared by all policies created from one accepted NPDS stream generation. -// When the NPDS stream restarts, new policies get a fresh state object while old policies keep the -// old one until the old policy map has quiesced and been retired. +// Same-stream selector-only updates publish a newer selector version into this object so existing +// policies follow immediately. When the NPDS stream restarts, new policies get a fresh state +// object while old policies keep the old one until the old policy map has quiesced and been +// retired. This allows the new stream to reuse selector resource names so that the xDS server +// need not keep selector resource names in stable storage accross restarts. class PolicyStreamState { public: - explicit PolicyStreamState(uint64_t stream_generation) : stream_generation_(stream_generation) {} + explicit PolicyStreamState(uint64_t stream_generation, SelectorVersion version = versionMin) + : stream_generation_(stream_generation), version_(version) {} uint64_t streamGeneration() const { return stream_generation_; } + SelectorVersion version() const { return version_.load(std::memory_order_acquire); } + + void publishVersion(SelectorVersion version) { + version_.store(version, std::memory_order_release); + } + private: const uint64_t stream_generation_; + std::atomic version_; }; using PolicyStreamStateSharedPtr = std::shared_ptr; using PolicyStreamStateConstSharedPtr = std::shared_ptr; +// A specific version of a selector used in a policy. Each update yields a new instance. +class SelectorInstance : public VersionedNode, + public absl::flat_hash_set {}; + +// Read-only series of specific selector insteances. +class NamedSelectorReadable : public VersionedReadable { +public: + explicit NamedSelectorReadable(const std::string& name) : name_(name) {} + + const std::string& name() const { return name_; } + +private: + std::string name_; +}; + +// Stable handle on a read-only selector for accessing specific versions of the selector. +using SelectorHandle = std::shared_ptr; + +// Writable series of selector versions for main-thread updates +class NamedSelectorValue : public VersionedValue { +public: + explicit NamedSelectorValue(const std::string& name) + : VersionedValue(name) {} +}; + +// Map of named selectors, keyed with xDS resource name +class SelectorMap : public VersionedMap { +public: + using VersionedMap::VersionedMap; +}; + class PolicyInstanceImpl; // Stable policy map, usually keyed with endpoint IP (IPv4 and IPv6). @@ -157,6 +213,10 @@ class ResourceKey { std::string policy_name; }; + struct SelectorResourceEntry { + SelectorHandle handle; + }; + static ResourceKey policyResource(const std::shared_ptr& policy) { return ResourceKey(PolicyResourceEntry{policy}); } @@ -165,10 +225,18 @@ class ResourceKey { return ResourceKey(PolicyEndpointIpEntry{name}); } + static ResourceKey selectorResource(const SelectorHandle& handle) { + return ResourceKey(SelectorResourceEntry{handle}); + } + const PolicyResourceEntry* policyResourceEntry() const { return absl::get_if(&value_); } + const SelectorResourceEntry* selectorResourceEntry() const { + return absl::get_if(&value_); + } + const PolicyEndpointIpEntry* policyEndpointIpEntry() const { return absl::get_if(&value_); } @@ -180,8 +248,9 @@ class ResourceKey { private: explicit ResourceKey(const PolicyResourceEntry& value) : value_(value) {} explicit ResourceKey(const PolicyEndpointIpEntry& value) : value_(value) {} + explicit ResourceKey(const SelectorResourceEntry& value) : value_(value) {} - absl::variant value_; + absl::variant value_; }; // Map of Delta xDS resources for name collision and duplicate name detection. @@ -217,6 +286,20 @@ class ResourceMapOverlay { std::string describeExistingResourceKey(const std::string& key) const; + SelectorHandle getSelectorHandleOrThrow(const std::string& selector) const { + const auto* entry = findEntry(selector); + if (entry == nullptr) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource rule references missing selector resource '{}'", selector)); + } + const auto* selector_entry = entry->selectorResourceEntry(); + if (selector_entry == nullptr || selector_entry->handle == nullptr) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource rule references non-selector resource '{}'", selector)); + } + return selector_entry->handle; + } + bool emplace(const std::string& key, ResourceKey&& value) { if (findEntry(key)) { return false; @@ -225,6 +308,18 @@ class ResourceMapOverlay { return upserts_.emplace(key, std::move(value)).second; } + bool insertOrUpdateSelectorResource(const std::string& key, const SelectorHandle& handle) { + if (upserts_.contains(key)) { + return false; + } + const auto* existing = findEntry(key); + if (existing != nullptr && existing->selectorResourceEntry() == nullptr) { + return false; + } + removed_.erase(key); + return upserts_.emplace(key, ResourceKey::selectorResource(handle)).second; + } + void erase(const std::string& key) { upserts_.erase(key); if (base_ && base_->find(key) != base_->end()) { @@ -303,7 +398,7 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, public std::enable_shared_from_this { public: friend class PortNetworkPolicyRule; - NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, + NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source); ~NetworkPolicyMapImpl() override; @@ -312,11 +407,13 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, // This is used for testing with a file-based subscription void subscribe(std::unique_ptr&& subscription) { subscription_ = std::move(subscription); - config_source_ = desired_config_source_; + current_config_ = desired_config_; subscription_connected_ = false; } - const envoy::config::core::v3::ConfigSource& getConfigSource() const { return config_source_; } + const envoy::config::core::v3::ConfigSource& getConfigSource() const { + return current_config_.config_source_; + } // Config::SubscriptionCallbacks absl::Status onConfigUpdate(const std::vector& resources, @@ -339,26 +436,8 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, uint64_t streamGeneration() const { return subscription_stream_generation_; } void resetStreamForTest() { subscription_stream_generation_++; } - bool useDeltaXds() const { - if (!desired_config_source_.has_api_config_source()) { - return false; - } - const auto& api_type = desired_config_source_.api_config_source().api_type(); - return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || - api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; - } - - bool subscriptionUseDeltaXds() const { - if (!config_source_.has_api_config_source()) { - return false; - } - const auto& api_type = config_source_.api_config_source().api_type(); - return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || - api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; - } - - void setConfigSource(const envoy::config::core::v3::ConfigSource& config_source) { - desired_config_source_ = config_source; + void setConfig(bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source) { + desired_config_ = XdsConfig{use_nprds, config_source}; maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/false); } @@ -378,7 +457,14 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, std::pair, bool> createOrReusePolicy(const std::string& resource_name, const cilium::NetworkPolicy& config, - bool is_new_stream, const ResourceMap& resource_map); + const PolicyStreamStateConstSharedPtr& policy_stream_state, + const ResourceMap& resource_map, bool is_new_stream, + const ResourceMapOverlay* pending_resource_map) const; + + std::pair createOrReuseSelector(const std::string& resource_name, + const cilium::Selector& config, + uint64_t update_version, + const ResourceMap& resource_map); void installNewPolicyMap(ResourceMapOverlay&& pending_resource_map, Init::ManagerImpl& version_init_manager, std::string&& version_name, @@ -431,13 +517,16 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, void maybeRecreateSubscriptionInDesiredMode(bool transport_closed) { // only ever skip subscribe if we have a subscription already, and it is already connected in - // delta or desired mode, or still connecting in desired mode. + // Delta NPRDS or desired mode, or still connecting in desired mode. if (subscription_ && (subscription_connected_ || !transport_closed)) { - if (subscription_connected_ && subscriptionUseDeltaXds()) { - // Keep delta on a connected subscription until transport closes. + if (subscription_connected_ && configSourceIsDelta(current_config_.config_source_) && + current_config_.use_nprds_) { + // Keep Delta NPRDS on a connected subscription until transport closes. return; } - if (Protobuf::util::MessageDifferencer::Equals(config_source_, desired_config_source_)) { + if (current_config_.use_nprds_ == desired_config_.use_nprds_ && + Protobuf::util::MessageDifferencer::Equals(current_config_.config_source_, + desired_config_.config_source_)) { // Let the current subscription keep going when it is already in the desired mode. return; } @@ -488,6 +577,8 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, const PolicyInstance* getPolicyInstanceImpl(const std::string& endpoint_policy_name) const; PolicyInstanceConstSharedPtr getPolicyInstanceSharedImpl(const std::string& endpoint_policy_name) const; + uint64_t policySelectorStreamGenerationForTestImpl(const PolicyInstance& policy) const; + SelectorVersion policySelectorVersionForTestImpl(const PolicyInstance& policy) const; void removeInitManager(); static uint64_t instance_id_; @@ -496,8 +587,9 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, uint64_t subscription_id_{0}; bool subscription_should_start_{false}; - void scheduleDeferredDeletion(const PolicyMapSnapshot* old_policy_map); - + void scheduleSelectorDeferredDeletion(DeferredDeletion&& deferred); + void scheduleSelectorGCAndDeferredDeletion(uint64_t published_version, + const PolicyMapSnapshot* old_policy_map = nullptr); void startManagedSubscriptionForTest() { subscription_should_start_ = true; subscribe(); @@ -516,19 +608,27 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, Server::Configuration::ServerFactoryContext& context_; std::atomic map_ptr_; + SelectorMap selector_map_; // Policies hold a shared per-stream state object. A freshly installed stream stores its actual - // gRPC stream generation here. + // gRPC stream generation here, so same-stream selector-only updates advance existing policies + // immediately while old policies remain pinned to the latest selector version reached by their + // own stream. PolicyStreamStateSharedPtr policy_stream_state_{std::make_shared(0)}; const ResourceMap empty_resource_map_; ResourceMap resource_map_; Stats::ScopeSharedPtr npds_stats_scope_; Stats::ScopeSharedPtr policy_stats_scope_; - // We need a separate desired_config_source_ as it may be set to a pessimistic value via explicit + struct XdsConfig { + bool use_nprds_; + envoy::config::core::v3::ConfigSource config_source_; + }; + + // We need a separate desired_config_ as it may be set to a pessimistic value via explicit // BpfMetadata config in CiliumEnvoyConfig CRD, and we should not change to a "worse" (e.g., SotW) - // ConfigSource if "better" (e.g., Delta) is already up-and-running (in config_source_). - envoy::config::core::v3::ConfigSource desired_config_source_; - envoy::config::core::v3::ConfigSource config_source_; + // ConfigSource if "better" (e.g., Delta NPRDS) is already up-and-running (in current_config_). + XdsConfig desired_config_; + XdsConfig current_config_; // init target which starts gRPC subscription Init::TargetImpl init_target_; @@ -917,7 +1017,8 @@ class PortNetworkPolicyRule : public Logger::Loggable { tier_last_precedence_(0), pass_index_(0), l7_proto_("") {} PortNetworkPolicyRule(const NetworkPolicyMapImpl& parent, - const cilium::PortNetworkPolicyRule& rule) + const cilium::PortNetworkPolicyRule& rule, + const ResourceMapOverlay* resource_map) : name_(rule.name()), verdict_(rule.pass_precedence() ? RuleVerdict::Pass : (rule.deny() ? RuleVerdict::Deny : RuleVerdict::Allow)), @@ -928,10 +1029,26 @@ class PortNetworkPolicyRule : public Logger::Loggable { fmt::format("PortNetworkPolicyRule: pass_precedence {} must be lower than precedence {}", tier_last_precedence_, precedence_)); } - for (const auto& remote : rule.remote_policies()) { - ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", verdict_, - remote, name_); - remotes_.emplace(remote); + if (resource_map) { + if (rule.remote_policies_size()) { + throw EnvoyException( + "NetworkPolicyResource rule must use selectors instead of remote_policies"); + } + selectors_.reserve(rule.selectors_size()); + for (const auto& selector : rule.selectors()) { + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} selector {} by rule: {}", verdict_, + selector, name_); + selectors_.emplace_back(resource_map->getSelectorHandleOrThrow(selector)); + } + } else { + if (rule.selectors_size()) { + throw EnvoyException("NetworkPolicy rule must not use selectors"); + } + for (const auto remote : rule.remote_policies()) { + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", verdict_, + remote, name_); + remotes_.emplace(remote); + } } if (rule.has_downstream_tls_context()) { auto config = rule.downstream_tls_context(); @@ -968,22 +1085,41 @@ class PortNetworkPolicyRule : public Logger::Loggable { } } - bool isRemoteWildcard() const { return remotes_.empty(); } + bool isRemoteWildcard() const { return remotes_.empty() && selectors_.empty(); } + + bool matchesRemoteId(uint32_t remote_id, const SelectorVersion selector_version) const { + if (isRemoteWildcard()) { + return true; + } + if (!remotes_.empty()) { + return remotes_.contains(remote_id); + } + + for (const auto& selector : selectors_) { + const auto resolved_selector = selector->get(selector_version); + if (resolved_selector && resolved_selector->contains(remote_id)) { + return true; + } + } + return false; + } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id) const { + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, + const SelectorVersion selector_version) const { // proxy_id must match if we have any. if (proxy_id_ && proxy_id != proxy_id_) { return RuleVerdict::None; } // Remote ID must match if we have any. - if (!isRemoteWildcard() && !remotes_.contains(remote_id)) { + if (!matchesRemoteId(remote_id, selector_version)) { return RuleVerdict::None; // no verdict } ASSERT(verdict_ != RuleVerdict::None, "rule must have a verdict"); return verdict_; } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, + const SelectorVersion selector_version) const { // sni must match if we have any if (!allowed_snis_.empty() && (sni.empty() || std::ranges::none_of(allowed_snis_, [&](const auto& pattern) { @@ -991,13 +1127,14 @@ class PortNetworkPolicyRule : public Logger::Loggable { }))) { return RuleVerdict::None; } - return getVerdict(proxy_id, remote_id); + return getVerdict(proxy_id, remote_id, selector_version); } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, Envoy::Http::RequestHeaderMap& headers, - Cilium::AccessLog::Entry& log_entry) const { - auto verdict = getVerdict(proxy_id, remote_id); + Cilium::AccessLog::Entry& log_entry, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (!hasHttpRules() || verdict != RuleVerdict::Allow) { return verdict; } @@ -1021,8 +1158,9 @@ class PortNetworkPolicyRule : public Logger::Loggable { return (header_matched) ? RuleVerdict::Allow : RuleVerdict::None; } - RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { - auto verdict = getVerdict(proxy_id, remote_id); + RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (verdict != RuleVerdict::Allow) { return verdict; } @@ -1037,8 +1175,9 @@ class PortNetworkPolicyRule : public Logger::Loggable { // Envoy Metadata matcher, called after deny has already been checked for RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, - const envoy::config::core::v3::Metadata& metadata) const { - auto verdict = getVerdict(proxy_id, remote_id); + const envoy::config::core::v3::Metadata& metadata, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (verdict != RuleVerdict::Allow) { return verdict; } @@ -1089,8 +1228,18 @@ class PortNetworkPolicyRule : public Logger::Loggable { } void toString(int indent, std::string& res) const { - res.append(indent - 2, ' ').append("- remotes: ["); - res.append(fmt::format("{}", fmt::join(remotes_, ","))); + if (!selectors_.empty()) { + res.append(indent - 2, ' ').append("- selectors: ["); + std::vector quoted_selectors; + quoted_selectors.reserve(selectors_.size()); + for (const auto& selector : selectors_) { + quoted_selectors.emplace_back(fmt::format("\"{}\"", selector->name())); + } + res.append(fmt::format("{}", fmt::join(quoted_selectors, ","))); + } else { + res.append(indent - 2, ' ').append("- remotes: ["); + res.append(fmt::format("{}", fmt::join(remotes_, ","))); + } res.append("]\n"); if (!name_.empty()) { @@ -1158,6 +1307,7 @@ class PortNetworkPolicyRule : public Logger::Loggable { const uint32_t tier_last_precedence_; uint32_t pass_index_; absl::btree_set remotes_; + std::vector selectors_; std::vector allowed_snis_; // All SNIs allowed if empty. std::shared_ptr> @@ -1211,13 +1361,15 @@ class PortNetworkPolicyRules : public Logger::Loggable { // First call marks 'rules_' as initialized. Of further calls, if either is empty, // we must add a default allow rule to retain the semantics of empty rules. void append(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* selector_resource_map) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (const auto& it : rules) { - rules_.emplace_back(std::make_shared(parent, it)); + rules_.emplace_back( + std::make_shared(parent, it, selector_resource_map)); updateFor(rules_.back()); } initialized_ = true; @@ -1227,13 +1379,15 @@ class PortNetworkPolicyRules : public Logger::Loggable { // First call marks 'rules_' as initialized. Of further calls, if either is empty, // we must add a default allow rule to retain the semantics of an empty rules. void prepend(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* resource_map) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics rules_.emplace(rules_.begin(), std::make_shared()); } for (const auto& it : rules) { - rules_.emplace(rules_.begin(), std::make_shared(parent, it)); + rules_.emplace(rules_.begin(), + std::make_shared(parent, it, resource_map)); updateFor(rules_.front()); } initialized_ = true; @@ -1406,9 +1560,10 @@ class PortNetworkPolicyRules : public Logger::Loggable { RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, Envoy::Http::RequestHeaderMap& headers, - Cilium::AccessLog::Entry& log_entry) const { + Cilium::AccessLog::Entry& log_entry, + const SelectorVersion selector_version) const { auto verdict = forEachRule(can_short_circuit_, [&](const auto& rule) { - return rule.getVerdict(proxy_id, remote_id, headers, log_entry); + return rule.getVerdict(proxy_id, remote_id, headers, log_entry, selector_version); }); ENVOY_LOG(trace, @@ -1417,24 +1572,30 @@ class PortNetworkPolicyRules : public Logger::Loggable { return verdict; } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { - auto verdict = forEachRule( - true, [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }); + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, + const SelectorVersion selector_version) const { + auto verdict = forEachRule(true, [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }); ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRules(proxy_id: {}, remote_id: {}, sni: {}): {}", proxy_id, remote_id, sni, verdict); return verdict; } - RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { - return forEachRule( - true, [&](const auto& rule) { return rule.useProxylib(proxy_id, remote_id, l7_proto); }); + RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto, + const SelectorVersion selector_version) const { + return forEachRule(true, [&](const auto& rule) { + return rule.useProxylib(proxy_id, remote_id, l7_proto, selector_version); + }); } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, - const envoy::config::core::v3::Metadata& metadata) const { - auto verdict = forEachRule( - true, [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, metadata); }); + const envoy::config::core::v3::Metadata& metadata, + const SelectorVersion selector_version) const { + auto verdict = forEachRule(true, [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, metadata, selector_version); + }); ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRules(proxy_id: {}, remote_id: {}, metadata: {}): {}", @@ -1444,20 +1605,24 @@ class PortNetworkPolicyRules : public Logger::Loggable { } RuleVerdict getServerTlsContext(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, - Ssl::ContextSharedPtr& tls_ctx, - const Ssl::ContextConfig*& config) const { + Ssl::ContextSharedPtr& tls_ctx, const Ssl::ContextConfig*& config, + const SelectorVersion selector_version) const { tls_ctx = nullptr; return forEachRulePred( - [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }, + [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }, [&](const auto& rule) { return rule.getServerTlsContext(tls_ctx, config); }); } RuleVerdict getClientTlsContext(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, - Ssl::ContextSharedPtr& tls_ctx, - const Ssl::ContextConfig*& config) const { + Ssl::ContextSharedPtr& tls_ctx, const Ssl::ContextConfig*& config, + const SelectorVersion selector_version) const { tls_ctx = nullptr; return forEachRulePred( - [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }, + [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }, [&](const auto& rule) { return rule.getClientTlsContext(tls_ctx, config); }); } @@ -1538,13 +1703,14 @@ const PortNetworkPolicyRules* findPortRules(const PolicySnapshot& map, uint16_t } // namespace -PortPolicy::PortPolicy(const PolicySnapshot& map, uint16_t port) +PortPolicy::PortPolicy(const PolicySnapshot& map, uint16_t port, SelectorVersion selector_version) : port_rules_(findPortRules(map, port)), - has_http_rules_(port_rules_ && port_rules_->hasHttpRules()) {} + has_http_rules_(port_rules_ && port_rules_->hasHttpRules()), + selector_version_(selector_version) {} bool PortPolicy::useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { if (port_rules_) { - auto verdict = port_rules_->useProxylib(proxy_id, remote_id, l7_proto); + auto verdict = port_rules_->useProxylib(proxy_id, remote_id, l7_proto, selector_version_); if (verdict == RuleVerdict::Allow) { return true; } @@ -1564,14 +1730,15 @@ bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, headers, log_entry) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, headers, log_entry, selector_version_) == + RuleVerdict::Allow; } bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, sni) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, sni, selector_version_) == RuleVerdict::Allow; } bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, @@ -1579,7 +1746,8 @@ bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, metadata) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, metadata, selector_version_) == + RuleVerdict::Allow; } Ssl::ContextSharedPtr PortPolicy::getServerTlsContext(uint16_t proxy_id, uint32_t remote_id, @@ -1591,7 +1759,8 @@ Ssl::ContextSharedPtr PortPolicy::getServerTlsContext(uint16_t proxy_id, uint32_ config = nullptr; raw_socket_allowed = false; if (port_rules_) { - auto verdict = port_rules_->getServerTlsContext(proxy_id, remote_id, sni, tls_ctx, config); + auto verdict = port_rules_->getServerTlsContext(proxy_id, remote_id, sni, tls_ctx, config, + selector_version_); raw_socket_allowed = verdict == RuleVerdict::Allow && tls_ctx == nullptr && config == nullptr; } return tls_ctx; @@ -1606,7 +1775,8 @@ Ssl::ContextSharedPtr PortPolicy::getClientTlsContext(uint16_t proxy_id, uint32_ config = nullptr; raw_socket_allowed = false; if (port_rules_) { - auto verdict = port_rules_->getClientTlsContext(proxy_id, remote_id, sni, tls_ctx, config); + auto verdict = port_rules_->getClientTlsContext(proxy_id, remote_id, sni, tls_ctx, config, + selector_version_); raw_socket_allowed = verdict == RuleVerdict::Allow && tls_ctx == nullptr && config == nullptr; } return tls_ctx; @@ -1623,7 +1793,8 @@ bool inline rangesOverlap(const PortRange& a, const PortRange& b) { class PortNetworkPolicy : public Logger::Loggable { public: PortNetworkPolicy(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* resource_map) { for (const auto& rule : rules) { // Only TCP supported for HTTP if (rule.protocol() == envoy::config::core::v3::SocketAddress::TCP) { @@ -1782,10 +1953,10 @@ class PortNetworkPolicy : public Logger::Loggable { // so the relative order of rules from this batch is reversed. This // is harmless: equal-precedence rules are evaluated as alternatives // (stable sort only affects presentation/debug ordering). - rules.prepend(parent, rule.rules()); + rules.prepend(parent, rule.rules(), resource_map); } else { // Rules with a non-trivial range go to the back of the list - rules.append(parent, rule.rules()); + rules.append(parent, rule.rules(), resource_map); } } } else { @@ -1837,7 +2008,9 @@ class PortNetworkPolicy : public Logger::Loggable { } } - const PortPolicy findPortPolicy(uint16_t port) const { return PortPolicy(rules_, port); } + const PortPolicy findPortPolicy(uint16_t port, const SelectorVersion selector_version) const { + return PortPolicy(rules_, port, selector_version); + } void toString(int indent, std::string& res) const { if (rules_.empty()) { @@ -1862,10 +2035,13 @@ class PolicyInstanceImpl : public PolicyInstance { public: friend class NetworkPolicyMapImpl; PolicyInstanceImpl(const NetworkPolicyMapImpl& parent, uint64_t hash, - const cilium::NetworkPolicy& proto) + const cilium::NetworkPolicy& proto, + const PolicyStreamStateConstSharedPtr& policy_stream_state, + const ResourceMapOverlay* resource_map) : endpoint_id_(proto.endpoint_id()), hash_(hash), policy_proto_(proto), endpoint_ips_(proto), - parent_(parent), ingress_(parent, policy_proto_.ingress_per_port_policies()), - egress_(parent, policy_proto_.egress_per_port_policies()) {} + parent_(parent), policy_stream_state_(policy_stream_state), + ingress_(parent, policy_proto_.ingress_per_port_policies(), resource_map), + egress_(parent, policy_proto_.egress_per_port_policies(), resource_map) {} bool allowed(bool ingress, uint16_t proxy_id, uint32_t remote_id, uint16_t port, Envoy::Http::RequestHeaderMap& headers, @@ -1884,7 +2060,9 @@ class PolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool ingress, uint16_t port) const override { - return ingress ? ingress_.findPortPolicy(port) : egress_.findPortPolicy(port); + const auto selector_version = policy_stream_state_->version(); + return ingress ? ingress_.findPortPolicy(port, selector_version) + : egress_.findPortPolicy(port, selector_version); } bool useProxylib(bool ingress, uint16_t proxy_id, uint32_t remote_id, uint16_t port, @@ -1916,6 +2094,7 @@ class PolicyInstanceImpl : public PolicyInstance { private: const NetworkPolicyMapImpl& parent_; + const PolicyStreamStateConstSharedPtr policy_stream_state_; const PortNetworkPolicy ingress_; const PortNetworkPolicy egress_; }; @@ -1947,10 +2126,12 @@ std::string describePolicyResourceForLog(const std::string& resource_name, policy.endpoint_id(), endpointIpsForLog(policy.endpoint_ips())); } +// pass nullptr pending_resource_map for non-selector based policy! std::pair, bool> -NetworkPolicyMapImpl::createOrReusePolicy(const std::string& resource_name, - const cilium::NetworkPolicy& config, bool is_new_stream, - const ResourceMap& resource_map) { +NetworkPolicyMapImpl::createOrReusePolicy( + const std::string& resource_name, const cilium::NetworkPolicy& config, + const PolicyStreamStateConstSharedPtr& policy_stream_state, const ResourceMap& resource_map, + bool is_new_stream, const ResourceMapOverlay* pending_resource_map) const { const uint64_t new_hash = MessageUtil::hash(config); if (!is_new_stream) { const auto* old_entry = resource_map.findEntry(resource_name); @@ -1965,7 +2146,42 @@ NetworkPolicyMapImpl::createOrReusePolicy(const std::string& resource_name, } } } - return {std::make_shared(*this, new_hash, config), false}; + return {std::make_shared(*this, new_hash, config, policy_stream_state, + pending_resource_map), + false}; +} + +std::pair +NetworkPolicyMapImpl::createOrReuseSelector(const std::string& resource_name, + const cilium::Selector& config, uint64_t update_version, + const ResourceMap& resource_map) { + // Compare against the selector visible in the currently prepared update version, not just the + // last published one. Under the single-update-in-flight VersionedMap contract, any selector + // visible in 'update_version' is also the indefinite selector value for that candidate update. + const auto* old_entry = resource_map.findEntry(resource_name); + if (old_entry) { + const auto* old_selector_entry = old_entry->selectorResourceEntry(); + if (old_selector_entry) { + const auto& old_selector_handle = old_selector_entry->handle; + if (old_selector_handle) { + const auto* old_selector = old_selector_handle->get(update_version); + if (old_selector && + old_selector->size() == static_cast(config.remote_identities_size()) && + std::ranges::all_of(config.remote_identities(), [&](const auto remote_identity) { + return old_selector->contains(remote_identity); + })) { + return {old_selector_handle, true}; + } + } + } + } + // otherwise create a new one + auto selector = new SelectorInstance(); + selector->reserve(config.remote_identities_size()); + for (const auto remote_identity : config.remote_identities()) { + selector->emplace(remote_identity); + } + return {selector_map_.insert(std::string(resource_name), selector), false}; } template @@ -2018,6 +2234,9 @@ std::string ResourceMapOverlay::describeExistingResourceKey(const std::string& k if (entry == nullptr) { return fmt::format("resource key '{}'", key); } + if (entry->selectorResourceEntry() != nullptr) { + return fmt::format("selector resource '{}'", key); + } if (const auto* policy_entry = entry->policyResourceEntry(); policy_entry != nullptr && policy_entry->policy != nullptr) { return describePolicyResourceForLog(key, policy_entry->policy); @@ -2070,11 +2289,11 @@ void ResourceMapOverlay::erasePolicyResource( // Common base constructor // This is used directly for testing with a file-based subscription -NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& context, +NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source, bool subscribe) : context_(context.serverFactoryContext()) { - impl_ = std::make_shared(context, config_source); + impl_ = std::make_shared(context, use_nprds, config_source); if (subscribe) { impl_->subscribe(); @@ -2106,10 +2325,9 @@ bool NetworkPolicyMap::exists(const std::string& endpoint_policy_name) const { return impl_->getPolicyInstanceImpl(endpoint_policy_name); } -bool NetworkPolicyMap::useDeltaXds() const { return impl_->useDeltaXds(); } - -void NetworkPolicyMap::setConfigSource(const envoy::config::core::v3::ConfigSource& config_source) { - impl_->setConfigSource(config_source); +void NetworkPolicyMap::setConfig(bool use_nprds, + const envoy::config::core::v3::ConfigSource& config_source) { + impl_->setConfig(use_nprds, config_source); } void NetworkPolicyMap::startSubscriptionForTest( @@ -2132,7 +2350,10 @@ void NetworkPolicyMap::onSubscriptionTransportCloseForTest() { } bool NetworkPolicyMap::subscriptionUseDeltaXdsForTest() const { - return impl_->subscriptionUseDeltaXds(); + return configSourceIsDelta(impl_->getConfigSource()); +} +bool NetworkPolicyMap::desiredUseDeltaXdsForTest() const { + return configSourceIsDelta(impl_->desired_config_.config_source_); } bool NetworkPolicyMap::subscriptionConnectedForTest() const { @@ -2152,13 +2373,22 @@ NetworkPolicyMap::getPolicyInstanceSharedForTest(const std::string& endpoint_pol return impl_->getPolicyInstanceSharedImpl(endpoint_policy_name); } +uint64_t +NetworkPolicyMap::policySelectorStreamGenerationForTest(const PolicyInstance& policy) const { + return impl_->policySelectorStreamGenerationForTestImpl(policy); +} + +SelectorVersion NetworkPolicyMap::policySelectorVersionForTest(const PolicyInstance& policy) const { + return impl_->policySelectorVersionForTestImpl(policy); +} + NetworkPolicyMapImpl::NetworkPolicyMapImpl( - Server::Configuration::FactoryContext& context, + Server::Configuration::FactoryContext& context, bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source) : context_(context.serverFactoryContext()), map_ptr_(nullptr), npds_stats_scope_(context_.serverScope().createScope("cilium.npds.")), policy_stats_scope_(context_.serverScope().createScope("cilium.policy.")), - desired_config_source_(config_source), config_source_(config_source), + desired_config_(XdsConfig{use_nprds, config_source}), current_config_(desired_config_), init_target_(fmt::format("Cilium NetworkPolicy subscription start"), [this]() { // Production subscription is allowed to start from now on. @@ -2201,11 +2431,12 @@ NetworkPolicyMapImpl::~NetworkPolicyMapImpl() { void NetworkPolicyMapImpl::subscribe() { subscription_connected_ = false; - config_source_ = desired_config_source_; + current_config_ = desired_config_; ++subscription_id_; if (subscription_factory_for_test_) { - subscription_ = subscription_factory_for_test_(subscriptionUseDeltaXds()); + subscription_ = + subscription_factory_for_test_(configSourceIsDelta(current_config_.config_source_)); if (subscription_should_start_) { startSubscription(); } @@ -2219,9 +2450,18 @@ void NetworkPolicyMapImpl::subscribe() { } }; - subscription_ = - Cilium::subscribe(NetworkPolicyTypeUrl, config_source_, context_, *npds_stats_scope_, *this, - std::make_shared(), std::move(on_stream_event)); + absl::string_view type_url; + Config::OpaqueResourceDecoderSharedPtr decoder; + if (current_config_.use_nprds_) { + type_url = NetworkPolicyResourceTypeUrl; + decoder = std::make_shared(); + } else { + type_url = NetworkPolicyTypeUrl; + decoder = std::make_shared(); + } + + subscription_ = Cilium::subscribe(type_url, current_config_.config_source_, context_, + *npds_stats_scope_, *this, decoder, std::move(on_stream_event)); if (subscription_should_start_) { startSubscription(); @@ -2245,6 +2485,13 @@ void NetworkPolicyMapImpl::installNewPolicyMap( // Initialize SDS secrets. We do not wait for the completion. version_init_manager.initialize(Init::WatcherImpl(std::move(version_name), []() {})); + // Publish selector data before publishing the new policy map. New policies created above already + // point at 'policy_stream_state', so any worker that can observe the swapped-in policy map must + // also be able to observe the selector version those policies expect to use. + auto new_version = selector_map_.publishNextVersion(); + if (new_version > 0) { + policy_stream_state->publishVersion(new_version); + } policy_stream_state_ = policy_stream_state; stats_.policy_stream_generation_.set(policy_stream_state_->streamGeneration()); @@ -2255,7 +2502,8 @@ void NetworkPolicyMapImpl::installNewPolicyMap( // old version can be GC'd once all worker threads have quiesced. // Delete the old map once all worker threads have entered their event queues, as this is proof // that they no longer refer to the old map. - scheduleDeferredDeletion(exchange(new PolicyMapSnapshot(std::move(new_policy_map)))); + scheduleSelectorGCAndDeferredDeletion(new_version, + exchange(new PolicyMapSnapshot(std::move(new_policy_map)))); } // removeInitManager must be called at the end of each policy update @@ -2298,13 +2546,32 @@ void NetworkPolicyMapImpl::removeInitManager() { transport_factory_context_->setInitManager(*parked_init_manager_); } -void NetworkPolicyMapImpl::scheduleDeferredDeletion(const PolicyMapSnapshot* old_policy_map) { - if (old_policy_map == nullptr) { +void NetworkPolicyMapImpl::scheduleSelectorDeferredDeletion( + DeferredDeletion&& deferred) { + if (deferred.empty()) { return; } - runAfterAllThreads([old_policy_map]() { + auto deferred_owner = std::make_shared>(std::move(deferred)); + // The callback exists only to keep the deferred-deletion batch alive until all workers have + // quiesced once more. The batch deletes its nodes from the closure destructor. + runAfterAllThreads([deferred_owner]() {}); +} + +void NetworkPolicyMapImpl::scheduleSelectorGCAndDeferredDeletion( + uint64_t published_version, const PolicyMapSnapshot* old_policy_map) { + if (published_version == 0 && old_policy_map == nullptr) { + return; + } + runAfterAllThreads([shared_this = shared_from_this(), published_version, old_policy_map]() { // Clean-up in the main thread after all worker threads have scheduled. + // Delete the old policy map before selector GC. Old policies are the only remaining users of + // old-stream selector versions; once the old map is gone after this quiescence point, those + // selector versions may be unlinked and deferred for deletion. delete old_policy_map; + if (published_version == 0) { + return; + } + shared_this->scheduleSelectorDeferredDeletion(shared_this->selector_map_.gc(published_version)); }); } @@ -2335,7 +2602,9 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( ResourceMapOverlay pending_resource_map; const auto policy_stream_state = - is_new_stream ? std::make_shared(stream_generation) : policy_stream_state_; + is_new_stream + ? std::make_shared(stream_generation, selector_map_.getVersion()) + : policy_stream_state_; try { // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, @@ -2343,36 +2612,127 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, // so open it before the workers get a chance to enforce policy on the new IDs. if (is_new_stream) { - ENVOY_LOG(info, "New NetworkPolicy stream {}", stream_generation); + ENVOY_LOG(info, "New NetworkPolicy{} stream {}", current_config_.use_nprds_ ? "Resource" : "", + stream_generation); reopenIpcache(); } - for (const auto& resource : resources) { - const auto& config = dynamic_cast(resource.get().resource()); - const std::string& resource_name = resource.get().name(); - validateResourceName(resource_name, "NetworkPolicy resource name"); - validatePolicy(resource_name, config, version_info); - ENVOY_LOG(debug, - "Received NetworkPolicy for endpoint {}, endpoint_ip {} in onConfigUpdate() " - "version {}", - config.endpoint_id(), config.endpoint_ips()[0], version_info); - - auto [policy, reused] = - createOrReusePolicy(resource_name, config, is_new_stream, resource_map); - if (reused) { - ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + // Handle NPRDS and NPDS separately + if (current_config_.use_nprds_) { + const auto selector_update_version = selector_map_.prepareNextVersion(); + absl::flat_hash_set incoming_selector_names; + + // First validate and collect selector resource names + for (const auto& resource : resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicyResource resource name"); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kPolicy: + validatePolicy(resource_name, typed_resource.policy(), version_info); + break; + case cilium::NetworkPolicyResource::kSelector: + ENVOY_LOG(debug, + "Received NetworkPolicyResource selector {} in onConfigUpdate() version {}", + resource_name, version_info); + incoming_selector_names.emplace(resource_name); + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("NetworkPolicyResource has no payload"); + } + } + + // SotW resources are a complete snapshot. Selectors omitted from this snapshot are removed. + for (const auto& [resource_name, resource_key] : resource_map_) { + if (resource_key.selectorResourceEntry() && + !incoming_selector_names.contains(resource_name)) { + selector_map_.clear(resource_name); + } + } + + // Add selectors before policies so policies can resolve selectors from the same snapshot. + for (const auto& resource : resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kSelector) { + continue; + } + const std::string& resource_name = resource.get().name(); + auto [selector_handle, reused] = createOrReuseSelector( + resource_name, typed_resource.selector(), selector_update_version, resource_map); + if (reused) { + ENVOY_LOG(trace, "New selector is equal to old one, not updating."); + } + if (!pending_resource_map.insertOrUpdateSelectorResource(resource_name, selector_handle)) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}': " + "incoming selector resource '{}' collides with existing {}", + version_info, resource_name, resource_name, + pending_resource_map.describeExistingResourceKey(resource_name))); + } + } + + // Finally add policies + for (const auto& resource : resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kPolicy) { + continue; + } + const std::string& resource_name = resource.get().name(); + const auto& config = typed_resource.policy(); + auto [policy, reused] = + createOrReusePolicy(resource_name, config, policy_stream_state, resource_map, + is_new_stream, &pending_resource_map); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + } + insertPolicyResource( + pending_resource_map, config, version_info, is_new_stream, resource_name, policy, + [](absl::string_view ip) { + ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", ip); + }); + } + } else { + // NPDS just adds policy resources + for (const auto& resource : resources) { + const auto& config = dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicy resource name"); + validatePolicy(resource_name, config, version_info); + + auto [policy, reused] = createOrReusePolicy(resource_name, config, policy_stream_state, + resource_map, is_new_stream, nullptr); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + } + insertPolicyResource( + pending_resource_map, config, version_info, is_new_stream, resource_name, policy, + [](absl::string_view ip) { + ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", ip); + }); } - insertPolicyResource( - pending_resource_map, config, version_info, is_new_stream, resource_name, policy, - [](absl::string_view ip) { - ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", ip); - }); } } catch (const EnvoyException& e) { - ENVOY_LOG(warn, "NetworkPolicy update for version {} failed: {}", version_info, e.what()); + ENVOY_LOG(warn, "NetworkPolicy{} update for version {} failed: {}", + current_config_.use_nprds_ ? "Resource" : "", version_info, e.what()); stats_.updates_rejected_.inc(); removeInitManager(); + if (selector_map_.resetNextVersionIfNotChanged()) { + scheduleSelectorDeferredDeletion(selector_map_.revert()); + } throw; // re-throw + } catch (const std::bad_cast&) { + ENVOY_LOG(warn, "NetworkPolicy{} update for version {} failed: invalid resource type", + current_config_.use_nprds_ ? "Resource" : "", version_info); + stats_.updates_rejected_.inc(); + removeInitManager(); + if (selector_map_.resetNextVersionIfNotChanged()) { + scheduleSelectorDeferredDeletion(selector_map_.revert()); + } + throw EnvoyException("Invalid resource type for Cilium NetworkPolicy"); } stats_.update_success_.inc(); @@ -2429,7 +2789,9 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( ResourceMapOverlay pending_resource_map = is_new_stream ? ResourceMapOverlay() : ResourceMapOverlay(resource_map); const auto policy_stream_state = - is_new_stream ? std::make_shared(stream_generation) : policy_stream_state_; + is_new_stream + ? std::make_shared(stream_generation, selector_map_.getVersion()) + : policy_stream_state_; try { // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, @@ -2437,59 +2799,278 @@ absl::Status NetworkPolicyMapImpl::onConfigUpdate( // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, // so open it before the workers get a chance to enforce policy on the new IDs. if (is_new_stream) { - ENVOY_LOG(info, "New NetworkPolicy stream {}", stream_generation); + ENVOY_LOG(info, "New NetworkPolicy{} stream {}", current_config_.use_nprds_ ? "Resource" : "", + stream_generation); reopenIpcache(); } - for (const auto& removed_resource : removed_resources) { - ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource {}", removed_resource); - const auto* resource_entry = pending_resource_map.findEntry(removed_resource); - if (resource_entry == nullptr) { - continue; + // Handle NPRDS and NPDS separately + if (current_config_.use_nprds_) { + // First validate and find whether this is a selector-only update. + bool adds_policies = false; + bool removes_policies = false; + bool adds_selectors = false; + bool removes_selectors = false; + for (const auto& removed_resource : removed_resources) { + validateResourceName(removed_resource, "NetworkPolicyResource removed resource name"); + auto resource_it = resource_map.find(removed_resource); + if (resource_it == resource_map.end()) { + continue; + } + if (resource_it->second.selectorResourceEntry()) { + removes_selectors = true; + } else if (resource_it->second.policyResourceEntry()) { + removes_policies = true; + } else { + throw EnvoyException(fmt::format( + "NetworkPolicyResource removed resource '{}' is a policy endpoint IP alias, " + "not a resource name", + removed_resource)); + } } - if (pending_resource_map.erasePolicyResourceIfPresent(removed_resource)) { - continue; + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicyResource added resource name"); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kPolicy: + validatePolicy(resource_name, typed_resource.policy(), system_version_info); + adds_policies = true; + break; + case cilium::NetworkPolicyResource::kSelector: + ENVOY_LOG(trace, + "Received NetworkPolicyResource selector {} in onConfigUpdate() " + "version {}", + resource_name, system_version_info); + adds_selectors = true; + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("NetworkPolicyResource has no payload"); + } } - throw EnvoyException( - fmt::format("NetworkPolicyResource removed resource '{}' is a policy endpoint IP alias, " - "not a resource name", - removed_resource)); - } - for (const auto& resource : added_resources) { - const std::string& resource_name = resource.get().name(); - validateResourceName(resource_name, "NetworkPolicy added resource name"); - pending_resource_map.erasePolicyResourceIfPresent(resource_name); - - const auto& config = dynamic_cast(resource.get().resource()); - validatePolicy(resource_name, config, system_version_info); ENVOY_LOG(debug, - "Received NetworkPolicyResource {} for endpoint {}, endpoint_ip {} in " - "onConfigUpdate() version {}", - resource_name, config.endpoint_id(), config.endpoint_ips()[0], system_version_info); - - auto [policy, reused] = - createOrReusePolicy(resource_name, config, is_new_stream, resource_map); - if (reused) { - ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + "NetworkPolicyMapImpl::onConfigUpdate({}), " + "version: {}, stream {}, updates_selectors: {}, updates_policies: {}", + instance_id_, system_version_info, stream_generation, + removes_selectors || adds_selectors, removes_policies || adds_policies); + + const auto selector_update_version = selector_map_.prepareNextVersion(); + + // Is this a selector only update? + if (!is_new_stream && (removes_selectors || adds_selectors) && + !(removes_policies || adds_policies)) { + for (const auto& resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource selector {}", resource); + const auto* resource_entry = pending_resource_map.findEntry(resource); + if (resource_entry == nullptr) { + ENVOY_LOG( + debug, + "NetworkPolicyResource removed selector name '{}' not found from resource map", + resource); + continue; + } + if (resource_entry->isPolicyEndpointIpEntry()) { + throw EnvoyException(fmt::format("NetworkPolicyResource removed selector name " + "'{}' is a policy endpoint IP alias, " + "not a resource name", + resource)); + } + if (resource_entry->policyResourceEntry()) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource removed selector name '{}' refers to a policy resource", + resource)); + } + selector_map_.clear(resource); + pending_resource_map.erase(resource); + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kSelector: { + ENVOY_LOG(debug, + "Received NetworkPolicyResource selector {} in onConfigUpdate() " + "version {}", + resource_name, system_version_info); + auto [selector_handle, reused] = createOrReuseSelector( + resource_name, typed_resource.selector(), selector_update_version, resource_map); + if (reused) { + ENVOY_LOG(trace, "New selector is equal to old one, not updating."); + } + if (!pending_resource_map.insertOrUpdateSelectorResource(resource_name, + selector_handle)) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource selector update for version {} " + "has duplicate resource key '{}' on {} " + "stream: incoming selector resource '{}' collides with existing {}", + system_version_info, resource_name, is_new_stream ? "a new" : "an old", + resource_name, pending_resource_map.describeExistingResourceKey(resource_name))); + } + break; + } + case cilium::NetworkPolicyResource::kPolicy: + IS_ENVOY_BUG( + "Selector-only NetworkPolicyResource update unexpectedly included a policy"); + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("NetworkPolicyResource has no payload"); + } + } + } else { // Policy and selector update path + // First remove removed resources. + for (const auto& removed_resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource {}", removed_resource); + const auto* resource_entry = pending_resource_map.findEntry(removed_resource); + if (!resource_entry) { + continue; + } + if (resource_entry->selectorResourceEntry()) { + selector_map_.clear(removed_resource); + pending_resource_map.erase(removed_resource); + } else if (const auto* policy_entry = resource_entry->policyResourceEntry()) { + pending_resource_map.erasePolicyResource(removed_resource, policy_entry->policy); + } + } + // Then remove policies that are being replaced so duplicate endpoint IP detection sees the + // pending state. Selector resources are updated in place below to preserve their stable + // handles. + if (adds_policies) { + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() == cilium::NetworkPolicyResource::kPolicy) { + const std::string& resource_name = resource.get().name(); + pending_resource_map.erasePolicyResourceIfPresent(resource_name); + } + } + } + // Then add selectors so policies can bind to selectors introduced by the same update. + if (adds_selectors) { + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kSelector) { + continue; + } + const std::string& resource_name = resource.get().name(); + auto [selector_handle, reused] = createOrReuseSelector( + resource_name, typed_resource.selector(), selector_update_version, resource_map); + if (reused) { + ENVOY_LOG(trace, "New selector is equal to old one, not updating."); + } + if (!pending_resource_map.insertOrUpdateSelectorResource(resource_name, + selector_handle)) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource selector update for version {} has duplicate resource " + "key '{}' on {} " + "stream: incoming selector resource '{}' collides with existing {}", + system_version_info, resource_name, is_new_stream ? "a new" : "an old", + resource_name, pending_resource_map.describeExistingResourceKey(resource_name))); + } + } + } + // Finally add policies. + if (adds_policies) { + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kPolicy) { + continue; + } + const std::string& resource_name = resource.get().name(); + const auto& config = typed_resource.policy(); + auto [policy, reused] = + createOrReusePolicy(resource_name, config, policy_stream_state, resource_map, + is_new_stream, &pending_resource_map); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + } + insertPolicyResource(pending_resource_map, config, system_version_info, is_new_stream, + resource_name, policy, [](absl::string_view ip) { + ENVOY_LOG(trace, + "Cilium updating network policy for endpoint {}", ip); + }); + } + } + } + } else { + // Delta NPDS carries only NetworkPolicy resources, so skip the selector scan/update paths. + for (const auto& removed_resource : removed_resources) { + validateResourceName(removed_resource, "NetworkPolicy removed resource name"); + ENVOY_LOG(trace, "Cilium removing NetworkPolicy {}", removed_resource); + const auto* resource_entry = pending_resource_map.findEntry(removed_resource); + if (resource_entry == nullptr) { + continue; + } + if (const auto* policy_entry = resource_entry->policyResourceEntry()) { + pending_resource_map.erasePolicyResource(removed_resource, policy_entry->policy); + continue; + } + if (resource_entry->isPolicyEndpointIpEntry()) { + throw EnvoyException( + fmt::format("NetworkPolicy removed resource '{}' is a policy endpoint " + "IP alias, not a resource name", + removed_resource)); + } + throw EnvoyException(fmt::format( + "NetworkPolicy removed resource '{}' is not a policy resource", removed_resource)); + } + + // First validate and remove policies that are being replaced so duplicate endpoint IP + // detection sees the pending state. + for (const auto& resource : added_resources) { + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicy added resource name"); + const auto& config = dynamic_cast(resource.get().resource()); + validatePolicy(resource_name, config, system_version_info); + pending_resource_map.erasePolicyResourceIfPresent(resource_name); + } + + for (const auto& resource : added_resources) { + const std::string& resource_name = resource.get().name(); + const auto& config = dynamic_cast(resource.get().resource()); + auto [policy, reused] = createOrReusePolicy(resource_name, config, policy_stream_state, + resource_map, is_new_stream, nullptr); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + } + insertPolicyResource(pending_resource_map, config, system_version_info, is_new_stream, + resource_name, policy, [](absl::string_view ip) { + ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", + ip); + }); } - insertPolicyResource(pending_resource_map, config, system_version_info, is_new_stream, - resource_name, policy, [](absl::string_view ip) { - ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", ip); - }); } } catch (const EnvoyException& e) { - ENVOY_LOG(warn, "NetworkPolicy update for version {} failed: {}", system_version_info, - e.what()); + ENVOY_LOG(warn, "NetworkPolicy{} update for version {} failed: {}", + current_config_.use_nprds_ ? "Resource" : "", system_version_info, e.what()); stats_.updates_rejected_.inc(); removeInitManager(); + if (selector_map_.resetNextVersionIfNotChanged()) { + scheduleSelectorDeferredDeletion(selector_map_.revert()); + } throw; // re-throw + } catch (const std::bad_cast&) { + ENVOY_LOG(warn, "NetworkPolicy{} update for version {} failed: invalid resource type", + current_config_.use_nprds_ ? "Resource" : "", system_version_info); + stats_.updates_rejected_.inc(); + removeInitManager(); + if (selector_map_.resetNextVersionIfNotChanged()) { + scheduleSelectorDeferredDeletion(selector_map_.revert()); + } + throw EnvoyException("Invalid resource type for Cilium NetworkPolicy"); } stats_.update_success_.inc(); removeInitManager(); - // Do not carry over any resources from an old stream + // Do not carry over any resources from an old stream. if (is_new_stream) { resource_map_.clear(); } @@ -2547,7 +3128,7 @@ class AllowAllEgressPolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool ingress, uint16_t) const override { - return ingress ? PortPolicy(empty_map_, 0) : PortPolicy(empty_map_, 1); + return ingress ? PortPolicy(empty_map_, 0, versionMin) : PortPolicy(empty_map_, 1, versionMin); } bool useProxylib(bool, uint16_t, uint32_t, uint16_t, std::string&) const override { @@ -2590,7 +3171,7 @@ class DenyAllPolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool, uint16_t) const override { - return PortPolicy(empty_map_, 0); + return PortPolicy(empty_map_, 0, versionMin); } bool useProxylib(bool, uint16_t, uint32_t, uint16_t, std::string&) const override { @@ -2638,6 +3219,22 @@ NetworkPolicyMapImpl::getPolicyInstanceSharedImpl(const std::string& endpoint_ip return nullptr; } +uint64_t NetworkPolicyMapImpl::policySelectorStreamGenerationForTestImpl( + const PolicyInstance& policy) const { + if (const auto* policy_impl = dynamic_cast(&policy)) { + return policy_impl->policy_stream_state_->streamGeneration(); + } + return 0; +} + +SelectorVersion +NetworkPolicyMapImpl::policySelectorVersionForTestImpl(const PolicyInstance& policy) const { + if (const auto* policy_impl = dynamic_cast(&policy)) { + return policy_impl->policy_stream_state_->version(); + } + return versionMin; +} + // getPolicyInstance return a const reference to a policy in the policy map for the given // 'endpoint_ip'. If there is no policy for the given IP, a default policy is returned, // controlled by the 'default_allow_egress' argument as follows: diff --git a/cilium/network_policy.h b/cilium/network_policy.h index 449a1fb6f..e6d041f6d 100644 --- a/cilium/network_policy.h +++ b/cilium/network_policy.h @@ -7,6 +7,7 @@ #include #include +#include "envoy/common/exception.h" #include "envoy/common/pure.h" #include "envoy/common/regex.h" #include "envoy/config/core/v3/base.pb.h" @@ -41,6 +42,7 @@ namespace Cilium { class PortNetworkPolicyRules; class PolicySnapshot; +using SelectorVersion = uint64_t; // PortPolicy holds a reference to a set of rules in a policy map that apply to the given port. // Methods then iterate through the set to determine if policy allows or denies. This is needed to @@ -51,7 +53,7 @@ class PortPolicy : public Logger::Loggable { friend class PortNetworkPolicy; friend class DenyAllPolicyInstanceImpl; friend class AllowAllEgressPolicyInstanceImpl; - PortPolicy(const PolicySnapshot& map, uint16_t port); + PortPolicy(const PolicySnapshot& map, uint16_t port, SelectorVersion selector_version); public: // If hasHttpRules() returns false, then HTTP policy enforcement can be skipped, @@ -97,6 +99,7 @@ class PortPolicy : public Logger::Loggable { // rules. const PortNetworkPolicyRules* port_rules_; const bool has_http_rules_; + const SelectorVersion selector_version_; }; class IpAddressPair { @@ -168,6 +171,35 @@ class NetworkPolicyDecoder : public Envoy::Config::OpaqueResourceDecoder { ProtobufMessage::ValidationVisitor& validation_visitor_; }; +// cilium::NetworkPolicyResource does not carry a resource name, but relies on the +// DeltaDiscoveryRespons Resource wrapper to have the name. Hence can not use +// Envoy::Config::OpaqueResourceDecoderImpl +class NetworkPolicyResourceDecoder : public Envoy::Config::OpaqueResourceDecoder { +public: + NetworkPolicyResourceDecoder() + : validation_visitor_(ProtobufMessage::getNullValidationVisitor()) {} + + // Config::OpaqueResourceDecoder + ProtobufTypes::MessagePtr decodeResource(const Protobuf::Any& resource) override { + auto typed_message = std::make_unique(); + // If the Any is a synthetic empty message (e.g. because the resource field + // was not set in Resource, this might be empty, so we shouldn't decode. + if (!resource.type_url().empty()) { + MessageUtil::anyConvertAndValidate(resource, *typed_message, + validation_visitor_); + } + return typed_message; + } + + std::string resourceName(const Protobuf::Message&) override { + throw EnvoyException( + "NetworkPolicyResource does not carry a name and must be wrapped in Resource"); + } + +private: + ProtobufMessage::ValidationVisitor& validation_visitor_; +}; + /** * All Cilium L7 filter stats. @see stats_macros.h */ @@ -194,16 +226,15 @@ class NetworkPolicyMapImpl; class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable { public: - NetworkPolicyMap(Server::Configuration::FactoryContext& context, + NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source, bool subscribe = false); ~NetworkPolicyMap() override; bool exists(const std::string& endpoint_policy_name) const; - bool useDeltaXds() const; - void setConfigSource(const envoy::config::core::v3::ConfigSource& config_source); + void setConfig(bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source); const PolicyInstance& getPolicyInstance(const std::string& endpoint_policy_name, bool allow_egress) const; @@ -221,11 +252,14 @@ class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable&& subscription); void startManagedSubscriptionForTest(); void setSubscriptionFactoryForTest(SubscriptionFactoryForTest factory); void onSubscriptionConnectedForTest(); void onSubscriptionTransportCloseForTest(); + bool desiredUseDeltaXdsForTest() const; bool subscriptionUseDeltaXdsForTest() const; bool subscriptionConnectedForTest() const; Envoy::Config::SubscriptionCallbacks& subscriptionCallbacksForTest() const; diff --git a/cilium/versioned.h b/cilium/versioned.h new file mode 100644 index 000000000..c41e94d0f --- /dev/null +++ b/cilium/versioned.h @@ -0,0 +1,531 @@ +#pragma once + +// NOLINT(namespace-envoy) + +#include +#include +#include +#include +#include +#include + +#include "source/common/common/assert.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" + +constexpr uint64_t versionMin = 0; +constexpr uint64_t versionMax = std::numeric_limits::max() - 1; +constexpr uint64_t versionNotRemoved = std::numeric_limits::max(); + +// Versioned.h provides a lock-free reader / single-writer versioned-value container, +// based on earlier implementations in OVS (C) and Cilium (Go). +// +// API contract and intended use: +// +// 1. Object model +// - T is a CRTP node type inheriting from VersionedNode. +// - VersionedValue stores a linked list of historical T nodes, newest first. +// - VersionedReadable exposes only the worker-safe read API: get(version). +// - VersionedHandle is a shared_ptr, so handles intentionally +// expose only the readable API to worker/runtime code. Main-thread mutation goes through +// VersionedMap, which keeps mutable shared_ptr internally. +// - The optional readable base R allows a handle to expose extra read-only metadata (for +// example selector names) without exposing VersionedValue mutators. +// +// 2. Threading model +// - There is exactly one mutator thread: the main thread. +// - Worker threads may concurrently call VersionedReadable::get(version) and may traverse nodes +// reached from a handle without additional locking. +// - Any future cross-thread mutation must still preserve the exclusive-writer rule, e.g. by +// holding an exclusive lock around all mutation. +// - VersionedMap itself is main-thread-only. Its name directory (map_), dirty set, and +// transactional state must not be accessed from worker threads. +// +// 3. VersionedMap transaction flow +// - prepareNextVersion() starts a new unpublished update and returns the candidate version to +// use for any main-thread lookups against in-flight state. +// - insert(key, value) reuses the existing stable handle if the key is still present in the +// main-thread directory; otherwise it creates a fresh handle. +// - clear(key) marks the key's current value invisible in the candidate version but does not +// erase the key from the main-thread directory immediately. +// - publishNextVersion() publishes all pending changes, removes keys whose handle has no visible +// value in the published version, and returns the new published version, or 0 if there was +// nothing to publish. +// - revert() discards the unpublished update, restores visibility of older nodes as needed, and +// removes keys whose handle has no value in the published version. +// - find(key) returns the stable readable handle parked under the key in the main-thread +// directory. It does not itself answer whether the key is visible in any specific version; use +// handle->get(version) for that. +// +// 4. Stable-handle semantics +// - Updating an active key writes a new version onto the same stable handle. +// - clear() followed by re-add of the same key in the same unpublished update also reuses that +// same handle; the caller should compare against the candidate version returned from +// prepareNextVersion(), not against the currently published version. +// - Once a key has been fully removed across a published version boundary, the name disappears +// from VersionedMap. A later same-name add creates a fresh stable handle. +// - This is why old published policies can keep using an old cleared handle while refreshed +// policies bind a new one after a later same-name re-add. +// +// 5. Reader semantics +// - get(version) walks from the current head and returns the first node visible in the requested +// version. +// - The walk stops at the first node that was added before the requested version but is no +// longer visible there; all remaining nodes are older and therefore already invisible in that +// version. +// - Querying an older published version after newer publishes is logically stale and may produce +// outdated results, but it must remain memory-safe. +// +// 6. Deferred deletion / grace periods +// - Unpublished nodes removed by revert() were never visible to workers, but are still unlinked +// into a DeferredDeletion batch so that any concurrent traversal that had already stepped into +// them cannot race with delete. +// - gc(published_version) is only the first GC phase. It must be called only after all workers +// have quiesced for that published version, unlinks nodes that are no longer visible to +// anyone, and returns them in a DeferredDeletion batch. +// - The returned DeferredDeletion batch must stay alive until all workers have quiesced once +// more. Destroying the batch performs the actual node deletion. +// - The production pattern is therefore: publish version N, wait for worker quiescence, call +// gc(N), then keep the returned DeferredDeletion alive for one more quiescence round. +// +// 7. High-level memory-model rationale +// - The main thread publishes new list links with 'release' stores to the list node pointers. +// - Workers traverse with 'acquire' loads from the list node pointers, so they either observe +// the old reachable chain or the newly published/unlinked chain, but not torn pointer state. +// - published_version_ is stored with 'release' in publishNextVersion() and loaded with +// 'acquire' by readers, so readers that observe a new published version also observe the +// prior main-thread mutations that made that version reachable. +// - remove_version_ uses 'relaxed' atomic loads/stores because it controls logical visibility, +// not structural reachability. Structural coherence and memory safety come from the +// 'release'/'acquire' ordering on list node pointers plus the extra deferred-deletion grace +// period. +// - Many mutation-side loads/stores are 'relaxed' because mutation is single-threaded on the +// main thread. +// +// 8. DeferredDeletion implementation detail +// - DeferredDeletion reuses VersionedNode::next_ to chain together unlinked nodes awaiting +// destruction. This keeps deferred deletion self-contained without an extra node container. +// - Public code must not rely on any semantics of detached invisible tails beyond memory safety; +// only get(version) is the supported read API. +// +// 9. Test coverage in tests/versioned_test.cc +// - Value-level tests cover visibility rules, shadowing, clear(), revert(), first-phase GC, and +// deferred deletion ownership/move semantics. +// - Map-level tests cover version numbering, publish-without-change, insert/update/clear/revert, +// same-handle reuse for active updates and same-update clear+read, fresh-handle behavior +// after published removal, and dirty-handle retention across multiple future GC runs. +// - Stable validator tests verify consistent published snapshots and post-unlink/pre-deletion +// states. +// - A multithreaded chaos test exercises one writer with multiple busy-loop readers, repeated +// publish/revert/GC cycles, stale-version reads, and traversal safety under deferred deletion. +// +template class VersionedReadable; +template > class VersionedValue; +template class DeferredDeletion; +template > struct VersionedTestAccess; + +// VersionedNode is CRTP type on T, T must inherit from VersionedNode +template class VersionedNode { +public: + template + VersionedNode() + : next_(nullptr), add_version_(versionNotRemoved), remove_version_(versionNotRemoved) {} + +private: + template friend class VersionedReadable; + template friend class VersionedValue; + template friend class DeferredDeletion; + template friend struct VersionedTestAccess; + + bool isUnpublished() const { return add_version_ == versionNotRemoved; } + + void setAddVersion(uint64_t version) { + ASSERT(version < versionNotRemoved); + add_version_ = version; // set before published + } + + void removeInVersion(uint64_t version) { + remove_version_.store(version, std::memory_order_relaxed); + } + + bool isVisibleInVersion(uint64_t version) const { + return add_version_ <= version && version < remove_version_.load(std::memory_order_relaxed); + } + + bool isAddedAfterVersion(uint64_t version) const { return add_version_ > version; } + + bool isAddedBeforeVersion(uint64_t version) const { return add_version_ < version; } + + bool isEventuallyInvisible() const { + return remove_version_.load(std::memory_order_relaxed) < versionNotRemoved; + } + + bool isVisibleOnlyBeforeVersion(uint64_t version) const { + return remove_version_.load(std::memory_order_relaxed) <= version; + } + + bool isRemovedAfterVersion(uint64_t version) const { + const auto remove_version = remove_version_.load(std::memory_order_relaxed); + return remove_version < versionNotRemoved && version < remove_version; + } + + VersionedNode* getNext() const { return next_.load(std::memory_order_acquire); } + + void setNext(VersionedNode* next) { return next_.store(next, std::memory_order_release); } + + VersionedNode* getNextProtected() const { return next_.load(std::memory_order_relaxed); } + std::atomic*>* getNextP() { return &next_; } + + T* value() { return static_cast(this); } + const T* value() const { return static_cast(this); } + + std::atomic*> next_; + + uint64_t add_version_; // Version object was added in. + std::atomic remove_version_; // Version object is removed in. +}; + +template class VersionedReadable { +public: + const T* get(uint64_t version) const { + for (auto node = head_.load(std::memory_order_acquire); node; node = node->getNext()) { + if (node->isVisibleInVersion(version)) { + return node->value(); + } + // stop traveral on first non-visible version that was added before this version, all + // remaining nodes are already invisible. + if (node->isAddedBeforeVersion(version)) { + break; + } + } + return nullptr; // not found + } + +protected: + VersionedReadable() : head_(nullptr) {} + + template friend class VersionedValue; + template friend class DeferredDeletion; + template friend struct VersionedTestAccess; + + std::atomic*> head_; +}; + +// T is the CRTP node type stored in the version chain. R is the read-only interface exposed +// through VersionedHandle: it defaults to VersionedReadable, but callers may provide a richer +// readable base when handles need extra read-side API without exposing the main-thread mutators. +template class VersionedValue : public R { +public: + using Readable = R; + using R::head_; + + VersionedValue() = default; + + template >> + explicit VersionedValue(Args&&... args) : R(std::forward(args)...) {} + + ~VersionedValue() { + VersionedNode* next; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + delete node->value(); + } + } + + // make all versions invisible starting at "version" + void clear(uint64_t version) { + for (auto node = head_.load(std::memory_order_relaxed); node; node = node->getNextProtected()) { + if (!node->isEventuallyInvisible()) { + node->removeInVersion(version); + } + } + } + + void set(uint64_t version, VersionedNode* node) { + // publish a new version that is visible starting at "version" + ASSERT(node->isUnpublished()); // node not pushed into a list yet + node->setAddVersion(version); + node->setNext(head_.load(std::memory_order_relaxed)); + head_.store(node, std::memory_order_release); + + // make all other versions invisible at "version", starting from the first old node. + for (node = node->getNextProtected(); node; node = node->getNextProtected()) { + if (node->isVisibleInVersion(version)) { + node->removeInVersion(version); + } + } + } + + bool empty() const { return head_.load(std::memory_order_relaxed) == nullptr; } + + // Unlink unpublished versions added after 'version' and append them to 'deferred' for deletion + // after an additional grace period. Removed nodes that were still visible in 'version' are + // restored. + void revert(uint64_t version, DeferredDeletion& deferred) { + auto prev = &head_; + VersionedNode* next; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + if (node->isAddedAfterVersion(version)) { + // unlink node by pointing prev to next + prev->store(next, std::memory_order_release); + deferred.push(node); + // prev stays + continue; + } + if (node->isRemovedAfterVersion(version)) { + // visibility was limited, restore + node->removeInVersion(versionNotRemoved); + } + prev = node->getNextP(); + } + } + + // Unlink all versions not visible to anyone after the "version" was published and all readers + // have quiesced, but do not delete them yet. Unlinked nodes are appended to 'deferred' and are + // only deleted after one more worker-thread quiescence round. Any nodes not yet visible at + // "version" must not be unlinked. + // Returns true if there are no future version removals left in this handle after this first + // phase GC run. + bool gcForVersion(uint64_t version, DeferredDeletion& deferred) { + ASSERT(version < versionNotRemoved); + auto prev = &head_; + VersionedNode* next; + bool has_future_gc_work = false; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + if (node->isVisibleOnlyBeforeVersion(version)) { + // unlink node by pointing prev to next + prev->store(next, std::memory_order_release); + deferred.push(node); + // prev stays + continue; + } + if (node->isEventuallyInvisible()) { + has_future_gc_work = true; + } + prev = node->getNextP(); + } + return !has_future_gc_work; + } + +protected: + friend class DeferredDeletion; + template friend struct VersionedTestAccess; +}; + +template class DeferredDeletion : protected VersionedValue { +public: + // Bring `head_` from the templated base class into this scope so the rest of the code can use + // normal unqualified member access. Without this, dependent-base lookup would require + // `this->head_` everywhere. + using VersionedValue::head_; + + DeferredDeletion() = default; + DeferredDeletion(const DeferredDeletion&) = delete; + DeferredDeletion& operator=(const DeferredDeletion&) = delete; + DeferredDeletion(DeferredDeletion&& other) noexcept { + head_.store(other.head_.exchange(nullptr, std::memory_order_relaxed), + std::memory_order_relaxed); + tail_ = std::exchange(other.tail_, nullptr); + } + DeferredDeletion& operator=(DeferredDeletion&&) noexcept = delete; + ~DeferredDeletion() = default; + + bool empty() const { return head_.load(std::memory_order_relaxed) == nullptr; } + +private: + template friend class VersionedValue; + + void push(VersionedNode* node) { + node->setNext(nullptr); + if (tail_) { + tail_->setNext(node); + } else { + // Only the main thread is accessing the deferred-deletion head, hence relaxed. + head_.store(node, std::memory_order_relaxed); + } + tail_ = node; + } + + VersionedNode* tail_{nullptr}; +}; + +template > +using VersionedHandle = std::shared_ptr; + +// VersionedMap is main-thread / exclusive-writer only: map_, pending_keys_, dirty_values_, and +// all mutation methods are accessed only by the single publishing thread. Worker threads only use +// the published Handle objects and then call the read-only handle API. +template > class VersionedMap { +public: + static_assert(std::is_base_of_v); + static_assert(std::is_base_of_v, typename V::Readable>); + using Handle = VersionedHandle; + + uint64_t getVersion() const { return published_version_.load(std::memory_order_acquire); } + + Handle find(const K& key) const { + auto it = map_.find(key); + if (it != map_.cend()) { + return it->second; + } + return nullptr; + } + + uint64_t prepareNextVersion() { + ASSERT(pending_keys_.empty()); + return ++next_version_; + } + + Handle insert(const K& key, T* value) { + ASSERT(next_version_ > published_version_.load(std::memory_order_relaxed)); + auto [it, inserted] = map_.try_emplace(key); + if (inserted) { + if constexpr (std::is_constructible_v) { + it->second = std::make_shared(key); + } else { + it->second = std::make_shared(); + } + } + auto& handle = it->second; + handle->set(next_version_, value); + pending_keys_.emplace(key); + return handle; + } + + void clear(const K& key) { + ASSERT(next_version_ > published_version_.load(std::memory_order_relaxed)); + auto it = map_.find(key); + if (it != map_.cend()) { + it->second->clear(next_version_); + pending_keys_.emplace(key); + } + } + + // returns true if changes need to be reverted or published. Otherwise + // resets the next version back to current version. + bool resetNextVersionIfNotChanged() { + auto version = published_version_.load(std::memory_order_relaxed); + if (next_version_ <= version || pending_keys_.empty()) { + // no changes, revert back to the published version + next_version_ = version; + pending_keys_.clear(); + return false; + } + return true; + } + + // returns the newly published version, or 0 if there was nothing to publish + uint64_t publishNextVersion() { + auto version = published_version_.load(std::memory_order_relaxed); + if (next_version_ <= version || pending_keys_.empty()) { + // no changes, revert back to the published version + next_version_ = version; + pending_keys_.clear(); + return 0; + } + for (const auto& key : pending_keys_) { + auto it = map_.find(key); + ASSERT(it != map_.end()); + dirty_values_.emplace(it->second); + if (it->second->get(next_version_) == nullptr) { + map_.erase(it); + } + } + published_version_.store(next_version_, std::memory_order_release); + pending_keys_.clear(); + return next_version_; + } + + DeferredDeletion revert() { + auto version = published_version_.load(std::memory_order_relaxed); + DeferredDeletion deferred; + for (const auto& key : pending_keys_) { + auto it = map_.find(key); + ASSERT(it != map_.end()); + it->second->revert(version, deferred); + if (it->second->get(version) == nullptr) { + map_.erase(it); + } + } + pending_keys_.clear(); + next_version_ = version; + return deferred; + } + + // First phase GC after all readers have quiesced for 'published_version'. This unlinks nodes that + // are no longer visible and returns them in a deferred-deletion batch that must survive until all + // readers have quiesced once more before it is destroyed. + DeferredDeletion gc(uint64_t published_version) { + DeferredDeletion deferred; + for (auto it = dirty_values_.begin(); it != dirty_values_.end();) { + const auto& handle = *it; + if (handle->gcForVersion(published_version, deferred)) { + dirty_values_.erase(it++); + continue; + } + ++it; + } + return deferred; + } + +private: + friend struct VersionedTestAccess; + using MutableHandle = std::shared_ptr; + + // Persistent published state. + std::atomic published_version_{versionMin}; + absl::flat_hash_map map_; + absl::flat_hash_set dirty_values_; + + // Transactional state for the currently prepared unpublished update. + // Valid only after prepareNextVersion() and until publishNextVersion() or revert(). + // `next_version_` is the candidate version being built, and `pending_keys_` + // contains the keys touched in that candidate update. The container is reused + // across updates to avoid repeated allocation churn. + uint64_t next_version_{versionMin}; + absl::flat_hash_set pending_keys_; +}; + +template struct VersionedTestAccess { + using Map = VersionedMap; + using Handle = typename Map::Handle; + using MutableHandle = typename Map::MutableHandle; + using Node = VersionedNode; + + static const absl::flat_hash_map& entries(const Map& map) { return map.map_; } + + static const absl::flat_hash_set& dirtyValues(const Map& map) { + return map.dirty_values_; + } + + static const Node* head(const Handle& handle) { + if (handle == nullptr) { + return nullptr; + } + auto value = std::static_pointer_cast(handle); + return value->head_.load(std::memory_order_acquire); + } + + static const Node* next(const Node* node) { + return node ? node->next_.load(std::memory_order_acquire) : nullptr; + } + + static const T* value(const Node* node) { return node ? node->value() : nullptr; } + + static uint64_t addVersion(const Node* node) { return node->add_version_; } + + static uint64_t removeVersion(const Node* node) { + return node->remove_version_.load(std::memory_order_relaxed); + } + + static bool isVisibleInVersion(const Node* node, uint64_t version) { + return node && node->isVisibleInVersion(version); + } + + static bool isAddedAfterVersion(const Node* node, uint64_t version) { + return node && node->isAddedAfterVersion(version); + } +}; diff --git a/go/cilium/api/bpf_metadata.pb.go b/go/cilium/api/bpf_metadata.pb.go index 81281988a..22993b3cd 100644 --- a/go/cilium/api/bpf_metadata.pb.go +++ b/go/cilium/api/bpf_metadata.pb.go @@ -24,6 +24,53 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Policy type to use +type BpfMetadata_PolicyType int32 + +const ( + BpfMetadata_NPDS BpfMetadata_PolicyType = 0 // Legacy NPDS (default) + BpfMetadata_NPRDS BpfMetadata_PolicyType = 1 // New NetworkPolicyResource (NPRDS) +) + +// Enum value maps for BpfMetadata_PolicyType. +var ( + BpfMetadata_PolicyType_name = map[int32]string{ + 0: "NPDS", + 1: "NPRDS", + } + BpfMetadata_PolicyType_value = map[string]int32{ + "NPDS": 0, + "NPRDS": 1, + } +) + +func (x BpfMetadata_PolicyType) Enum() *BpfMetadata_PolicyType { + p := new(BpfMetadata_PolicyType) + *p = x + return p +} + +func (x BpfMetadata_PolicyType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (BpfMetadata_PolicyType) Descriptor() protoreflect.EnumDescriptor { + return file_cilium_api_bpf_metadata_proto_enumTypes[0].Descriptor() +} + +func (BpfMetadata_PolicyType) Type() protoreflect.EnumType { + return &file_cilium_api_bpf_metadata_proto_enumTypes[0] +} + +func (x BpfMetadata_PolicyType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use BpfMetadata_PolicyType.Descriptor instead. +func (BpfMetadata_PolicyType) EnumDescriptor() ([]byte, []int) { + return file_cilium_api_bpf_metadata_proto_rawDescGZIP(), []int{0, 0} +} + type BpfMetadata struct { state protoimpl.MessageState `protogen:"open.v1"` // File system root for bpf. Bpf will not be used if left empty. @@ -90,6 +137,7 @@ type BpfMetadata struct { // Configuration for the source of Cilium xDS updates. // Used for all cilium-specific xDS protocol, e.g., NPHDS, NPDS, and Secrets (SDS) therein. CiliumConfigSource *v3.ConfigSource `protobuf:"bytes,16,opt,name=cilium_config_source,json=ciliumConfigSource,proto3" json:"cilium_config_source,omitempty"` + PolicyType bool `protobuf:"varint,17,opt,name=policy_type,json=policyType,proto3" json:"policy_type,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -236,11 +284,18 @@ func (x *BpfMetadata) GetCiliumConfigSource() *v3.ConfigSource { return nil } +func (x *BpfMetadata) GetPolicyType() bool { + if x != nil { + return x.PolicyType + } + return false +} + var File_cilium_api_bpf_metadata_proto protoreflect.FileDescriptor const file_cilium_api_bpf_metadata_proto_rawDesc = "" + "\n" + - "\x1dcilium/api/bpf_metadata.proto\x12\x06cilium\x1a(envoy/config/core/v3/config_source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x17validate/validate.proto\"\xea\x06\n" + + "\x1dcilium/api/bpf_metadata.proto\x12\x06cilium\x1a(envoy/config/core/v3/config_source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x17validate/validate.proto\"\xae\a\n" + "\vBpfMetadata\x12\x19\n" + "\bbpf_root\x18\x01 \x01(\tR\abpfRoot\x12\x1d\n" + "\n" + @@ -259,7 +314,13 @@ const file_cilium_api_bpf_metadata_proto_rawDesc = "" + "\tuse_nphds\x18\r \x01(\bR\buseNphds\x12A\n" + "\x0fcache_entry_ttl\x18\x0e \x01(\v2\x19.google.protobuf.DurationR\rcacheEntryTtl\x12E\n" + "\x11cache_gc_interval\x18\x0f \x01(\v2\x19.google.protobuf.DurationR\x0fcacheGcInterval\x12T\n" + - "\x14cilium_config_source\x18\x10 \x01(\v2\".envoy.config.core.v3.ConfigSourceR\x12ciliumConfigSourceB!\n" + + "\x14cilium_config_source\x18\x10 \x01(\v2\".envoy.config.core.v3.ConfigSourceR\x12ciliumConfigSource\x12\x1f\n" + + "\vpolicy_type\x18\x11 \x01(\bR\n" + + "policyType\"!\n" + + "\n" + + "PolicyType\x12\b\n" + + "\x04NPDS\x10\x00\x12\t\n" + + "\x05NPRDS\x10\x01B!\n" + "\x1f_original_source_so_linger_timeB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( @@ -274,17 +335,19 @@ func file_cilium_api_bpf_metadata_proto_rawDescGZIP() []byte { return file_cilium_api_bpf_metadata_proto_rawDescData } +var file_cilium_api_bpf_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_cilium_api_bpf_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_cilium_api_bpf_metadata_proto_goTypes = []any{ - (*BpfMetadata)(nil), // 0: cilium.BpfMetadata - (*durationpb.Duration)(nil), // 1: google.protobuf.Duration - (*v3.ConfigSource)(nil), // 2: envoy.config.core.v3.ConfigSource + (BpfMetadata_PolicyType)(0), // 0: cilium.BpfMetadata.PolicyType + (*BpfMetadata)(nil), // 1: cilium.BpfMetadata + (*durationpb.Duration)(nil), // 2: google.protobuf.Duration + (*v3.ConfigSource)(nil), // 3: envoy.config.core.v3.ConfigSource } var file_cilium_api_bpf_metadata_proto_depIdxs = []int32{ - 1, // 0: cilium.BpfMetadata.policy_update_warning_limit:type_name -> google.protobuf.Duration - 1, // 1: cilium.BpfMetadata.cache_entry_ttl:type_name -> google.protobuf.Duration - 1, // 2: cilium.BpfMetadata.cache_gc_interval:type_name -> google.protobuf.Duration - 2, // 3: cilium.BpfMetadata.cilium_config_source:type_name -> envoy.config.core.v3.ConfigSource + 2, // 0: cilium.BpfMetadata.policy_update_warning_limit:type_name -> google.protobuf.Duration + 2, // 1: cilium.BpfMetadata.cache_entry_ttl:type_name -> google.protobuf.Duration + 2, // 2: cilium.BpfMetadata.cache_gc_interval:type_name -> google.protobuf.Duration + 3, // 3: cilium.BpfMetadata.cilium_config_source:type_name -> envoy.config.core.v3.ConfigSource 4, // [4:4] is the sub-list for method output_type 4, // [4:4] is the sub-list for method input_type 4, // [4:4] is the sub-list for extension type_name @@ -303,13 +366,14 @@ func file_cilium_api_bpf_metadata_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cilium_api_bpf_metadata_proto_rawDesc), len(file_cilium_api_bpf_metadata_proto_rawDesc)), - NumEnums: 0, + NumEnums: 1, NumMessages: 1, NumExtensions: 0, NumServices: 0, }, GoTypes: file_cilium_api_bpf_metadata_proto_goTypes, DependencyIndexes: file_cilium_api_bpf_metadata_proto_depIdxs, + EnumInfos: file_cilium_api_bpf_metadata_proto_enumTypes, MessageInfos: file_cilium_api_bpf_metadata_proto_msgTypes, }.Build() File_cilium_api_bpf_metadata_proto = out.File diff --git a/go/cilium/api/bpf_metadata.pb.validate.go b/go/cilium/api/bpf_metadata.pb.validate.go index 8dc8b53aa..b8f72a926 100644 --- a/go/cilium/api/bpf_metadata.pb.validate.go +++ b/go/cilium/api/bpf_metadata.pb.validate.go @@ -204,6 +204,8 @@ func (m *BpfMetadata) validate(all bool) error { } } + // no validation rules for PolicyType + if m.OriginalSourceSoLingerTime != nil { // no validation rules for OriginalSourceSoLingerTime } diff --git a/go/cilium/api/npds.pb.go b/go/cilium/api/npds.pb.go index 5fad4195e..ad1714305 100644 --- a/go/cilium/api/npds.pb.go +++ b/go/cilium/api/npds.pb.go @@ -75,7 +75,7 @@ func (x HeaderMatch_MatchAction) Number() protoreflect.EnumNumber { // Deprecated: Use HeaderMatch_MatchAction.Descriptor instead. func (HeaderMatch_MatchAction) EnumDescriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5, 0} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7, 0} } type HeaderMatch_MismatchAction int32 @@ -130,7 +130,139 @@ func (x HeaderMatch_MismatchAction) Number() protoreflect.EnumNumber { // Deprecated: Use HeaderMatch_MismatchAction.Descriptor instead. func (HeaderMatch_MismatchAction) EnumDescriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5, 1} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7, 1} +} + +// An NPRDS resource that carries either an endpoint policy or a shared selector. +type NetworkPolicyResource struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Resource: + // + // *NetworkPolicyResource_Policy + // *NetworkPolicyResource_Selector + Resource isNetworkPolicyResource_Resource `protobuf_oneof:"resource"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NetworkPolicyResource) Reset() { + *x = NetworkPolicyResource{} + mi := &file_cilium_api_npds_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NetworkPolicyResource) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NetworkPolicyResource) ProtoMessage() {} + +func (x *NetworkPolicyResource) ProtoReflect() protoreflect.Message { + mi := &file_cilium_api_npds_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NetworkPolicyResource.ProtoReflect.Descriptor instead. +func (*NetworkPolicyResource) Descriptor() ([]byte, []int) { + return file_cilium_api_npds_proto_rawDescGZIP(), []int{0} +} + +func (x *NetworkPolicyResource) GetResource() isNetworkPolicyResource_Resource { + if x != nil { + return x.Resource + } + return nil +} + +func (x *NetworkPolicyResource) GetPolicy() *NetworkPolicy { + if x != nil { + if x, ok := x.Resource.(*NetworkPolicyResource_Policy); ok { + return x.Policy + } + } + return nil +} + +func (x *NetworkPolicyResource) GetSelector() *Selector { + if x != nil { + if x, ok := x.Resource.(*NetworkPolicyResource_Selector); ok { + return x.Selector + } + } + return nil +} + +type isNetworkPolicyResource_Resource interface { + isNetworkPolicyResource_Resource() +} + +type NetworkPolicyResource_Policy struct { + Policy *NetworkPolicy `protobuf:"bytes,1,opt,name=policy,proto3,oneof"` +} + +type NetworkPolicyResource_Selector struct { + Selector *Selector `protobuf:"bytes,2,opt,name=selector,proto3,oneof"` +} + +func (*NetworkPolicyResource_Policy) isNetworkPolicyResource_Resource() {} + +func (*NetworkPolicyResource_Selector) isNetworkPolicyResource_Resource() {} + +// A shared set of remote identities referenced by selector resource name. +// Unlike the old state-of-the-world remote identity lists, an empty selector +// matches nothing. +type Selector struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The set of numeric remote security IDs selected by this selector. + // If empty, this selector selects no remote identities. + RemoteIdentities []uint32 `protobuf:"varint,1,rep,packed,name=remote_identities,json=remoteIdentities,proto3" json:"remote_identities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Selector) Reset() { + *x = Selector{} + mi := &file_cilium_api_npds_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Selector) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Selector) ProtoMessage() {} + +func (x *Selector) ProtoReflect() protoreflect.Message { + mi := &file_cilium_api_npds_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Selector.ProtoReflect.Descriptor instead. +func (*Selector) Descriptor() ([]byte, []int) { + return file_cilium_api_npds_proto_rawDescGZIP(), []int{1} +} + +func (x *Selector) GetRemoteIdentities() []uint32 { + if x != nil { + return x.RemoteIdentities + } + return nil } // A network policy that is enforced by a filter on the network flows to/from @@ -161,7 +293,7 @@ type NetworkPolicy struct { func (x *NetworkPolicy) Reset() { *x = NetworkPolicy{} - mi := &file_cilium_api_npds_proto_msgTypes[0] + mi := &file_cilium_api_npds_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -173,7 +305,7 @@ func (x *NetworkPolicy) String() string { func (*NetworkPolicy) ProtoMessage() {} func (x *NetworkPolicy) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[0] + mi := &file_cilium_api_npds_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -186,7 +318,7 @@ func (x *NetworkPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkPolicy.ProtoReflect.Descriptor instead. func (*NetworkPolicy) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{0} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{2} } func (x *NetworkPolicy) GetEndpointIps() []string { @@ -240,7 +372,7 @@ type PortNetworkPolicy struct { func (x *PortNetworkPolicy) Reset() { *x = PortNetworkPolicy{} - mi := &file_cilium_api_npds_proto_msgTypes[1] + mi := &file_cilium_api_npds_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -252,7 +384,7 @@ func (x *PortNetworkPolicy) String() string { func (*PortNetworkPolicy) ProtoMessage() {} func (x *PortNetworkPolicy) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[1] + mi := &file_cilium_api_npds_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -265,7 +397,7 @@ func (x *PortNetworkPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use PortNetworkPolicy.ProtoReflect.Descriptor instead. func (*PortNetworkPolicy) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{1} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{3} } func (x *PortNetworkPolicy) GetPort() uint32 { @@ -328,7 +460,7 @@ type TLSContext struct { func (x *TLSContext) Reset() { *x = TLSContext{} - mi := &file_cilium_api_npds_proto_msgTypes[2] + mi := &file_cilium_api_npds_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -340,7 +472,7 @@ func (x *TLSContext) String() string { func (*TLSContext) ProtoMessage() {} func (x *TLSContext) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[2] + mi := &file_cilium_api_npds_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -353,7 +485,7 @@ func (x *TLSContext) ProtoReflect() protoreflect.Message { // Deprecated: Use TLSContext.ProtoReflect.Descriptor instead. func (*TLSContext) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{2} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{4} } func (x *TLSContext) GetTrustedCa() string { @@ -433,6 +565,11 @@ type PortNetworkPolicyRule struct { // applied on the flow's remote host is contained in this set. // Optional. If not specified, any remote host is matched by this predicate. RemotePolicies []uint32 `protobuf:"varint,7,rep,packed,name=remote_policies,json=remotePolicies,proto3" json:"remote_policies,omitempty"` + // Optional selector resource names that can be resolved to shared remote + // policy sets in delta NPDS. + // Selector references are matched by exact selector resource name. + // Optional. If not specified, any remote host is matched by this predicate. + Selectors []string `protobuf:"bytes,11,rep,name=selectors,proto3" json:"selectors,omitempty"` // Optional downstream TLS context. If present, the incoming connection must // be a TLS connection. DownstreamTlsContext *TLSContext `protobuf:"bytes,3,opt,name=downstream_tls_context,json=downstreamTlsContext,proto3" json:"downstream_tls_context,omitempty"` @@ -473,7 +610,7 @@ type PortNetworkPolicyRule struct { func (x *PortNetworkPolicyRule) Reset() { *x = PortNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[3] + mi := &file_cilium_api_npds_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -485,7 +622,7 @@ func (x *PortNetworkPolicyRule) String() string { func (*PortNetworkPolicyRule) ProtoMessage() {} func (x *PortNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[3] + mi := &file_cilium_api_npds_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -498,7 +635,7 @@ func (x *PortNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use PortNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*PortNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{3} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{5} } func (x *PortNetworkPolicyRule) GetPrecedence() uint32 { @@ -554,6 +691,13 @@ func (x *PortNetworkPolicyRule) GetRemotePolicies() []uint32 { return nil } +func (x *PortNetworkPolicyRule) GetSelectors() []string { + if x != nil { + return x.Selectors + } + return nil +} + func (x *PortNetworkPolicyRule) GetDownstreamTlsContext() *TLSContext { if x != nil { return x.DownstreamTlsContext @@ -679,7 +823,7 @@ type HttpNetworkPolicyRules struct { func (x *HttpNetworkPolicyRules) Reset() { *x = HttpNetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[4] + mi := &file_cilium_api_npds_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -691,7 +835,7 @@ func (x *HttpNetworkPolicyRules) String() string { func (*HttpNetworkPolicyRules) ProtoMessage() {} func (x *HttpNetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[4] + mi := &file_cilium_api_npds_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -704,7 +848,7 @@ func (x *HttpNetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpNetworkPolicyRules.ProtoReflect.Descriptor instead. func (*HttpNetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{4} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{6} } func (x *HttpNetworkPolicyRules) GetHttpRules() []*HttpNetworkPolicyRule { @@ -729,7 +873,7 @@ type HeaderMatch struct { func (x *HeaderMatch) Reset() { *x = HeaderMatch{} - mi := &file_cilium_api_npds_proto_msgTypes[5] + mi := &file_cilium_api_npds_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -741,7 +885,7 @@ func (x *HeaderMatch) String() string { func (*HeaderMatch) ProtoMessage() {} func (x *HeaderMatch) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[5] + mi := &file_cilium_api_npds_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -754,7 +898,7 @@ func (x *HeaderMatch) ProtoReflect() protoreflect.Message { // Deprecated: Use HeaderMatch.ProtoReflect.Descriptor instead. func (*HeaderMatch) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7} } func (x *HeaderMatch) GetName() string { @@ -822,7 +966,7 @@ type HttpNetworkPolicyRule struct { func (x *HttpNetworkPolicyRule) Reset() { *x = HttpNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[6] + mi := &file_cilium_api_npds_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -834,7 +978,7 @@ func (x *HttpNetworkPolicyRule) String() string { func (*HttpNetworkPolicyRule) ProtoMessage() {} func (x *HttpNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[6] + mi := &file_cilium_api_npds_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -847,7 +991,7 @@ func (x *HttpNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*HttpNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{6} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{8} } func (x *HttpNetworkPolicyRule) GetHeaders() []*v31.HeaderMatcher { @@ -877,7 +1021,7 @@ type KafkaNetworkPolicyRules struct { func (x *KafkaNetworkPolicyRules) Reset() { *x = KafkaNetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[7] + mi := &file_cilium_api_npds_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -889,7 +1033,7 @@ func (x *KafkaNetworkPolicyRules) String() string { func (*KafkaNetworkPolicyRules) ProtoMessage() {} func (x *KafkaNetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[7] + mi := &file_cilium_api_npds_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -902,7 +1046,7 @@ func (x *KafkaNetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use KafkaNetworkPolicyRules.ProtoReflect.Descriptor instead. func (*KafkaNetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{7} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{9} } func (x *KafkaNetworkPolicyRules) GetKafkaRules() []*KafkaNetworkPolicyRule { @@ -941,7 +1085,7 @@ type KafkaNetworkPolicyRule struct { func (x *KafkaNetworkPolicyRule) Reset() { *x = KafkaNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[8] + mi := &file_cilium_api_npds_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -953,7 +1097,7 @@ func (x *KafkaNetworkPolicyRule) String() string { func (*KafkaNetworkPolicyRule) ProtoMessage() {} func (x *KafkaNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[8] + mi := &file_cilium_api_npds_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -966,7 +1110,7 @@ func (x *KafkaNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use KafkaNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*KafkaNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{8} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{10} } func (x *KafkaNetworkPolicyRule) GetApiVersion() int32 { @@ -1017,7 +1161,7 @@ type L7NetworkPolicyRules struct { func (x *L7NetworkPolicyRules) Reset() { *x = L7NetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[9] + mi := &file_cilium_api_npds_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1029,7 +1173,7 @@ func (x *L7NetworkPolicyRules) String() string { func (*L7NetworkPolicyRules) ProtoMessage() {} func (x *L7NetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[9] + mi := &file_cilium_api_npds_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1042,7 +1186,7 @@ func (x *L7NetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use L7NetworkPolicyRules.ProtoReflect.Descriptor instead. func (*L7NetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{9} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{11} } func (x *L7NetworkPolicyRules) GetL7AllowRules() []*L7NetworkPolicyRule { @@ -1080,7 +1224,7 @@ type L7NetworkPolicyRule struct { func (x *L7NetworkPolicyRule) Reset() { *x = L7NetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[10] + mi := &file_cilium_api_npds_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1092,7 +1236,7 @@ func (x *L7NetworkPolicyRule) String() string { func (*L7NetworkPolicyRule) ProtoMessage() {} func (x *L7NetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[10] + mi := &file_cilium_api_npds_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1105,7 +1249,7 @@ func (x *L7NetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use L7NetworkPolicyRule.ProtoReflect.Descriptor instead. func (*L7NetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{10} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{12} } func (x *L7NetworkPolicyRule) GetName() string { @@ -1140,7 +1284,7 @@ type NetworkPoliciesConfigDump struct { func (x *NetworkPoliciesConfigDump) Reset() { *x = NetworkPoliciesConfigDump{} - mi := &file_cilium_api_npds_proto_msgTypes[11] + mi := &file_cilium_api_npds_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1152,7 +1296,7 @@ func (x *NetworkPoliciesConfigDump) String() string { func (*NetworkPoliciesConfigDump) ProtoMessage() {} func (x *NetworkPoliciesConfigDump) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[11] + mi := &file_cilium_api_npds_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1165,7 +1309,7 @@ func (x *NetworkPoliciesConfigDump) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkPoliciesConfigDump.ProtoReflect.Descriptor instead. func (*NetworkPoliciesConfigDump) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{11} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{13} } func (x *NetworkPoliciesConfigDump) GetNetworkpolicies() []*NetworkPolicy { @@ -1179,7 +1323,14 @@ var File_cilium_api_npds_proto protoreflect.FileDescriptor const file_cilium_api_npds_proto_rawDesc = "" + "\n" + - "\x15cilium/api/npds.proto\x12\x06cilium\x1a\"envoy/config/core/v3/address.proto\x1a,envoy/config/route/v3/route_components.proto\x1a*envoy/service/discovery/v3/discovery.proto\x1a$envoy/type/matcher/v3/metadata.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"\x95\x02\n" + + "\x15cilium/api/npds.proto\x12\x06cilium\x1a\"envoy/config/core/v3/address.proto\x1a,envoy/config/route/v3/route_components.proto\x1a*envoy/service/discovery/v3/discovery.proto\x1a$envoy/type/matcher/v3/metadata.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"\x84\x01\n" + + "\x15NetworkPolicyResource\x12/\n" + + "\x06policy\x18\x01 \x01(\v2\x15.cilium.NetworkPolicyH\x00R\x06policy\x12.\n" + + "\bselector\x18\x02 \x01(\v2\x10.cilium.SelectorH\x00R\bselectorB\n" + + "\n" + + "\bresource\"7\n" + + "\bSelector\x12+\n" + + "\x11remote_identities\x18\x01 \x03(\rR\x10remoteIdentities\"\x95\x02\n" + "\rNetworkPolicy\x123\n" + "\fendpoint_ips\x18\x01 \x03(\tB\x10\xfaB\r\x92\x01\n" + "\b\x01\x10\x02\"\x04r\x02\x10\x01R\vendpointIps\x12\x1f\n" + @@ -1202,7 +1353,7 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\fserver_names\x18\x04 \x03(\tR\vserverNames\x12A\n" + "\x1dvalidation_context_sds_secret\x18\x05 \x01(\tR\x1avalidationContextSdsSecret\x12$\n" + "\x0etls_sds_secret\x18\x06 \x01(\tR\ftlsSdsSecret\x12%\n" + - "\x0ealpn_protocols\x18\a \x03(\tR\ralpnProtocols\"\xfb\x05\n" + + "\x0ealpn_protocols\x18\a \x03(\tR\ralpnProtocols\"\x99\x06\n" + "\x15PortNetworkPolicyRule\x12\x1e\n" + "\n" + "precedence\x18\n" + @@ -1212,7 +1363,8 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\x04deny\x18\b \x01(\bH\x00R\x04deny\x12$\n" + "\bproxy_id\x18\t \x01(\rB\t\xfaB\x06*\x04\x18\xff\xff\x03R\aproxyId\x12\x12\n" + "\x04name\x18\x05 \x01(\tR\x04name\x12'\n" + - "\x0fremote_policies\x18\a \x03(\rR\x0eremotePolicies\x12H\n" + + "\x0fremote_policies\x18\a \x03(\rR\x0eremotePolicies\x12\x1c\n" + + "\tselectors\x18\v \x03(\tR\tselectors\x12H\n" + "\x16downstream_tls_context\x18\x03 \x01(\v2\x12.cilium.TLSContextR\x14downstreamTlsContext\x12D\n" + "\x14upstream_tls_context\x18\x04 \x01(\v2\x12.cilium.TLSContextR\x12upstreamTlsContext\x12\xa1\x01\n" + "\fserver_names\x18\x06 \x03(\tB~\xfaB{\x92\x01x\"vrt2r^(([*]{1,2}|[*]?[-a-zA-Z0-9_]+([*][-a-zA-Z0-9_]+)*[*]?)[.])*([*]{1,2}|[*]?[-a-zA-Z0-9_]+([*][-a-zA-Z0-9_]+)*[*]?)$R\vserverNames\x12\x19\n" + @@ -1271,7 +1423,11 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\x14DeltaNetworkPolicies\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x12z\n" + "\x15StreamNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\x9e\x01\n" + "\x14FetchNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\")\x82\xd3\xe4\x93\x02#:\x01*\"\x1e/v3/discovery:network_policies\x1a\x1c\x8a\xa4\x96\xf3\a\x16\n" + - "\x14cilium.NetworkPolicyB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" + "\x14cilium.NetworkPolicy2\xde\x02\n" + + "%NetworkPolicyResourceDiscoveryService\x12\x81\x01\n" + + "\x1cStreamNetworkPolicyResources\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\x8a\x01\n" + + "\x1bDeltaNetworkPolicyResources\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x1a$\x8a\xa4\x96\xf3\a\x1e\n" + + "\x1ccilium.NetworkPolicyResourceB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( file_cilium_api_npds_proto_rawDescOnce sync.Once @@ -1286,63 +1442,71 @@ func file_cilium_api_npds_proto_rawDescGZIP() []byte { } var file_cilium_api_npds_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_cilium_api_npds_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_cilium_api_npds_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_cilium_api_npds_proto_goTypes = []any{ (HeaderMatch_MatchAction)(0), // 0: cilium.HeaderMatch.MatchAction (HeaderMatch_MismatchAction)(0), // 1: cilium.HeaderMatch.MismatchAction - (*NetworkPolicy)(nil), // 2: cilium.NetworkPolicy - (*PortNetworkPolicy)(nil), // 3: cilium.PortNetworkPolicy - (*TLSContext)(nil), // 4: cilium.TLSContext - (*PortNetworkPolicyRule)(nil), // 5: cilium.PortNetworkPolicyRule - (*HttpNetworkPolicyRules)(nil), // 6: cilium.HttpNetworkPolicyRules - (*HeaderMatch)(nil), // 7: cilium.HeaderMatch - (*HttpNetworkPolicyRule)(nil), // 8: cilium.HttpNetworkPolicyRule - (*KafkaNetworkPolicyRules)(nil), // 9: cilium.KafkaNetworkPolicyRules - (*KafkaNetworkPolicyRule)(nil), // 10: cilium.KafkaNetworkPolicyRule - (*L7NetworkPolicyRules)(nil), // 11: cilium.L7NetworkPolicyRules - (*L7NetworkPolicyRule)(nil), // 12: cilium.L7NetworkPolicyRule - (*NetworkPoliciesConfigDump)(nil), // 13: cilium.NetworkPoliciesConfigDump - nil, // 14: cilium.L7NetworkPolicyRule.RuleEntry - (v3.SocketAddress_Protocol)(0), // 15: envoy.config.core.v3.SocketAddress.Protocol - (*v31.HeaderMatcher)(nil), // 16: envoy.config.route.v3.HeaderMatcher - (*v32.MetadataMatcher)(nil), // 17: envoy.type.matcher.v3.MetadataMatcher - (*v33.DeltaDiscoveryRequest)(nil), // 18: envoy.service.discovery.v3.DeltaDiscoveryRequest - (*v33.DiscoveryRequest)(nil), // 19: envoy.service.discovery.v3.DiscoveryRequest - (*v33.DeltaDiscoveryResponse)(nil), // 20: envoy.service.discovery.v3.DeltaDiscoveryResponse - (*v33.DiscoveryResponse)(nil), // 21: envoy.service.discovery.v3.DiscoveryResponse + (*NetworkPolicyResource)(nil), // 2: cilium.NetworkPolicyResource + (*Selector)(nil), // 3: cilium.Selector + (*NetworkPolicy)(nil), // 4: cilium.NetworkPolicy + (*PortNetworkPolicy)(nil), // 5: cilium.PortNetworkPolicy + (*TLSContext)(nil), // 6: cilium.TLSContext + (*PortNetworkPolicyRule)(nil), // 7: cilium.PortNetworkPolicyRule + (*HttpNetworkPolicyRules)(nil), // 8: cilium.HttpNetworkPolicyRules + (*HeaderMatch)(nil), // 9: cilium.HeaderMatch + (*HttpNetworkPolicyRule)(nil), // 10: cilium.HttpNetworkPolicyRule + (*KafkaNetworkPolicyRules)(nil), // 11: cilium.KafkaNetworkPolicyRules + (*KafkaNetworkPolicyRule)(nil), // 12: cilium.KafkaNetworkPolicyRule + (*L7NetworkPolicyRules)(nil), // 13: cilium.L7NetworkPolicyRules + (*L7NetworkPolicyRule)(nil), // 14: cilium.L7NetworkPolicyRule + (*NetworkPoliciesConfigDump)(nil), // 15: cilium.NetworkPoliciesConfigDump + nil, // 16: cilium.L7NetworkPolicyRule.RuleEntry + (v3.SocketAddress_Protocol)(0), // 17: envoy.config.core.v3.SocketAddress.Protocol + (*v31.HeaderMatcher)(nil), // 18: envoy.config.route.v3.HeaderMatcher + (*v32.MetadataMatcher)(nil), // 19: envoy.type.matcher.v3.MetadataMatcher + (*v33.DeltaDiscoveryRequest)(nil), // 20: envoy.service.discovery.v3.DeltaDiscoveryRequest + (*v33.DiscoveryRequest)(nil), // 21: envoy.service.discovery.v3.DiscoveryRequest + (*v33.DeltaDiscoveryResponse)(nil), // 22: envoy.service.discovery.v3.DeltaDiscoveryResponse + (*v33.DiscoveryResponse)(nil), // 23: envoy.service.discovery.v3.DiscoveryResponse } var file_cilium_api_npds_proto_depIdxs = []int32{ - 3, // 0: cilium.NetworkPolicy.ingress_per_port_policies:type_name -> cilium.PortNetworkPolicy - 3, // 1: cilium.NetworkPolicy.egress_per_port_policies:type_name -> cilium.PortNetworkPolicy - 15, // 2: cilium.PortNetworkPolicy.protocol:type_name -> envoy.config.core.v3.SocketAddress.Protocol - 5, // 3: cilium.PortNetworkPolicy.rules:type_name -> cilium.PortNetworkPolicyRule - 4, // 4: cilium.PortNetworkPolicyRule.downstream_tls_context:type_name -> cilium.TLSContext - 4, // 5: cilium.PortNetworkPolicyRule.upstream_tls_context:type_name -> cilium.TLSContext - 6, // 6: cilium.PortNetworkPolicyRule.http_rules:type_name -> cilium.HttpNetworkPolicyRules - 9, // 7: cilium.PortNetworkPolicyRule.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRules - 11, // 8: cilium.PortNetworkPolicyRule.l7_rules:type_name -> cilium.L7NetworkPolicyRules - 8, // 9: cilium.HttpNetworkPolicyRules.http_rules:type_name -> cilium.HttpNetworkPolicyRule - 0, // 10: cilium.HeaderMatch.match_action:type_name -> cilium.HeaderMatch.MatchAction - 1, // 11: cilium.HeaderMatch.mismatch_action:type_name -> cilium.HeaderMatch.MismatchAction - 16, // 12: cilium.HttpNetworkPolicyRule.headers:type_name -> envoy.config.route.v3.HeaderMatcher - 7, // 13: cilium.HttpNetworkPolicyRule.header_matches:type_name -> cilium.HeaderMatch - 10, // 14: cilium.KafkaNetworkPolicyRules.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRule - 12, // 15: cilium.L7NetworkPolicyRules.l7_allow_rules:type_name -> cilium.L7NetworkPolicyRule - 12, // 16: cilium.L7NetworkPolicyRules.l7_deny_rules:type_name -> cilium.L7NetworkPolicyRule - 14, // 17: cilium.L7NetworkPolicyRule.rule:type_name -> cilium.L7NetworkPolicyRule.RuleEntry - 17, // 18: cilium.L7NetworkPolicyRule.metadata_rule:type_name -> envoy.type.matcher.v3.MetadataMatcher - 2, // 19: cilium.NetworkPoliciesConfigDump.networkpolicies:type_name -> cilium.NetworkPolicy - 18, // 20: cilium.NetworkPolicyDiscoveryService.DeltaNetworkPolicies:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest - 19, // 21: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 19, // 22: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 20, // 23: cilium.NetworkPolicyDiscoveryService.DeltaNetworkPolicies:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse - 21, // 24: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 21, // 25: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 23, // [23:26] is the sub-list for method output_type - 20, // [20:23] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 4, // 0: cilium.NetworkPolicyResource.policy:type_name -> cilium.NetworkPolicy + 3, // 1: cilium.NetworkPolicyResource.selector:type_name -> cilium.Selector + 5, // 2: cilium.NetworkPolicy.ingress_per_port_policies:type_name -> cilium.PortNetworkPolicy + 5, // 3: cilium.NetworkPolicy.egress_per_port_policies:type_name -> cilium.PortNetworkPolicy + 17, // 4: cilium.PortNetworkPolicy.protocol:type_name -> envoy.config.core.v3.SocketAddress.Protocol + 7, // 5: cilium.PortNetworkPolicy.rules:type_name -> cilium.PortNetworkPolicyRule + 6, // 6: cilium.PortNetworkPolicyRule.downstream_tls_context:type_name -> cilium.TLSContext + 6, // 7: cilium.PortNetworkPolicyRule.upstream_tls_context:type_name -> cilium.TLSContext + 8, // 8: cilium.PortNetworkPolicyRule.http_rules:type_name -> cilium.HttpNetworkPolicyRules + 11, // 9: cilium.PortNetworkPolicyRule.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRules + 13, // 10: cilium.PortNetworkPolicyRule.l7_rules:type_name -> cilium.L7NetworkPolicyRules + 10, // 11: cilium.HttpNetworkPolicyRules.http_rules:type_name -> cilium.HttpNetworkPolicyRule + 0, // 12: cilium.HeaderMatch.match_action:type_name -> cilium.HeaderMatch.MatchAction + 1, // 13: cilium.HeaderMatch.mismatch_action:type_name -> cilium.HeaderMatch.MismatchAction + 18, // 14: cilium.HttpNetworkPolicyRule.headers:type_name -> envoy.config.route.v3.HeaderMatcher + 9, // 15: cilium.HttpNetworkPolicyRule.header_matches:type_name -> cilium.HeaderMatch + 12, // 16: cilium.KafkaNetworkPolicyRules.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRule + 14, // 17: cilium.L7NetworkPolicyRules.l7_allow_rules:type_name -> cilium.L7NetworkPolicyRule + 14, // 18: cilium.L7NetworkPolicyRules.l7_deny_rules:type_name -> cilium.L7NetworkPolicyRule + 16, // 19: cilium.L7NetworkPolicyRule.rule:type_name -> cilium.L7NetworkPolicyRule.RuleEntry + 19, // 20: cilium.L7NetworkPolicyRule.metadata_rule:type_name -> envoy.type.matcher.v3.MetadataMatcher + 4, // 21: cilium.NetworkPoliciesConfigDump.networkpolicies:type_name -> cilium.NetworkPolicy + 20, // 22: cilium.NetworkPolicyDiscoveryService.DeltaNetworkPolicies:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 21, // 23: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 21, // 24: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 21, // 25: cilium.NetworkPolicyResourceDiscoveryService.StreamNetworkPolicyResources:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 20, // 26: cilium.NetworkPolicyResourceDiscoveryService.DeltaNetworkPolicyResources:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 22, // 27: cilium.NetworkPolicyDiscoveryService.DeltaNetworkPolicies:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 23, // 28: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 23, // 29: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 23, // 30: cilium.NetworkPolicyResourceDiscoveryService.StreamNetworkPolicyResources:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 22, // 31: cilium.NetworkPolicyResourceDiscoveryService.DeltaNetworkPolicyResources:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 27, // [27:32] is the sub-list for method output_type + 22, // [22:27] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_cilium_api_npds_proto_init() } @@ -1350,7 +1514,11 @@ func file_cilium_api_npds_proto_init() { if File_cilium_api_npds_proto != nil { return } - file_cilium_api_npds_proto_msgTypes[3].OneofWrappers = []any{ + file_cilium_api_npds_proto_msgTypes[0].OneofWrappers = []any{ + (*NetworkPolicyResource_Policy)(nil), + (*NetworkPolicyResource_Selector)(nil), + } + file_cilium_api_npds_proto_msgTypes[5].OneofWrappers = []any{ (*PortNetworkPolicyRule_PassPrecedence)(nil), (*PortNetworkPolicyRule_Deny)(nil), (*PortNetworkPolicyRule_HttpRules)(nil), @@ -1363,9 +1531,9 @@ func file_cilium_api_npds_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cilium_api_npds_proto_rawDesc), len(file_cilium_api_npds_proto_rawDesc)), NumEnums: 2, - NumMessages: 13, + NumMessages: 15, NumExtensions: 0, - NumServices: 1, + NumServices: 2, }, GoTypes: file_cilium_api_npds_proto_goTypes, DependencyIndexes: file_cilium_api_npds_proto_depIdxs, diff --git a/go/cilium/api/npds.pb.validate.go b/go/cilium/api/npds.pb.validate.go index 418649f39..ea344bc3f 100644 --- a/go/cilium/api/npds.pb.validate.go +++ b/go/cilium/api/npds.pb.validate.go @@ -39,6 +39,294 @@ var ( _ = corev3.SocketAddress_Protocol(0) ) +// Validate checks the field values on NetworkPolicyResource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *NetworkPolicyResource) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on NetworkPolicyResource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// NetworkPolicyResourceMultiError, or nil if none found. +func (m *NetworkPolicyResource) ValidateAll() error { + return m.validate(true) +} + +func (m *NetworkPolicyResource) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + switch v := m.Resource.(type) { + case *NetworkPolicyResource_Policy: + if v == nil { + err := NetworkPolicyResourceValidationError{ + field: "Resource", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetPolicy()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetPolicy()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + } + } + } + + case *NetworkPolicyResource_Selector: + if v == nil { + err := NetworkPolicyResourceValidationError{ + field: "Resource", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetSelector()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetSelector()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + } + } + } + + default: + _ = v // ensures v is used + } + + if len(errors) > 0 { + return NetworkPolicyResourceMultiError(errors) + } + + return nil +} + +// NetworkPolicyResourceMultiError is an error wrapping multiple validation +// errors returned by NetworkPolicyResource.ValidateAll() if the designated +// constraints aren't met. +type NetworkPolicyResourceMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m NetworkPolicyResourceMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m NetworkPolicyResourceMultiError) AllErrors() []error { return m } + +// NetworkPolicyResourceValidationError is the validation error returned by +// NetworkPolicyResource.Validate if the designated constraints aren't met. +type NetworkPolicyResourceValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e NetworkPolicyResourceValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e NetworkPolicyResourceValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e NetworkPolicyResourceValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e NetworkPolicyResourceValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e NetworkPolicyResourceValidationError) ErrorName() string { + return "NetworkPolicyResourceValidationError" +} + +// Error satisfies the builtin error interface +func (e NetworkPolicyResourceValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sNetworkPolicyResource.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = NetworkPolicyResourceValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = NetworkPolicyResourceValidationError{} + +// Validate checks the field values on Selector with the rules defined in the +// proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *Selector) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on Selector with the rules defined in +// the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in SelectorMultiError, or nil +// if none found. +func (m *Selector) ValidateAll() error { + return m.validate(true) +} + +func (m *Selector) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SelectorMultiError(errors) + } + + return nil +} + +// SelectorMultiError is an error wrapping multiple validation errors returned +// by Selector.ValidateAll() if the designated constraints aren't met. +type SelectorMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SelectorMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SelectorMultiError) AllErrors() []error { return m } + +// SelectorValidationError is the validation error returned by +// Selector.Validate if the designated constraints aren't met. +type SelectorValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SelectorValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SelectorValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SelectorValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SelectorValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SelectorValidationError) ErrorName() string { return "SelectorValidationError" } + +// Error satisfies the builtin error interface +func (e SelectorValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSelector.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SelectorValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SelectorValidationError{} + // Validate checks the field values on NetworkPolicy with the rules defined in // the proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. diff --git a/go/cilium/api/npds_grpc.pb.go b/go/cilium/api/npds_grpc.pb.go index 98ee186bf..edda91e1c 100644 --- a/go/cilium/api/npds_grpc.pb.go +++ b/go/cilium/api/npds_grpc.pb.go @@ -30,6 +30,7 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. type NetworkPolicyDiscoveryServiceClient interface { DeltaNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) StreamNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) @@ -85,6 +86,7 @@ func (c *networkPolicyDiscoveryServiceClient) FetchNetworkPolicies(ctx context.C // for forward compatibility. // // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. type NetworkPolicyDiscoveryServiceServer interface { DeltaNetworkPolicies(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error StreamNetworkPolicies(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error @@ -190,3 +192,136 @@ var NetworkPolicyDiscoveryService_ServiceDesc = grpc.ServiceDesc{ }, Metadata: "cilium/api/npds.proto", } + +const ( + NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResources_FullMethodName = "/cilium.NetworkPolicyResourceDiscoveryService/StreamNetworkPolicyResources" + NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_FullMethodName = "/cilium.NetworkPolicyResourceDiscoveryService/DeltaNetworkPolicyResources" +) + +// NetworkPolicyResourceDiscoveryServiceClient is the client API for NetworkPolicyResourceDiscoveryService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Policy and selector resource names are exact-match identifiers in NPRDS. +type NetworkPolicyResourceDiscoveryServiceClient interface { + StreamNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) + DeltaNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) +} + +type networkPolicyResourceDiscoveryServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewNetworkPolicyResourceDiscoveryServiceClient(cc grpc.ClientConnInterface) NetworkPolicyResourceDiscoveryServiceClient { + return &networkPolicyResourceDiscoveryServiceClient{cc} +} + +func (c *networkPolicyResourceDiscoveryServiceClient) StreamNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyResourceDiscoveryService_ServiceDesc.Streams[0], NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResources_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DiscoveryRequest, v3.DiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResourcesClient = grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse] + +func (c *networkPolicyResourceDiscoveryServiceClient) DeltaNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyResourceDiscoveryService_ServiceDesc.Streams[1], NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResourcesClient = grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + +// NetworkPolicyResourceDiscoveryServiceServer is the server API for NetworkPolicyResourceDiscoveryService service. +// All implementations must embed UnimplementedNetworkPolicyResourceDiscoveryServiceServer +// for forward compatibility. +// +// Policy and selector resource names are exact-match identifiers in NPRDS. +type NetworkPolicyResourceDiscoveryServiceServer interface { + StreamNetworkPolicyResources(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error + DeltaNetworkPolicyResources(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error + mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() +} + +// UnimplementedNetworkPolicyResourceDiscoveryServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedNetworkPolicyResourceDiscoveryServiceServer struct{} + +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) StreamNetworkPolicyResources(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method StreamNetworkPolicyResources not implemented") +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) DeltaNetworkPolicyResources(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method DeltaNetworkPolicyResources not implemented") +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() { +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) testEmbeddedByValue() {} + +// UnsafeNetworkPolicyResourceDiscoveryServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to NetworkPolicyResourceDiscoveryServiceServer will +// result in compilation errors. +type UnsafeNetworkPolicyResourceDiscoveryServiceServer interface { + mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() +} + +func RegisterNetworkPolicyResourceDiscoveryServiceServer(s grpc.ServiceRegistrar, srv NetworkPolicyResourceDiscoveryServiceServer) { + // If the following call panics, it indicates UnimplementedNetworkPolicyResourceDiscoveryServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&NetworkPolicyResourceDiscoveryService_ServiceDesc, srv) +} + +func _NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResources_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyResourceDiscoveryServiceServer).StreamNetworkPolicyResources(&grpc.GenericServerStream[v3.DiscoveryRequest, v3.DiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResourcesServer = grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse] + +func _NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyResourceDiscoveryServiceServer).DeltaNetworkPolicyResources(&grpc.GenericServerStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResourcesServer = grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + +// NetworkPolicyResourceDiscoveryService_ServiceDesc is the grpc.ServiceDesc for NetworkPolicyResourceDiscoveryService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var NetworkPolicyResourceDiscoveryService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "cilium.NetworkPolicyResourceDiscoveryService", + HandlerType: (*NetworkPolicyResourceDiscoveryServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamNetworkPolicyResources", + Handler: _NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResources_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "DeltaNetworkPolicyResources", + Handler: _NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "cilium/api/npds.proto", +} diff --git a/tests/BUILD b/tests/BUILD index 1f6db5971..c084fdf34 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -122,6 +122,15 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "versioned_test", + srcs = ["versioned_test.cc"], + repository = "@envoy", + deps = [ + "//cilium:versioned_lib", + ], +) + envoy_cc_test( name = "bpf_metadata_config_test", srcs = ["bpf_metadata_config_test.cc"], diff --git a/tests/bpf_metadata.cc b/tests/bpf_metadata.cc index c7eda8ada..5528694c7 100644 --- a/tests/bpf_metadata.cc +++ b/tests/bpf_metadata.cc @@ -112,8 +112,8 @@ TestHelper::createPolicyMap(const std::string& config, policy_path, context.serverFactoryContext().api())); Envoy::Config::SubscriptionStats stats = Envoy::Config::Utility::generateStats(context.scope()); - auto map = - std::make_shared(context, Cilium::CILIUM_XDS_API_CONFIG); + auto map = std::make_shared(context, false, + Cilium::CILIUM_XDS_API_CONFIG); auto subscription = std::make_unique( context.serverFactoryContext().mainThreadDispatcher(), Envoy::Config::makePathConfigSource(policy_path), map->subscriptionCallbacksForTest(), diff --git a/tests/bpf_metadata_integration_test.cc b/tests/bpf_metadata_integration_test.cc index 67dd74184..885bca028 100644 --- a/tests/bpf_metadata_integration_test.cc +++ b/tests/bpf_metadata_integration_test.cc @@ -41,6 +41,7 @@ namespace Envoy { namespace { const std::string NetworkPolicyTypeUrl = "type.googleapis.com/cilium.NetworkPolicy"; +const std::string NetworkPolicyResourceTypeUrl = "type.googleapis.com/cilium.NetworkPolicyResource"; const std::string NetworkPolicyHostsTypeUrl = "type.googleapis.com/cilium.NetworkPolicyHosts"; struct NetworkPolicyResourceConfig { @@ -75,32 +76,22 @@ const std::string invalid_policy = R"EOF( endpoint_id: 8192 )EOF"; -const std::string policy_host1 = R"EOF( - policy: 111 - host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] -)EOF"; - -const std::string policy_host2 = R"EOF( - policy: 222 - host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] -)EOF"; - -const NetworkPolicyResourceConfig policy_host1_resource = {"111", "1", R"EOF( - policy: 111 - host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +const NetworkPolicyResourceConfig selector1_resource = {"selector-1", "1", R"EOF( + selector: + remote_identities: [ 43 ] )EOF"}; -const NetworkPolicyResourceConfig policy_host2_resource = {"222", "1", R"EOF( - policy: 222 - host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] +const NetworkPolicyResourceConfig selector2_resource = {"selector-2", "1", R"EOF( + selector: + remote_identities: [ 44 ] )EOF"}; -const NetworkPolicyResourceConfig policy_host1_new_stream_resource = {"111", "2", R"EOF( - policy: 111 - host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +const NetworkPolicyResourceConfig selector3_resource = {"selector-3", "1", R"EOF( + selector: + remote_identities: [ 45 ] )EOF"}; -const NetworkPolicyResourceConfig policy42_resource = {"policy-42", "1", R"EOF( +const NetworkPolicyResourceConfig policy42 = {"policy-42", "1", R"EOF( endpoint_ips: - '10.1.2.3' endpoint_id: 42 @@ -110,7 +101,18 @@ const NetworkPolicyResourceConfig policy42_resource = {"policy-42", "1", R"EOF( - remote_policies: [ 222 ] )EOF"}; -const NetworkPolicyResourceConfig policy43_resource = {"policy-43", "1", R"EOF( +const NetworkPolicyResourceConfig policy42_resource = {"policy-42", "1", R"EOF( + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy43 = {"policy-43", "1", R"EOF( endpoint_ips: - '10.2.3.4' endpoint_id: 43 @@ -120,7 +122,18 @@ const NetworkPolicyResourceConfig policy43_resource = {"policy-43", "1", R"EOF( - remote_policies: [ 111 ] )EOF"}; -const NetworkPolicyResourceConfig policy42_new_stream_resource = {"policy-42", "2", R"EOF( +const NetworkPolicyResourceConfig policy43_resource = {"policy-43", "1", R"EOF( + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy42_new_stream = {"policy-42", "2", R"EOF( endpoint_ips: - '10.1.2.3' endpoint_id: 42 @@ -130,6 +143,42 @@ const NetworkPolicyResourceConfig policy42_new_stream_resource = {"policy-42", " - remote_policies: [ 222 ] )EOF"}; +const NetworkPolicyResourceConfig policy42_new_stream_resource = {"policy-42", "2", R"EOF( + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF"}; + +const std::string policy_host1 = R"EOF( + policy: 111 + host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +)EOF"; + +const std::string policy_host2 = R"EOF( + policy: 222 + host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] +)EOF"; + +const NetworkPolicyResourceConfig policy_host1_resource = {"111", "1", R"EOF( + policy: 111 + host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy_host2_resource = {"222", "1", R"EOF( + policy: 222 + host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy_host1_new_stream_resource = {"111", "2", R"EOF( + policy: 111 + host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +)EOF"}; + class BpfMetadataIntegrationTest : public BaseIntegrationTest, public Grpc::GrpcClientIntegrationParamTest { public: @@ -189,7 +238,8 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, void setBpfMetadataNpdsConfig(::cilium::BpfMetadata& bpf_config, bool use_ads, envoy::config::core::v3::ApiConfigSource::ApiType api_type = - envoy::config::core::v3::ApiConfigSource::GRPC) { + envoy::config::core::v3::ApiConfigSource::GRPC, + bool use_nprds = false) { auto* config_source = bpf_config.mutable_cilium_config_source(); config_source->Clear(); if (use_ads) { @@ -198,12 +248,14 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, } else { setGrpcApiConfigSource(*config_source, api_type); } + bpf_config.set_policy_type(use_nprds); } // Inject the cilium.bpf_metadata listener filter with config_source into the listener. void addBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, bool use_ads, envoy::config::core::v3::ApiConfigSource::ApiType api_type = - envoy::config::core::v3::ApiConfigSource::GRPC) { + envoy::config::core::v3::ApiConfigSource::GRPC, + bool use_nprds = false) { auto* listener_filter = listener.add_listener_filters(); listener_filter->set_name("cilium.bpf_metadata"); @@ -211,13 +263,14 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, bpf_config.set_is_ingress(false); bpf_config.set_use_nphds(true); - setBpfMetadataNpdsConfig(bpf_config, use_ads, api_type); + setBpfMetadataNpdsConfig(bpf_config, use_ads, api_type, use_nprds); listener_filter->mutable_typed_config()->PackFrom(bpf_config); } void updateBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, - envoy::config::core::v3::ApiConfigSource::ApiType api_type) { + envoy::config::core::v3::ApiConfigSource::ApiType api_type, + bool use_nprds = false) { for (auto& listener_filter : *listener.mutable_listener_filters()) { if (listener_filter.name() != "cilium.bpf_metadata") { continue; @@ -226,7 +279,7 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, ::cilium::BpfMetadata bpf_config; RELEASE_ASSERT(listener_filter.typed_config().UnpackTo(&bpf_config), "failed to unpack cilium.bpf_metadata listener filter"); - setBpfMetadataNpdsConfig(bpf_config, /*use_ads=*/false, api_type); + setBpfMetadataNpdsConfig(bpf_config, /*use_ads=*/false, api_type, use_nprds); listener_filter.mutable_typed_config()->PackFrom(bpf_config); return; } @@ -310,7 +363,7 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, bool expect_delta = false) { createXdsConnection(); - for (int i = 0; i < 4; i++) { + for (int i = 0; i < 8; i++) { FakeStreamPtr stream; createXdsStream(stream); @@ -329,10 +382,16 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, request_type_url = request.type_url(); } - if (request_type_url == NetworkPolicyTypeUrl) { + if (request_type_url == NetworkPolicyResourceTypeUrl) { + ENVOY_LOG_MISC(info, "GOT NPRDS STREAM"); + nprds_stream_ = std::move(stream); + if (request_type_url == type_url && is_delta == expect_delta) { + return; + } + } else if (request_type_url == NetworkPolicyTypeUrl) { ENVOY_LOG_MISC(info, "GOT NPDS STREAM"); npds_stream_ = std::move(stream); - if (type_url == NetworkPolicyTypeUrl && is_delta == expect_delta) { + if (request_type_url == type_url && is_delta == expect_delta) { return; } } else if (request_type_url == Envoy::Config::TestTypeUrl::get().Listener) { @@ -415,6 +474,25 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, stream.sendGrpcMessage(response); } + void sendNprdsResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs = { + selector1_resource, selector2_resource, policy42_resource, + policy43_resource}) { + envoy::service::discovery::v3::DiscoveryResponse response; + response.set_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyResourceTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource resource; + resource.set_name(resource_config.name); + resource.set_version(resource_config.version); + resource.mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + response.add_resources()->PackFrom(resource); + } + stream.sendGrpcMessage(response); + } + void sendNphdsResponse(FakeStream& stream, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); @@ -449,6 +527,26 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, stream.sendGrpcMessage(response); } + void sendNprdsDeltaResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs, + const std::vector& removed_resources = {}) { + envoy::service::discovery::v3::DeltaDiscoveryResponse response; + response.set_system_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyResourceTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource* resource = response.add_resources(); + resource->set_name(resource_config.name); + resource->set_version(resource_config.version); + resource->mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + } + for (const auto& removed_resource : removed_resources) { + response.add_removed_resources(removed_resource); + } + stream.sendGrpcMessage(response); + } + void sendNphdsDeltaResponse(FakeStream& stream, const std::string& version, const std::vector& resource_configs, const std::vector& removed_resources = {}) { @@ -475,6 +573,12 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, /*expect_node=*/false); } + AssertionResult compareNprdsAck() { + return compareDeltaDiscoveryRequest(NetworkPolicyResourceTypeUrl, {}, {}, nprds_stream_.get(), + Grpc::Status::WellKnownGrpcStatus::Ok, "", + /*expect_node=*/false); + } + AssertionResult compareNphdsAck() { return compareDeltaDiscoveryRequest(NetworkPolicyHostsTypeUrl, {}, {}, nphds_stream_.get(), Grpc::Status::WellKnownGrpcStatus::Ok, "", @@ -496,6 +600,8 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, void resetNpdsStream() { resetGrpcStream(npds_stream_); } + void resetNprdsStream() { resetGrpcStream(nprds_stream_); } + void resetNphdsStream() { resetGrpcStream(nphds_stream_); } void resetConnections() { @@ -511,6 +617,8 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, cds_stream_.reset(); npds_stream_.reset(); nphds_stream_.reset(); + nprds_stream_.reset(); + retired_streams_.clear(); } uint64_t policyStreamGeneration() const { @@ -567,6 +675,7 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, FakeStreamPtr cds_stream_; FakeStreamPtr npds_stream_; FakeStreamPtr nphds_stream_; + FakeStreamPtr nprds_stream_; std::vector retired_streams_; }; @@ -629,7 +738,7 @@ TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedAdsGrpcSt TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedSotwGrpcStreams) { on_server_init_function_ = [&]() { addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); - createSotWStreams("1"); + createStreamsUntil("1", NetworkPolicyTypeUrl); }; initializeSotw(); test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); @@ -646,7 +755,7 @@ TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedSotwGrpcS resetConnections(); EXPECT_EQ(policyStreamGeneration(), first_generation); - createSotWStreams("2"); + createStreamsUntil("2", NetworkPolicyTypeUrl); sendNpdsResponse(*npds_stream_, "3", {invalid_policy}); // The invalid policy is rejected by the real gRPC subscription decoder/validator before // NetworkPolicyMapImpl::onConfigUpdate() runs, so this increments NPDS subscription stats @@ -659,6 +768,43 @@ TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedSotwGrpcS waitForPolicyStreamGenerationAfter(first_generation); } +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedSotwGrpcNprdsStreams) { + on_server_init_function_ = [&]() { + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false, + envoy::config::core::v3::ApiConfigSource::GRPC, + /*use_nprds=*/true); + createStreamsUntil("1", NetworkPolicyResourceTypeUrl); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + + sendNprdsResponse(*nprds_stream_, "1", + {selector1_resource, selector2_resource, policy42_resource, policy43_resource}); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + sendNprdsResponse(*nprds_stream_, "2", {selector3_resource, policy42_new_stream_resource}); + test_server_->waitForCounterGe("cilium.policy.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_FALSE(policy_map->exists("10.2.3.4")); + + resetConnections(); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + createStreamsUntil("2", NetworkPolicyResourceTypeUrl); + sendNprdsResponse(*nprds_stream_, "3", {selector3_resource, policy42_new_stream_resource}); + test_server_->waitForCounterGe("cilium.policy.update_success", 3); + waitForPolicyStreamGenerationAfter(first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_FALSE(policy_map->exists("10.2.3.4")); +} + TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNpdsStreams) { on_server_init_function_ = [&]() { // Step 1: establish the initial SotW LDS, CDS, NPHDS, and NPDS streams. @@ -689,7 +835,7 @@ TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNpds EXPECT_EQ(policyStreamGeneration(), sotw_generation); // Step 5: accept the first Delta NPDS update and retire the prior SotW policy resources. - sendNpdsDeltaResponse(*npds_stream_, "1", {policy42_resource, policy43_resource}); + sendNpdsDeltaResponse(*npds_stream_, "1", {policy42, policy43}); EXPECT_TRUE(compareNpdsAck()); const uint64_t first_generation = waitForPolicyStreamGenerationAfter(sotw_generation); EXPECT_FALSE(policy_map->exists("10.1.1.1")); @@ -698,7 +844,7 @@ TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNpds EXPECT_TRUE(policy_map->exists("10.2.3.4")); // Step 6: accept a same-stream Delta update; stream generation and omitted resources stay put. - sendNpdsDeltaResponse(*npds_stream_, "2", {policy42_new_stream_resource}); + sendNpdsDeltaResponse(*npds_stream_, "2", {policy42_new_stream}); EXPECT_TRUE(compareNpdsAck()); EXPECT_EQ(policyStreamGeneration(), first_generation); EXPECT_TRUE(policy_map->exists("10.1.2.3")); @@ -713,7 +859,70 @@ TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNpds EXPECT_TRUE(policy_map->exists("10.2.3.4")); // Step 9: accept the first update on the new stream and retire resources from the old stream. - sendNpdsDeltaResponse(*npds_stream_, "3", {policy42_new_stream_resource}); + sendNpdsDeltaResponse(*npds_stream_, "3", {policy42_new_stream}); + waitForPolicyStreamGenerationAfter(first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_FALSE(policy_map->exists("10.2.3.4")); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNprdsStreams) { + on_server_init_function_ = [&]() { + // Step 1: establish the initial SotW LDS, CDS, NPHDS, and NPDS streams. + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createSotWStreams("1"); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + + // Step 2: accept a real SotW NPDS response so the starting mode has installed policy. + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t sotw_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.1.1")); + EXPECT_TRUE(policy_map->exists("10.2.2.2")); + + // Step 3: update the BpfMetadata config source; this is evidence that Delta NPRDS is available. + updateBpfMetadataListenerFilter(listener_config_, + envoy::config::core::v3::ApiConfigSource::DELTA_GRPC, + /*use_nprds=*/true); + sendLdsResponse(*lds_stream_, {listener_config_}, "2"); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 2); + + // Step 4: observe the immediate switch to Delta NPRDS without advancing accepted policy state. + createStreamsUntil("2", NetworkPolicyResourceTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + + // Step 5: accept the first Delta NPRDS update and retire the prior SotW policy resources. + sendNprdsDeltaResponse( + *nprds_stream_, "1", + {selector1_resource, selector2_resource, policy42_resource, policy43_resource}); + EXPECT_TRUE(compareNprdsAck()); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(sotw_generation); + EXPECT_FALSE(policy_map->exists("10.1.1.1")); + EXPECT_FALSE(policy_map->exists("10.2.2.2")); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 6: accept a same-stream Delta update; stream generation and omitted resources stay put. + sendNprdsDeltaResponse(*nprds_stream_, "2", {selector3_resource, policy42_new_stream_resource}); + EXPECT_TRUE(compareNprdsAck()); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 7: reset the Delta NPRDS stream. + resetNprdsStream(); + + // Step 8: open the replacement Delta stream; reconnect alone must not advance policy state. + createStreamsUntil("3", NetworkPolicyResourceTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 9: accept the first update on the new stream and retire resources from the old stream. + sendNprdsDeltaResponse(*nprds_stream_, "3", {selector3_resource, policy42_new_stream_resource}); waitForPolicyStreamGenerationAfter(first_generation); EXPECT_TRUE(policy_map->exists("10.1.2.3")); EXPECT_FALSE(policy_map->exists("10.2.3.4")); diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index bf54669b9..86a8c573c 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -3,10 +3,12 @@ #include #include +#include #include #include #include #include +#include #include "envoy/common/exception.h" #include "envoy/common/optref.h" @@ -33,6 +35,7 @@ #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" +#include "absl/container/flat_hash_set.h" #include "absl/strings/string_view.h" #include "cilium/accesslog.h" #include "cilium/network_policy.h" @@ -71,6 +74,42 @@ extern const envoy::config::core::v3::ConfigSource CILIUM_XDS_API_CONFIG; return secret_provider; \ })) +namespace { + +envoy::config::core::v3::ConfigSource configSourceForUseDeltaXds(bool use_delta_xds) { + auto config_source = CILIUM_XDS_API_CONFIG; + config_source.mutable_api_config_source()->set_api_type( + use_delta_xds ? envoy::config::core::v3::ApiConfigSource::DELTA_GRPC + : envoy::config::core::v3::ApiConfigSource::GRPC); + return config_source; +} + +struct FakeSubscriptionState { + int start_calls_{0}; + std::vector> start_resources_; +}; + +class FakeSubscription : public Envoy::Config::Subscription { +public: + explicit FakeSubscription(std::shared_ptr state) + : state_(std::move(state)) {} + + void start(const absl::flat_hash_set& resource_names) override { + ++state_->start_calls_; + auto& started = state_->start_resources_.emplace_back(); + started.insert(started.end(), resource_names.begin(), resource_names.end()); + std::sort(started.begin(), started.end()); + } + + void updateResourceInterest(const absl::flat_hash_set&) override {} + void requestOnDemandUpdate(const absl::flat_hash_set&) override {} + +private: + std::shared_ptr state_; +}; + +} // namespace + class CiliumNetworkPolicyTest : public ::testing::Test { protected: CiliumNetworkPolicyTest() { @@ -92,8 +131,10 @@ class CiliumNetworkPolicyTest : public ::testing::Test { ON_CALL_SDS_SECRET_PROVIDER(secret_manager_, TlsSessionTicketKeysContext, TlsSessionTicketKeys); ON_CALL_SDS_SECRET_PROVIDER(secret_manager_, GenericSecret, GenericSecret); - policy_map_ = - std::make_shared(factory_context_, Cilium::CILIUM_XDS_API_CONFIG); + bool use_delta = useDeltaXds(); + bool use_nprds = use_delta; + policy_map_ = std::make_shared(factory_context_, use_nprds, + configSourceForUseDeltaXds(use_delta)); } void TearDown() override { @@ -101,6 +142,8 @@ class CiliumNetworkPolicyTest : public ::testing::Test { policy_map_.reset(); } + virtual bool useDeltaXds() const { return false; } + Envoy::Config::SubscriptionCallbacks& subscriptionCallbacks() const { return policy_map_->subscriptionCallbacksForTest(); } @@ -120,6 +163,23 @@ class CiliumNetworkPolicyTest : public ::testing::Test { return message.version_info(); } + std::string deltaUpdateFromYaml(const std::string& config) { + envoy::service::discovery::v3::DeltaDiscoveryResponse message; + MessageUtil::loadFromYaml(config, message, ProtobufMessage::getNullValidationVisitor()); + NetworkPolicyResourceDecoder network_policy_resource_decoder; + auto decoded_resources = std::make_unique(); + for (const auto& resource : message.resources()) { + decoded_resources->pushBack( + Config::DecodedResourceImpl::fromResource(network_policy_resource_decoder, resource)); + } + + EXPECT_TRUE(subscriptionCallbacks() + .onConfigUpdate(decoded_resources->refvec_, message.removed_resources(), + message.system_version_info()) + .ok()); + return message.system_version_info(); + } + testing::AssertionResult validate(const std::string& pod_ip, const std::string& expected) { const auto& policy = policy_map_->getPolicyInstance(pod_ip, false); auto str = policy.string(); @@ -239,6 +299,34 @@ class CiliumNetworkPolicyTest : public ::testing::Test { return policy_map_->statsForTest().updates_rejected_.name(); } + PolicyInstanceConstSharedPtr policyInstanceShared(const std::string& pod_ip) const { + return policy_map_->getPolicyInstanceSharedForTest(pod_ip); + } + + uint64_t selectorStreamGenerationForTest(const PolicyInstance& policy) const { + return policy_map_->policySelectorStreamGenerationForTest(policy); + } + + SelectorVersion selectorVersionForTest(const PolicyInstance& policy) const { + return policy_map_->policySelectorVersionForTest(policy); + } + + void resetStreamForTest() { policy_map_->resetStreamForTest(); } + bool configuredUseDeltaXds() const { return policy_map_->desiredUseDeltaXdsForTest(); } + void setPolicyConfig(bool use_delta_xds, bool use_nprds) const { + policy_map_->setConfig(use_nprds, configSourceForUseDeltaXds(use_delta_xds)); + } + void startManagedSubscriptionForTest() { policy_map_->startManagedSubscriptionForTest(); } + void setSubscriptionFactoryForTest(NetworkPolicyMap::SubscriptionFactoryForTest factory) { + policy_map_->setSubscriptionFactoryForTest(std::move(factory)); + } + void onSubscriptionConnectedForTest() { policy_map_->onSubscriptionConnectedForTest(); } + void onSubscriptionTransportCloseForTest() { policy_map_->onSubscriptionTransportCloseForTest(); } + bool subscriptionUseDeltaXdsForTest() const { + return policy_map_->subscriptionUseDeltaXdsForTest(); + } + bool subscriptionConnectedForTest() const { return policy_map_->subscriptionConnectedForTest(); } + NiceMock factory_context_; NiceMock secret_manager_; std::shared_ptr policy_map_; @@ -246,10 +334,379 @@ class CiliumNetworkPolicyTest : public ::testing::Test { uint16_t proxy_id_ = 42; }; +class CiliumNetworkPolicyDeltaTest : public CiliumNetworkPolicyTest { +protected: + bool useDeltaXds() const override { return true; } +}; + TEST_F(CiliumNetworkPolicyTest, UpdatesRejectedStatName) { EXPECT_EQ("cilium.policy.updates_rejected", updatesRejectedStatName()); } +TEST_F(CiliumNetworkPolicyTest, ManagedSubscriptionColdStartUsesConfiguredSotwMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + + ASSERT_EQ(created_modes.size(), 1); + EXPECT_FALSE(created_modes.front()); + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 1); + EXPECT_TRUE(state->start_resources_.front().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, ManagedSubscriptionColdStartUsesConfiguredDeltaMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + + ASSERT_EQ(created_modes.size(), 1); + EXPECT_TRUE(created_modes.front()); + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 1); + EXPECT_TRUE(state->start_resources_.front().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipFromSotwToDeltaOnHealthySubscriptionRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + + setPolicyConfig(true, true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipFromDeltaToSotwOnHealthySubscriptionWaitsForClose) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_FALSE(subscriptionUseDeltaXdsForTest()); + + setPolicyConfig(true, true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); + + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setPolicyConfig(false, false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_TRUE(subscriptionConnectedForTest()); + // Once we have an established delta subscription, keep it until transport close even if the + // configured desired mode flips back to SotW. + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + + onSubscriptionTransportCloseForTest(); + + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true, false)); + EXPECT_EQ(state->start_calls_, 3); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipWhileDisconnectedRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + ASSERT_FALSE(subscriptionConnectedForTest()); + + setPolicyConfig(true, true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, FlagFlipFromDisconnectedDeltaToSotwRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setPolicyConfig(false, false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, false)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DowngradeFromConnectedDeltaRecreatesDisconnectedRetryToSotw) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + // Agent restart/downgrade drops the established delta transport. While the desired mode is still + // delta we recreate immediately and begin retrying delta. + onSubscriptionTransportCloseForTest(); + + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 2); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true)); + EXPECT_TRUE(state->start_resources_.back().empty()); + + // When listener metadata later reveals the downgraded agent no longer supports delta, flip to + // SotW immediately rather than letting the disconnected delta retry loop forever. + setPolicyConfig(false, false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true, false)); + EXPECT_EQ(state->start_calls_, 3); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, TransportCloseWithoutFlagFlipRecreatesInCurrentMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + + onSubscriptionTransportCloseForTest(); + + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, false)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPoliciesUseDeltaSdsForGenericSecrets) { + EXPECT_CALL(secret_manager_, findOrCreateGenericSecretProvider(_, "nonexisting-sds-secret", _, _)) + .WillOnce(Invoke([](const envoy::config::core::v3::ConfigSource& sds_config_source, + const std::string& config_name, + Server::Configuration::ServerFactoryContext& server_context, + Init::Manager& init_manager) { + EXPECT_TRUE(sds_config_source.has_api_config_source()); + EXPECT_EQ(envoy::config::core::v3::ApiConfigSource::DELTA_GRPC, + sds_config_source.api_config_source().api_type()); + auto secret_provider = Secret::GenericSecretSdsApi::create( + server_context, sds_config_source, config_name, []() {}, false); + init_manager.add(*secret_provider->initTarget()); + return secret_provider; + })); + + std::string version; + EXPECT_NO_THROW(version = deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-43" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' + header_matches: + - name: 'bearer-token' + value_sds_secret: 'nonexisting-sds-secret' + mismatch_action: REPLACE_ON_MISMATCH +)EOF")); + EXPECT_EQ(version, "1"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, SotwDowngrade) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + using ApiType = envoy::config::core::v3::ApiConfigSource::ApiType; + auto expect_tls_sds_mode = [this](ApiType api_type) { + EXPECT_CALL(secret_manager_, findOrCreateTlsCertificateProvider(_, "secret1", _, _, _)) + .WillOnce(Invoke([api_type](const envoy::config::core::v3::ConfigSource& sds_config_source, + const std::string& config_name, + Server::Configuration::ServerFactoryContext& server_context, + OptRef init_manager, bool /*warm*/) { + EXPECT_TRUE(sds_config_source.has_api_config_source()); + EXPECT_EQ(api_type, sds_config_source.api_config_source().api_type()); + auto secret_provider = Secret::TlsCertificateSdsApi::create( + server_context, sds_config_source, config_name, []() {}, false); + if (init_manager.has_value()) { + init_manager->add(*secret_provider->initTarget()); + } + return secret_provider; + })) + .RetiresOnSaturation(); + EXPECT_CALL(secret_manager_, + findOrCreateCertificateValidationContextProvider(_, "cacerts", _, _)) + .WillOnce(Invoke([api_type](const envoy::config::core::v3::ConfigSource& sds_config_source, + const std::string& config_name, + Server::Configuration::ServerFactoryContext& server_context, + Init::Manager& init_manager) { + EXPECT_TRUE(sds_config_source.has_api_config_source()); + EXPECT_EQ(api_type, sds_config_source.api_config_source().api_type()); + auto secret_provider = Secret::CertificateValidationContextSdsApi::create( + server_context, sds_config_source, config_name, []() {}, false); + init_manager.add(*secret_provider->initTarget()); + return secret_provider; + })) + .RetiresOnSaturation(); + }; + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + expect_tls_sds_mode(envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); + std::string version; + EXPECT_NO_THROW(version = deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + egress_per_port_policies: + - port: 80 + rules: + - server_names: [ "example.com" ] + downstream_tls_context: + tls_sds_secret: "secret1" + upstream_tls_context: + validation_context_sds_secret: "cacerts" +)EOF")); + EXPECT_EQ(version, "1"); + + // Simulate xDS server restart and downgrade to SotW mode + onSubscriptionTransportCloseForTest(); + + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setPolicyConfig(false, false); + + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true, false)); + + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_FALSE(subscriptionUseDeltaXdsForTest()); + + // Same NetworkPolicy, but it must be updated for the secret watcher to also use SotW + expect_tls_sds_mode(envoy::config::core::v3::ApiConfigSource::GRPC); + EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + egress_per_port_policies: + - port: 80 + rules: + - server_names: [ "example.com" ] + downstream_tls_context: + tls_sds_secret: "secret1" + upstream_tls_context: + validation_context_sds_secret: "cacerts" +)EOF")); + EXPECT_EQ(version, "1"); +} + TEST_F(CiliumNetworkPolicyTest, EmptyPolicyUpdate) { EXPECT_TRUE(subscriptionCallbacks().onConfigUpdate({}, "1").ok()); EXPECT_FALSE(validate("10.1.2.3", "")); // Policy not found @@ -263,6 +720,1493 @@ TEST_F(CiliumNetworkPolicyTest, SimplePolicyUpdate) { EXPECT_FALSE(validate("10.1.2.3", "")); // Policy not found } +TEST_F(CiliumNetworkPolicyTest, RejectsWhitespaceInSotwWrappedResourceName) { + EXPECT_THROW_WITH_MESSAGE(updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/envoy.service.discovery.v3.Resource + name: "policy 42" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF"), + EnvoyException, + "NetworkPolicy resource name 'policy 42' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaIncrementalPolicyUpdate) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 8080)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSameStreamKeepsUntouchedResources) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 8080)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovesPolicyByResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + - "f00d::1" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("f00d::1")); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("f00d::1", 43, 80)); + + EXPECT_TRUE(policy_map_->exists("10.2.3.4")); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "policy-42" +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_FALSE(policy_map_->exists("f00d::1")); + EXPECT_FALSE(validate("10.1.2.3", "")); + EXPECT_FALSE(validate("f00d::1", "")); + + EXPECT_TRUE(policy_map_->exists("10.2.3.4")); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSelectorOnlyUpdateTakesEffectImmediately) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPolicyUpdateRejectsMissingSelectorResource) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-2" ] +)EOF"), + EnvoyException, + "NetworkPolicyResource rule references missing selector resource " + "'selector-2'"); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovedAndReaddedSelectorNameDoesNotRebindOldPolicy) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "selector-1" +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "3" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_EQ(old_policy.get(), policyInstanceShared("10.1.2.3").get()); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "4" +resources: +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_EQ(old_policy.get(), policyInstanceShared("10.1.2.3").get()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovedAndReaddedSelectorNameInSameUpdateActsAsUpdate) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "selector-1" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectedSelectorUpdateKeepsPublishedBehavior) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-42'.*endpoint_id 42.*10\.1\.2\.3)"); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPassUsesCurrentSelectorMembership) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaWildcardPortPassIsMergedToExactPortRules) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 0 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - port: 80 + rules: + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSamePrecedenceDenyWinsOverPass) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 1000 + deny: true + selectors: [ "selector-1" ] + - precedence: 500 + selectors: [ "selector-1" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSelectorOnlyUpdateChangesPassBehaviorImmediately) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyTest, SotwRejectsSelectorsInRules) { + EXPECT_THROW(updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitrarySelectorResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "7" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "7" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEmbeddedRemotePoliciesInRules) { + EXPECT_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 43 ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsAddedResourceNamesWithWhitespace) { + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector 1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + "NetworkPolicyResource added resource name 'selector 1' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovedResourceNamesWithWhitespace) { + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +removed_resources: +- "selector 1" +)EOF"), + EnvoyException, + "NetworkPolicyResource removed resource name 'selector 1' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicatePolicyResourceNamesInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming policy resource 'shared-name'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesThatDoNotMatchEndpointId) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsResourceNameEndpointIpCollisionsInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicateSelectorResourceNamesInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-selector" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +- name: "shared-selector" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 46, 47 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key 'shared-selector'.*incoming selector resource 'shared-selector'.*existing selector resource 'shared-selector')"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceNamesWithHyphens) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-qualified-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovesPoliciesByArbitraryResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-qualified-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "42" +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsWithExistingPolicies) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsPolicyResourceNameCollidingWithExistingEndpointIp) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource '10\.1\.2\.3'.*endpoint_id 43.*10\.1\.2\.4.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsEndpointIpCollidingWithExistingPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsSelectorResourceNameCollidingWithExistingPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming selector resource 'shared-name'.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsSelectorResourceNameCollidingWithExistingEndpointIp) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovingPolicyEndpointIpAlias) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "10.1.2.3" +)EOF"), + EnvoyException, + "NetworkPolicyResource removed resource '10.1.2.3' is a policy endpoint IP alias, not a " + "resource name"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAllowsEndpointIpReusingRemovedPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "10.1.2.4" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesWithNumericSuffixes) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-042" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-0" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsZeroEndpointIdRegardlessOfResourceName) { + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-0" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 0 +)EOF"), + EnvoyException, "NetworkPolicyResource endpoint_id must be non-zero"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsInconsistentPassPrecedence) { + EXPECT_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 100 + selectors: [ "selector-1" ] + - precedence: 900 + pass_precedence: 200 + selectors: [ "selector-2" ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaNewStreamReplacesStateWithFullSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 8080)); + EXPECT_FALSE(policy_map_->exists("10.2.3.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaNewStreamClearsStaleResourceNamesFromResourceMap) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.2.3.4")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "3" +resources: +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 46 ] +)EOF")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "4" +resources: +- name: "policy-42" + version: "3" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 9090 + rules: + - selectors: [ "policy-43" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 46, 9090)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, SameStreamSelectorOnlyUpdateUsesLatestSelectorSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + const auto initial_stream_generation = selectorStreamGenerationForTest(*old_policy); + + EXPECT_GT(initial_stream_generation, 0); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(1, selectorVersionForTest(*old_policy)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, NewStreamKeepsOldPolicyPinnedToOldSelectorSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + const auto initial_stream_generation = selectorStreamGenerationForTest(*old_policy); + + EXPECT_GT(initial_stream_generation, 0); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(1, selectorVersionForTest(*old_policy)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +)EOF")); + + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "selector-2" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto new_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, new_policy); + EXPECT_NE(old_policy.get(), new_policy.get()); + + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + EXPECT_EQ(initial_stream_generation + 1, selectorStreamGenerationForTest(*new_policy)); + EXPECT_EQ(3, selectorVersionForTest(*new_policy)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); +} + TEST_F(CiliumNetworkPolicyTest, OverlappingPortRange) { EXPECT_NO_THROW(updateFromYaml(R"EOF(version_info: "1" resources: diff --git a/tests/versioned_test.cc b/tests/versioned_test.cc new file mode 100644 index 000000000..be90c9ec5 --- /dev/null +++ b/tests/versioned_test.cc @@ -0,0 +1,1165 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "source/common/common/lock_guard.h" +#include "source/common/common/thread.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "cilium/versioned.h" + +// NOLINT(namespace-envoy) +namespace { + +class TrackedValue : public VersionedNode { +public: + explicit TrackedValue(int id) : TrackedValue(id, id) {} + + TrackedValue(int key_id, int generation) + : key_id_(key_id), generation_(generation), unique_id_(next_unique_id_++) { + ++constructed_count_; + ++live_count_; + } + + ~TrackedValue() { + ++destroyed_count_; + --live_count_; + ++destroyed_by_id_[generation_]; + } + + int id() const { return generation_; } + int keyId() const { return key_id_; } + int generation() const { return generation_; } + uint64_t uniqueId() const { return unique_id_; } + + static void resetCounters() { + constructed_count_ = 0; + destroyed_count_ = 0; + live_count_ = 0; + destroyed_by_id_.clear(); + next_unique_id_ = 1; + } + + static int constructedCount() { return constructed_count_; } + static int destroyedCount() { return destroyed_count_; } + static int liveCount() { return live_count_; } + + static int destroyedCountFor(int id) { + const auto it = destroyed_by_id_.find(id); + return it == destroyed_by_id_.end() ? 0 : it->second; + } + +private: + int key_id_; + int generation_; + uint64_t unique_id_; + + static inline int constructed_count_ = 0; + static inline int destroyed_count_ = 0; + static inline int live_count_ = 0; + static inline uint64_t next_unique_id_ = 1; + static inline std::map destroyed_by_id_; +}; + +using TrackedHandle = VersionedHandle; +using TrackedDeferredDeletion = DeferredDeletion; +using TrackedMap = VersionedMap; +using TrackedAccess = VersionedTestAccess; + +const TrackedValue* activeValue(const TrackedMap& map, const std::string& key, + uint64_t version = versionMax) { + auto handle = map.find(key); + return handle != nullptr ? handle->get(version) : nullptr; +} + +struct PublishedHandles { + uint64_t snapshot_id; + uint64_t version; + std::vector> handles; +}; + +struct ObservedNode { + const void* node; + const void* next; + const TrackedValue* value; + uint64_t add_version; + uint64_t remove_version; +}; + +enum class ValidationMode { + StrictConsistentView, + ConcurrentSafeInvisibleTail, +}; + +std::string describeValue(const TrackedValue* value) { + if (value == nullptr) { + return "nullptr"; + } + return testing::PrintToString( + std::make_tuple(value->keyId(), value->generation(), value->uniqueId())); +} + +std::string describeObservedChain(const std::vector& nodes) { + std::ostringstream out; + for (size_t i = 0; i < nodes.size(); ++i) { + if (i > 0) { + out << " -> "; + } + const auto& node = nodes[i]; + out << "{node=" << node.node << ", next=" << node.next + << ", value=" << describeValue(node.value) << ", add=" << node.add_version + << ", remove=" << node.remove_version << "}"; + } + if (nodes.empty()) { + out << ""; + } + return out.str(); +} + +std::string validateTraversalAtVersion(const std::string& label, + const VersionedNode* initial_head, + uint64_t version, ValidationMode mode, + const TrackedValue* expected = nullptr, + const void* handle = nullptr) { + const bool require_consistent_view = mode == ValidationMode::StrictConsistentView; + const TrackedValue* visible_value = nullptr; + size_t visible_count = 0; + uint64_t previous_add_version = versionNotRemoved; + bool in_invisible_tail = false; + absl::flat_hash_map visited_nodes; + std::vector observed_nodes; + for (const auto* node = initial_head; node != nullptr;) { + const auto* next = TrackedAccess::next(node); + observed_nodes.push_back(ObservedNode{node, next, TrackedAccess::value(node), + TrackedAccess::addVersion(node), + TrackedAccess::removeVersion(node)}); + const auto add_version = observed_nodes.back().add_version; + const auto [visited_it, inserted] = visited_nodes.emplace(node, add_version); + if (!inserted) { + if (visited_it->second == add_version) { + return "cycle detected for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + visited_it->second = add_version; + } + + const auto remove_version = observed_nodes.back().remove_version; + if (add_version >= versionNotRemoved) { + return "unpublished node observed for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + if ((require_consistent_view || !in_invisible_tail) && add_version > previous_add_version) { + return "add_version increased while following next links for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " previous_add=" + std::to_string(previous_add_version) + + " current_add=" + std::to_string(add_version) + + " chain=" + describeObservedChain(observed_nodes); + } + if (remove_version < versionNotRemoved && add_version > remove_version) { + return "invalid node visibility window for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + + if (add_version <= version && version < remove_version) { + ++visible_count; + visible_value = TrackedAccess::value(node); + if (require_consistent_view && visible_count > 1) { + return "multiple visible nodes at version " + std::to_string(version) + " for '" + label + + "' handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + } + // Once traversal reaches the first node that is already stale for this version, production + // readers no longer depend on per-handle ordering. First-phase GC may have already rewired the + // rest of the tail into a mixed deferred-deletion list, but those nodes must remain safe to + // walk and invisible for this version. + if (!require_consistent_view && in_invisible_tail && + TrackedAccess::isVisibleInVersion(node, version)) { + return "visible node observed after entering invisible tail for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + if (!TrackedAccess::isVisibleInVersion(node, version) && + !TrackedAccess::isAddedAfterVersion(node, version)) { + in_invisible_tail = true; + } + previous_add_version = add_version; + node = next; + } + + if (require_consistent_view && expected != visible_value) { + return "handle->get mismatch for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " expected=" + describeValue(expected) + ", traversed=" + describeValue(visible_value) + + ", chain=" + describeObservedChain(observed_nodes); + } + + return ""; +} + +std::string validateHandleAtVersion(const std::string& label, const TrackedHandle& handle, + uint64_t version, + ValidationMode mode = ValidationMode::StrictConsistentView) { + if (handle == nullptr) { + return "snapshot contains null handle for '" + label + "'"; + } + + return validateTraversalAtVersion( + label, TrackedAccess::head(handle), version, mode, + mode == ValidationMode::StrictConsistentView ? handle->get(version) : nullptr, handle.get()); +} + +std::string +validateNodeAtVersion(const std::string& label, const VersionedNode* node, + uint64_t version, + ValidationMode mode = ValidationMode::ConcurrentSafeInvisibleTail) { + if (node == nullptr) { + return "null node supplied for '" + label + "'"; + } + return validateTraversalAtVersion(label, node, version, mode); +} + +std::string +validatePublishedSnapshotAtVersion(const PublishedHandles& snapshot, uint64_t version, + ValidationMode mode = ValidationMode::StrictConsistentView) { + for (const auto& [label, handle] : snapshot.handles) { + auto error = validateHandleAtVersion(label, handle, version, mode); + if (!error.empty()) { + return error; + } + } + return ""; +} + +std::string validatePublishedSnapshot(const PublishedHandles& snapshot) { + return validatePublishedSnapshotAtVersion(snapshot, snapshot.version); +} + +std::shared_ptr makePublishedHandles(const TrackedMap& map, + uint64_t snapshot_id = 0) { + auto snapshot = std::make_shared(); + snapshot->snapshot_id = snapshot_id; + snapshot->version = map.getVersion(); + + absl::flat_hash_set seen_handles; + for (const auto& [key, handle] : TrackedAccess::entries(map)) { + if (handle != nullptr && seen_handles.insert(handle.get()).second) { + snapshot->handles.emplace_back(key, handle); + } + } + size_t dirty_index = 0; + for (const auto& handle : TrackedAccess::dirtyValues(map)) { + if (handle != nullptr && seen_handles.insert(handle.get()).second) { + snapshot->handles.emplace_back("dirty:" + std::to_string(dirty_index++), handle); + } + } + + return snapshot; +} + +class VersionedTest : public testing::Test { +protected: + void SetUp() override { TrackedValue::resetCounters(); } + + void TearDown() override { + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::constructedCount(), TrackedValue::destroyedCount()); + } +}; + +TEST_F(VersionedTest, VersionedValueFirstInsertedValueIsVisibleInPublishedVersion) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + + EXPECT_EQ(value.get(0), nullptr); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); +} + +TEST_F(VersionedTest, VersionedValueNewerValueShadowsOlderValueFromNewVersionOnward) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + EXPECT_EQ(TrackedValue::liveCount(), 2); +} + +TEST_F(VersionedTest, VersionedValueClearHidesOnlyFromThatVersionOnward) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.clear(2); + + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + EXPECT_EQ(value.get(2), nullptr); +} + +TEST_F(VersionedTest, VersionedValueRevertRestoresPendingClear) { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.clear(2); + + EXPECT_EQ(value.get(2), nullptr); + + value.revert(1, deferred); + + EXPECT_TRUE(deferred.empty()); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 1); +} + +TEST_F(VersionedTest, VersionedValueRevertDeletesUnpublishedReplacement) { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + value.revert(1, deferred); + + EXPECT_FALSE(deferred.empty()); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); +} + +TEST_F(VersionedTest, VersionedValueDeferredRevertDeletesReplacementOnBatchDestruction) { + { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + value.revert(1, deferred); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +TEST_F(VersionedTest, VersionedValueGcDefersDeletionUntilBatchDestruction) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + TrackedDeferredDeletion deferred; + EXPECT_FALSE(value.gcForVersion(1, deferred)); + + EXPECT_TRUE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + } + + { + TrackedDeferredDeletion deferred; + EXPECT_TRUE(value.gcForVersion(2, deferred)); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); +} + +TEST_F(VersionedTest, DeferredDeletionMoveTransfersDeletionOwnership) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + TrackedDeferredDeletion deferred; + EXPECT_TRUE(value.gcForVersion(2, deferred)); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + TrackedDeferredDeletion moved = std::move(deferred); + EXPECT_FALSE(moved.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, FirstPhaseGcKeepsCurrentAndHistoricalLookupsMemorySafe) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(validateHandleAtVersion("a", handle, version1), ""); + EXPECT_EQ(validateHandleAtVersion("a", handle, version2), ""); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, ConcurrentValidatorAcceptsMixedDeferredTailTraversal) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1, 1)); + map.insert("b", new TrackedValue(2, 1)); + map.publishNextVersion(); + + auto handle_a = map.find("a"); + auto handle_b = map.find("b"); + ASSERT_NE(handle_a, nullptr); + ASSERT_NE(handle_b, nullptr); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1, 2)); + map.insert("b", new TrackedValue(2, 2)); + const auto version2 = map.publishNextVersion(); + + const auto* stale_a = TrackedAccess::next(TrackedAccess::head(handle_a)); + const auto* stale_b = TrackedAccess::next(TrackedAccess::head(handle_b)); + ASSERT_NE(stale_a, nullptr); + ASSERT_NE(stale_b, nullptr); + + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + + EXPECT_EQ(validateHandleAtVersion("a", handle_a, version2), ""); + EXPECT_EQ(validateHandleAtVersion("b", handle_b, version2), ""); + EXPECT_EQ(validateNodeAtVersion("stale-a", stale_a, version2), ""); + EXPECT_EQ(validateNodeAtVersion("stale-b", stale_b, version2), ""); + } +} + +TEST_F(VersionedTest, VersionedMapGcRetainsDirtyHandleUntilAllFutureRemovalsAreReclaimed) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(3)); + const auto version3 = map.publishNextVersion(); + + EXPECT_EQ(TrackedValue::liveCount(), 3); + + { + auto deferred = map.gc(version2); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 3); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + ASSERT_NE(activeValue(map, "a", version2), nullptr); + EXPECT_EQ(activeValue(map, "a", version2)->id(), 2); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); + } + + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + + { + auto deferred = map.gc(version3); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); +} + +TEST_F(VersionedTest, VersionedMapVersionStartsAtMin) { + TrackedMap map; + + EXPECT_EQ(map.getVersion(), versionMin); +} + +TEST_F(VersionedTest, VersionedMapPublishWithNoPendingChangesReturnsZero) { + TrackedMap map; + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); + + map.prepareNextVersion(); + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); +} + +TEST_F(VersionedTest, VersionedMapInsertNewKeyThenRevertRemovesItEntirely) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + + ASSERT_NE(activeValue(map, "a", 1), nullptr); + + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(map.find("a"), nullptr); +} + +TEST_F(VersionedTest, VersionedMapDeferredRevertDeletesNewKeyAfterBatchDestruction) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(map.find("a"), nullptr); + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, VersionedMapUpdateExistingKeyAndRevertRestoresPublishedValue) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + EXPECT_EQ(map.getVersion(), version1); + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); +} + +TEST_F(VersionedTest, VersionedMapDeferredRevertDeletesReplacementAfterBatchDestruction) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +TEST_F(VersionedTest, VersionedMapClearExistingKeyAndRevertRestoresPublishedValue) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.clear("a"); + + EXPECT_EQ(activeValue(map, "a"), nullptr); + + auto deferred = map.revert(); + + EXPECT_TRUE(deferred.empty()); + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); +} + +TEST_F(VersionedTest, VersionedMapClearMissingKeyIsNoOp) { + TrackedMap map; + + map.prepareNextVersion(); + map.clear("missing"); + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); + EXPECT_EQ(TrackedValue::liveCount(), 0); +} + +TEST_F(VersionedTest, VersionedMapSnapshotsIsolateKeysAndVersions) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.insert("b", new TrackedValue(10)); + auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + published_version = map.publishNextVersion(); + const auto version2 = map.getVersion(); + + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a", version2), nullptr); + EXPECT_EQ(activeValue(map, "a", version2)->id(), 2); + ASSERT_NE(activeValue(map, "b", version1), nullptr); + EXPECT_EQ(activeValue(map, "b", version1)->id(), 10); + ASSERT_NE(activeValue(map, "b", version2), nullptr); + EXPECT_EQ(activeValue(map, "b", version2)->id(), 10); + + map.gc(published_version); +} + +TEST_F(VersionedTest, VersionedMapActiveUpdateReusesStableHandle) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + ASSERT_NE(handle->get(version2), nullptr); + EXPECT_EQ(handle->get(version2)->id(), 2); + + map.gc(version2); +} + +TEST_F(VersionedTest, VersionedMapClearAndReinsertSameKeyBeforePublishReusesStableHandle) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + + map.prepareNextVersion(); + map.clear("a"); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + ASSERT_NE(handle->get(version2), nullptr); + EXPECT_EQ(handle->get(version2)->id(), 2); + + map.gc(version2); +} + +TEST_F(VersionedTest, VersionedMapRemovedHandleStaysEmptyAfterSameNameReAdd) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.publishNextVersion(); + + auto old_handle = map.find("a"); + ASSERT_NE(old_handle, nullptr); + + map.prepareNextVersion(); + map.clear("a"); + const auto version2 = map.publishNextVersion(); + map.gc(version2); + + EXPECT_EQ(map.find("a"), nullptr); + EXPECT_EQ(old_handle->get(version2), nullptr); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version3 = map.publishNextVersion(); + + auto new_handle = map.find("a"); + ASSERT_NE(new_handle, nullptr); + EXPECT_NE(new_handle, old_handle); + ASSERT_NE(new_handle->get(version3), nullptr); + EXPECT_EQ(new_handle->get(version3)->id(), 2); + EXPECT_EQ(old_handle->get(version3), nullptr); + map.gc(version3); +} + +TEST_F(VersionedTest, VersionedMapValidatorsAcceptStablePublishedSnapshots) { + TrackedMap map; + std::array generations{}; + + auto next_value = [&](int key_id) { return new TrackedValue(key_id, ++generations[key_id]); }; + auto expect_snapshot_valid = [&](const std::string& phase) { + auto snapshot = makePublishedHandles(map); + EXPECT_EQ(validatePublishedSnapshot(*snapshot), "") << phase; + }; + + map.prepareNextVersion(); + map.insert("key-10", next_value(10)); + map.insert("key-11", next_value(11)); + const auto version1 = map.publishNextVersion(); + ASSERT_EQ(version1, 1); + expect_snapshot_valid("after initial publish"); + + // Exercise same-version churn on one handle: update, clear, and re-add in one transaction. + map.prepareNextVersion(); + map.insert("key-10", next_value(10)); // generation 2 + map.clear("key-10"); + map.insert("key-10", next_value(10)); // generation 3 + const auto version2 = map.publishNextVersion(); + ASSERT_EQ(version2, 2); + expect_snapshot_valid("after same-version clear and re-add publish"); + + // Exercise both the post-unlink/pre-deletion state and the stable post-deletion state. + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + expect_snapshot_valid("after first-phase gc of same-version churn publish"); + } + expect_snapshot_valid("after deferred deletion of same-version churn publish"); + + map.prepareNextVersion(); + map.clear("key-11"); + map.insert("key-12", next_value(12)); + const auto version3 = map.publishNextVersion(); + ASSERT_EQ(version3, 3); + expect_snapshot_valid("after mixed clear and insert publish"); + + { + auto deferred = map.gc(version3); + expect_snapshot_valid("after first-phase final gc"); + } + expect_snapshot_valid("after final deferred deletion"); +} + +TEST_F(VersionedTest, VersionedMapChaosConcurrentReadersAndWriter) { + TrackedMap map; + + constexpr size_t k_reader_count = 4; + constexpr int k_key_count = 32; + constexpr int k_initial_key_count = 16; + constexpr int k_max_ops_per_transaction = 8; + constexpr int k_transaction_count = 2000; + constexpr uint32_t k_seed = 1337; + + std::array key_names; + std::array generations{}; + for (int i = 0; i < k_key_count; ++i) { + key_names[i] = "key-" + std::to_string(i); + } + + auto next_value = [&](int key_id) { return new TrackedValue(key_id, ++generations[key_id]); }; + + map.prepareNextVersion(); + for (int i = 0; i < k_initial_key_count; ++i) { + map.insert(key_names[i], next_value(i)); + } + auto published_version = map.publishNextVersion(); + ASSERT_GT(published_version, 0); + + uint64_t next_snapshot_id = 1; + std::shared_ptr published_snapshot_owner = + makePublishedHandles(map, next_snapshot_id++); + std::shared_ptr previous_snapshot_owner; + std::atomic published_snapshot{published_snapshot_owner.get()}; + std::array, k_reader_count> reader_quiesced_versions; + std::array, k_reader_count> reader_snapshot_ids; + for (auto& reader_version : reader_quiesced_versions) { + reader_version.store(versionMin, std::memory_order_relaxed); + } + for (auto& reader_snapshot_id : reader_snapshot_ids) { + reader_snapshot_id.store(0, std::memory_order_relaxed); + } + + std::atomic stop{false}; + std::atomic failed{false}; + Envoy::Thread::MutexBasicLockable failure_mutex; + std::string failure_message; + Envoy::Thread::MutexBasicLockable history_mutex; + std::vector recent_history; + + auto record_failure = [&](const std::string& message) { + bool expected = false; + if (failed.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + Envoy::Thread::LockGuard lock(failure_mutex); + failure_message = message; + } + }; + + auto record_history = [&](std::string message) { + Envoy::Thread::LockGuard lock(history_mutex); + recent_history.push_back(std::move(message)); + constexpr size_t k_history_limit = 128; + if (recent_history.size() > k_history_limit) { + recent_history.erase(recent_history.begin(), + recent_history.begin() + (recent_history.size() - k_history_limit)); + } + }; + + std::vector readers; + readers.reserve(k_reader_count); + for (size_t i = 0; i < k_reader_count; ++i) { + readers.emplace_back([&, i]() { + while (!stop.load(std::memory_order_acquire)) { + const auto* snapshot = published_snapshot.load(std::memory_order_acquire); + if (snapshot == nullptr) { + continue; + } + if (const auto error = validatePublishedSnapshotAtVersion( + *snapshot, snapshot->version, ValidationMode::ConcurrentSafeInvisibleTail); + !error.empty()) { + record_failure("reader " + std::to_string(i) + + " snapshot_id=" + std::to_string(snapshot->snapshot_id) + ": " + error); + break; + } + reader_quiesced_versions[i].store(snapshot->version, std::memory_order_release); + reader_snapshot_ids[i].store(snapshot->snapshot_id, std::memory_order_release); + } + }); + } + + auto publish_snapshot = [&]() { + previous_snapshot_owner = std::move(published_snapshot_owner); + published_snapshot_owner = makePublishedHandles(map, next_snapshot_id++); + published_snapshot.store(published_snapshot_owner.get(), std::memory_order_release); + return published_snapshot_owner; + }; + + auto release_previous_snapshot = [&]() { previous_snapshot_owner.reset(); }; + + auto wait_for_readers_to_observe_snapshot = + [&](const std::shared_ptr& snapshot) { + while (!failed.load(std::memory_order_acquire)) { + bool all_quiesced = true; + for (size_t i = 0; i < k_reader_count; ++i) { + if (reader_quiesced_versions[i].load(std::memory_order_acquire) < snapshot->version || + reader_snapshot_ids[i].load(std::memory_order_acquire) < snapshot->snapshot_id) { + all_quiesced = false; + break; + } + } + if (all_quiesced) { + return true; + } + std::this_thread::yield(); + } + return false; + }; + + auto run_deferred_gc = [&](uint64_t version) { + record_history("gc phase1 " + std::to_string(version)); + auto deferred = map.gc(version); + if (deferred.empty()) { + return true; + } + + const auto post_gc_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(post_gc_snapshot)) { + return false; + } + release_previous_snapshot(); + record_history("gc phase2 " + std::to_string(version)); + return true; + }; + + auto build_active_set = [&]() { + std::array active{}; + for (int i = 0; i < k_key_count; ++i) { + active[i] = map.find(key_names[i]) != nullptr; + } + return active; + }; + + auto random_matching_key = [&](std::minstd_rand& rng, const std::array& active, + bool want_active) { + std::vector candidates; + candidates.reserve(k_key_count); + for (int i = 0; i < k_key_count; ++i) { + if (active[i] == want_active) { + candidates.push_back(i); + } + } + if (candidates.empty()) { + return -1; + } + return candidates[rng() % candidates.size()]; + }; + + std::minstd_rand rng(k_seed); + uint64_t transaction_index = 0; + while (transaction_index < k_transaction_count && !failed.load(std::memory_order_acquire)) { + auto active = build_active_set(); + const auto next_version = map.prepareNextVersion(); + ++transaction_index; + record_history("tx " + std::to_string(transaction_index) + + " prepare next=" + std::to_string(next_version)); + + const int op_count = 1 + (rng() % k_max_ops_per_transaction); + for (int op_index = 0; op_index < op_count; ++op_index) { + const bool has_active = std::ranges::any_of(active, [](bool is_active) { return is_active; }); + const bool has_absent = + std::ranges::any_of(active, [](bool is_active) { return !is_active; }); + + std::vector available_ops; + if (has_active) { + available_ops.push_back(0); // update existing + available_ops.push_back(1); // clear existing + available_ops.push_back(3); // clear and re-add + } + if (has_absent) { + available_ops.push_back(2); // insert absent + } + ASSERT_FALSE(available_ops.empty()); + + switch (available_ops[rng() % available_ops.size()]) { + case 0: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": update " + key_names[key_id] + " -> gen " + + std::to_string(generations[key_id] + 1)); + map.insert(key_names[key_id], next_value(key_id)); + break; + } + case 1: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": clear " + key_names[key_id]); + map.clear(key_names[key_id]); + active[key_id] = false; + break; + } + case 2: { + const int key_id = random_matching_key(rng, active, false); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": insert " + key_names[key_id] + " -> gen " + + std::to_string(generations[key_id] + 1)); + map.insert(key_names[key_id], next_value(key_id)); + active[key_id] = true; + break; + } + case 3: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": clear+insert " + key_names[key_id] + + " -> gen " + std::to_string(generations[key_id] + 1)); + map.clear(key_names[key_id]); + map.insert(key_names[key_id], next_value(key_id)); + active[key_id] = true; + break; + } + default: + FAIL() << "unexpected operation"; + } + } + + if ((rng() % 5) == 0) { + record_history("tx " + std::to_string(transaction_index) + " revert"); + auto deferred = map.revert(); + if (!deferred.empty()) { + const auto reverted_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(reverted_snapshot)) { + break; + } + record_history("revert deferred delete"); + } + continue; + } + + published_version = map.publishNextVersion(); + record_history("tx " + std::to_string(transaction_index) + " publish -> " + + std::to_string(published_version)); + if (published_version == 0) { + continue; + } + + const auto next_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(next_snapshot)) { + break; + } + release_previous_snapshot(); + if (!run_deferred_gc(published_version)) { + break; + } + } + + if (!failed.load(std::memory_order_acquire)) { + const auto final_version = map.getVersion(); + const auto final_snapshot = publish_snapshot(); + if (final_version > versionMin) { + EXPECT_TRUE(wait_for_readers_to_observe_snapshot(final_snapshot)); + release_previous_snapshot(); + if (!failed.load(std::memory_order_acquire)) { + EXPECT_TRUE(run_deferred_gc(final_version)); + } + } + } + + stop.store(true, std::memory_order_release); + for (auto& reader : readers) { + reader.join(); + } + published_snapshot.store(nullptr, std::memory_order_release); + previous_snapshot_owner.reset(); + published_snapshot_owner.reset(); + + if (failed.load(std::memory_order_acquire)) { + Envoy::Thread::LockGuard lock(failure_mutex); + std::ostringstream history; + { + Envoy::Thread::LockGuard history_lock(history_mutex); + for (const auto& entry : recent_history) { + history << "\n " << entry; + } + } + FAIL() << "seed=" << k_seed << " " << failure_message << "\nrecent history:" << history.str(); + } +} + +TEST_F(VersionedTest, VersionedMapGcAndDestructorDeleteValuesExactlyOnce) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + auto published_version = map.publishNextVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + published_version = map.publishNextVersion(); + + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + auto deferred = map.gc(published_version); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +} // namespace