diff --git a/WORKSPACE b/WORKSPACE index 15332cd5e..7f0120e3e 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -43,6 +43,7 @@ git_repository( "@//patches:0004-thread_local-reset-slot-in-worker-threads-first.patch", "@//patches:0005-http-header-expose-attribute.patch", "@//patches:0006-test-integration-Defer-fake-upstream-read-enable-un.patch", + "@//patches:0007-config-add-grpc-mux-stream-event-callback.patch", "@//patches:0008-repo-Make-yq-dependency-optional-for-CI-config-parsi.patch", ], # // clang-format off: Envoy's format check: Only repository_locations.bzl may contains URL references diff --git a/cilium/BUILD b/cilium/BUILD index ca9e1269e..1af2056e8 100644 --- a/cilium/BUILD +++ b/cilium/BUILD @@ -28,6 +28,17 @@ envoy_cc_library( ], ) +envoy_cc_library( + name = "versioned_lib", + hdrs = ["versioned.h"], + repository = "@envoy", + deps = [ + "@com_google_absl//absl/container:flat_hash_map", + "@com_google_absl//absl/container:flat_hash_set", + "@envoy//source/common/common:assert_lib", + ], +) + envoy_cc_library( name = "network_policy_lib", srcs = [ @@ -45,6 +56,7 @@ envoy_cc_library( "//cilium:conntrack_lib", "//cilium:grpc_subscription_lib", "//cilium:ipcache_lib", + "//cilium:versioned_lib", "//cilium/api:npds_cc_proto", "@envoy//envoy/singleton:manager_interface", "@envoy//source/common/common:logger_lib", diff --git a/cilium/api/bpf_metadata.proto b/cilium/api/bpf_metadata.proto index 4f29224b9..ee857812a 100644 --- a/cilium/api/bpf_metadata.proto +++ b/cilium/api/bpf_metadata.proto @@ -87,4 +87,11 @@ message BpfMetadata { // Configuration for the source of Cilium xDS updates. // Used for all cilium-specific xDS protocol, e.g., NPHDS, NPDS, and Secrets (SDS) therein. envoy.config.core.v3.ConfigSource cilium_config_source = 16; + + // Policy type to use + enum PolicyType { + NPDS = 0; // Legacy NPDS (default) + NPRDS = 1; // New NetworkPolicyResource (NPRDS) + } + bool policy_type = 17; } diff --git a/cilium/api/npds.proto b/cilium/api/npds.proto index 1a43692e9..f66937635 100644 --- a/cilium/api/npds.proto +++ b/cilium/api/npds.proto @@ -17,9 +17,14 @@ import "validate/validate.proto"; // [#protodoc-title: Network policy management and NPDS] // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. service NetworkPolicyDiscoveryService { option (envoy.annotations.resource).type = "cilium.NetworkPolicy"; + rpc DeltaNetworkPolicies(stream envoy.service.discovery.v3.DeltaDiscoveryRequest) + returns (stream envoy.service.discovery.v3.DeltaDiscoveryResponse) { + } + rpc StreamNetworkPolicies(stream envoy.service.discovery.v3.DiscoveryRequest) returns (stream envoy.service.discovery.v3.DiscoveryResponse) { } @@ -33,6 +38,36 @@ service NetworkPolicyDiscoveryService { } } +// Policy and selector resource names are exact-match identifiers in NPRDS. +service NetworkPolicyResourceDiscoveryService { + option (envoy.annotations.resource).type = "cilium.NetworkPolicyResource"; + + rpc StreamNetworkPolicyResources(stream envoy.service.discovery.v3.DiscoveryRequest) + returns (stream envoy.service.discovery.v3.DiscoveryResponse) { + } + + rpc DeltaNetworkPolicyResources(stream envoy.service.discovery.v3.DeltaDiscoveryRequest) + returns (stream envoy.service.discovery.v3.DeltaDiscoveryResponse) { + } +} + +// An NPRDS resource that carries either an endpoint policy or a shared selector. +message NetworkPolicyResource { + oneof resource { + NetworkPolicy policy = 1; + Selector selector = 2; + } +} + +// A shared set of remote identities referenced by selector resource name. +// Unlike the old state-of-the-world remote identity lists, an empty selector +// matches nothing. +message Selector { + // The set of numeric remote security IDs selected by this selector. + // If empty, this selector selects no remote identities. + repeated uint32 remote_identities = 1; +} + // A network policy that is enforced by a filter on the network flows to/from // associated hosts. message NetworkPolicy { @@ -153,6 +188,12 @@ message PortNetworkPolicyRule { // Optional. If not specified, any remote host is matched by this predicate. repeated uint32 remote_policies = 7; + // Optional selector resource names that can be resolved to shared remote + // policy sets in delta NPDS. + // Selector references are matched by exact selector resource name. + // Optional. If not specified, any remote host is matched by this predicate. + repeated string selectors = 11; + // Optional downstream TLS context. If present, the incoming connection must // be a TLS connection. TLSContext downstream_tls_context = 3; diff --git a/cilium/api/nphds.proto b/cilium/api/nphds.proto index 82a6dbb2a..bb5628510 100644 --- a/cilium/api/nphds.proto +++ b/cilium/api/nphds.proto @@ -26,6 +26,10 @@ service NetworkPolicyHostsDiscoveryService { body: "*" }; } + + rpc DeltaNetworkPolicyHosts(stream envoy.service.discovery.v3.DeltaDiscoveryRequest) + returns (stream envoy.service.discovery.v3.DeltaDiscoveryResponse) { + } } // The mapping of a network policy identifier to the IP addresses of all the diff --git a/cilium/bpf_metadata.cc b/cilium/bpf_metadata.cc index f386e94f4..405a24400 100644 --- a/cilium/bpf_metadata.cc +++ b/cilium/bpf_metadata.cc @@ -235,14 +235,15 @@ Config::Config(const ::cilium::BpfMetadata& config, config.ipv6_source_address())); } if (config.use_nphds()) { - hosts_ = - context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(cilium_host_map), - [&context, config_source = config_source_] { - auto map = std::make_shared(context.serverFactoryContext()); - map->startSubscription(context.serverFactoryContext(), config_source); - return map; - }); + hosts_ = context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(cilium_host_map), + [&context, config_source = config_source_] { + auto map = std::make_shared(context.serverFactoryContext()); + map->startSubscription(context.serverFactoryContext(), config_source); + return map; + }); + // update desired config source on the map + hosts_->setConfigSource(config_source_); } // Note: all instances use the bpf root of the first filter with non-empty @@ -279,12 +280,15 @@ Config::Config(const ::cilium::BpfMetadata& config, // instances! // Only created if either ipcache_ or hosts_ map exists if (ipcache_ || hosts_) { - npmap_ = - context.serverFactoryContext().singletonManager().getTyped( - SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), - [&context, config_source = config_source_] { - return std::make_shared(context, config_source, true); - }); + bool use_nprds = config.policy_type() == cilium::BpfMetadata::NPRDS; + npmap_ = context.serverFactoryContext().singletonManager().getTyped( + SINGLETON_MANAGER_REGISTERED_NAME(cilium_network_policy), + [&context, use_nprds, config_source = config_source_] { + return std::make_shared(context, use_nprds, config_source, + true); + }); + // update desired config on the map + npmap_->setConfig(use_nprds, config_source_); } } diff --git a/cilium/bpf_metadata.h b/cilium/bpf_metadata.h index 753578a5c..8ba3c9e5b 100644 --- a/cilium/bpf_metadata.h +++ b/cilium/bpf_metadata.h @@ -175,10 +175,10 @@ class Config : public Cilium::PolicyResolver, Random::RandomGenerator& random_; envoy::config::core::v3::ConfigSource config_source_; - std::shared_ptr npmap_; + std::shared_ptr npmap_; Cilium::CtMapSharedPtr ct_maps_; Cilium::IpCacheSharedPtr ipcache_; - std::shared_ptr hosts_; + std::shared_ptr hosts_; private: uint32_t resolveSourceIdentity(const Network::Address::Ip* sip, const Network::Address::Ip* dip, diff --git a/cilium/grpc_subscription.cc b/cilium/grpc_subscription.cc index 1bfdd9a9e..57013a4de 100644 --- a/cilium/grpc_subscription.cc +++ b/cilium/grpc_subscription.cc @@ -2,16 +2,18 @@ #include -#include +#include #include #include #include #include #include "envoy/annotations/resource.pb.h" +#include "envoy/common/callback.h" #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" #include "envoy/config/custom_config_validators.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/config/subscription_factory.h" #include "envoy/grpc/async_client.h" @@ -25,9 +27,12 @@ #include "source/common/grpc/common.h" #include "source/common/protobuf/protobuf.h" // IWYU pragma: keep #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" +#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" #include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" +#include "source/extensions/config_subscription/grpc/new_grpc_mux_impl.h" #include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" #include "absl/status/statusor.h" #include "absl/strings/match.h" #include "absl/strings/string_view.h" @@ -38,6 +43,31 @@ namespace Cilium { namespace { +class StreamEventSubscription : public Config::Subscription { +public: + StreamEventSubscription(std::unique_ptr subscription, + Common::CallbackHandlePtr stream_event_handle) + : subscription_(std::move(subscription)), + stream_event_handle_(std::move(stream_event_handle)) {} + + void start(const absl::flat_hash_set& resource_names) override { + subscription_->start(resource_names); + } + + void + updateResourceInterest(const absl::flat_hash_set& update_to_these_names) override { + subscription_->updateResourceInterest(update_to_these_names); + } + + void requestOnDemandUpdate(const absl::flat_hash_set& add_these_names) override { + subscription_->requestOnDemandUpdate(add_these_names); + } + +private: + std::unique_ptr subscription_; + Common::CallbackHandlePtr stream_event_handle_; +}; + // service RPC method fully qualified names. struct Service { std::string sotw_grpc_method_; @@ -57,6 +87,7 @@ TypeUrlToServiceMap* buildTypeUrlToServiceMap() { // https://www.mail-archive.com/protobuf@googlegroups.com/msg04540.html. for (absl::string_view name : { "cilium.NetworkPolicyDiscoveryService", + "cilium.NetworkPolicyResourceDiscoveryService", "cilium.NetworkPolicyHostsDiscoveryService", }) { const auto* service_desc = @@ -124,49 +155,81 @@ subscribe(const absl::string_view type_url, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, - std::chrono::milliseconds init_fetch_timeout) { - auto& api_config_source = config_source.api_config_source(); - THROW_IF_NOT_OK(Config::Utility::checkApiConfigSourceSubscriptionBackingCluster( - context.clusterManager().primaryClusters(), api_config_source)); - + Config::GrpcMuxStreamEventCallback on_stream_event) { + auto initial_fetch_timeout = Config::Utility::configSourceInitialFetchTimeout(config_source); Config::SubscriptionStats stats = Config::Utility::generateStats(scope); Envoy::Config::SubscriptionOptions options; - // No-op custom validators - Envoy::Config::CustomConfigValidatorsPtr nop_config_validators = - std::make_unique(); - auto factory_or_error = Config::Utility::factoryForGrpcApiConfigSource( - context.clusterManager().grpcAsyncClientManager(), api_config_source, scope, true, 0, false); - THROW_IF_NOT_OK_REF(factory_or_error.status()); - - absl::StatusOr rate_limit_settings_or_error = - Config::Utility::parseRateLimitSettings(api_config_source); - THROW_IF_NOT_OK_REF(rate_limit_settings_or_error.status()); - - Config::GrpcMuxContext grpc_mux_context{ - /*async_client_=*/THROW_OR_RETURN_VALUE( - factory_or_error.value()->createUncachedRawAsyncClient(), Grpc::RawAsyncClientPtr), - /*failover_async_client_=*/nullptr, - /*dispatcher_=*/context.mainThreadDispatcher(), - /*service_method_=*/sotwGrpcMethod(type_url), - /*local_info_=*/context.localInfo(), - /*rate_limit_settings_=*/rate_limit_settings_or_error.value(), - /*scope_=*/scope, - /*config_validators_=*/std::move(nop_config_validators), - /*xds_resources_delegate_=*/absl::nullopt, - /*xds_config_tracker_=*/absl::nullopt, - /*backoff_strategy_=*/ - std::make_unique( - Config::SubscriptionFactory::RetryInitialDelayMs, - Config::SubscriptionFactory::RetryMaxDelayMs, context.api().randomGenerator()), - /*target_xds_authority_=*/"", - /*eds_resources_cache_=*/nullptr, // EDS cache is only used for ADS. - /*skip_subsequent_node_=*/api_config_source.set_node_on_first_message_only(), - }; - - return std::make_unique( - std::make_shared(grpc_mux_context), callbacks, resource_decoder, stats, type_url, - context.mainThreadDispatcher(), init_fetch_timeout, /*is_aggregated*/ false, options); + std::shared_ptr grpc_mux; + bool is_aggregated = + config_source.config_source_specifier_case() == envoy::config::core::v3::ConfigSource::kAds; + if (is_aggregated) { + grpc_mux = std::static_pointer_cast(context.xdsManager().adsMux()); + } else { + auto& api_config_source = config_source.api_config_source(); + THROW_IF_NOT_OK(Config::Utility::checkApiConfigSourceSubscriptionBackingCluster( + context.clusterManager().primaryClusters(), api_config_source)); + + // No-op custom validators + Envoy::Config::CustomConfigValidatorsPtr nop_config_validators = + std::make_unique(); + auto factory_or_error = Config::Utility::factoryForGrpcApiConfigSource( + context.clusterManager().grpcAsyncClientManager(), api_config_source, scope, true, 0, + false); + THROW_IF_NOT_OK_REF(factory_or_error.status()); + + absl::StatusOr rate_limit_settings_or_error = + Config::Utility::parseRateLimitSettings(api_config_source); + THROW_IF_NOT_OK_REF(rate_limit_settings_or_error.status()); + + const auto& api_type = api_config_source.api_type(); + bool use_delta = api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; + const auto& service_method = use_delta ? deltaGrpcMethod(type_url) : sotwGrpcMethod(type_url); + + Config::GrpcMuxContext grpc_mux_context{ + THROW_OR_RETURN_VALUE(factory_or_error.value()->createUncachedRawAsyncClient(), + Grpc::RawAsyncClientPtr), + /*failover_async_client_=*/nullptr, + context.mainThreadDispatcher(), + service_method, + context.localInfo(), + rate_limit_settings_or_error.value(), + scope, + std::move(nop_config_validators), + /*xds_resources_delegate_=*/absl::nullopt, + /*xds_config_tracker_=*/absl::nullopt, + std::make_unique( + Config::SubscriptionFactory::RetryInitialDelayMs, + Config::SubscriptionFactory::RetryMaxDelayMs, context.api().randomGenerator()), + /*target_xds_authority_=*/"", + /*eds_resources_cache_=*/nullptr, // EDS cache is only used for ADS. + /*skip_subsequent_node_=*/api_config_source.set_node_on_first_message_only(), + }; + + grpc_mux = use_delta ? std::static_pointer_cast( + std::make_shared(grpc_mux_context)) + : std::static_pointer_cast( + std::make_shared(grpc_mux_context)); + } + + Common::CallbackHandlePtr stream_event_handle; + if (on_stream_event) { + auto stream_event_callback = std::move(on_stream_event); + stream_event_handle = grpc_mux->addStreamEventCallback(stream_event_callback); + if (grpc_mux->grpcStreamConnected()) { + stream_event_callback(Config::GrpcMuxStreamEvent::Established); + } + } + + auto subscription = std::make_unique( + grpc_mux, callbacks, resource_decoder, stats, type_url, context.mainThreadDispatcher(), + initial_fetch_timeout, is_aggregated, options); + if (stream_event_handle) { + return std::make_unique(std::move(subscription), + std::move(stream_event_handle)); + } + return subscription; } } // namespace Cilium diff --git a/cilium/grpc_subscription.h b/cilium/grpc_subscription.h index 10d5b5155..1e8e85f08 100644 --- a/cilium/grpc_subscription.h +++ b/cilium/grpc_subscription.h @@ -1,51 +1,24 @@ #pragma once -#include #include #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" -#include "envoy/server/factory_context.h" +#include "envoy/ssl/context_manager.h" #include "envoy/stats/scope.h" -#include "source/extensions/config_subscription/grpc/grpc_mux_context.h" -#include "source/extensions/config_subscription/grpc/grpc_mux_impl.h" - #include "absl/strings/string_view.h" namespace Envoy { namespace Cilium { -// GrpcMux wrapper to get access to control plane identifier -class GrpcMuxImpl : public Config::GrpcMuxImpl { -public: - GrpcMuxImpl(Config::GrpcMuxContext& grpc_mux_context) : Config::GrpcMuxImpl(grpc_mux_context) {} - - ~GrpcMuxImpl() override = default; - - void onStreamEstablished() override { - new_stream_ = true; - Config::GrpcMuxImpl::onStreamEstablished(); - } - - // isNewStream returns true for the first call after a new stream has been established - bool isNewStream() { - bool new_stream = new_stream_; - new_stream_ = false; - return new_stream; - } - -private: - bool new_stream_ = true; -}; - std::unique_ptr subscribe(const absl::string_view type_url, const envoy::config::core::v3::ConfigSource& config_source, Server::Configuration::CommonFactoryContext& context, Stats::Scope& scope, Config::SubscriptionCallbacks& callbacks, Config::OpaqueResourceDecoderSharedPtr resource_decoder, - std::chrono::milliseconds init_fetch_timeout = std::chrono::milliseconds(0)); - + Config::GrpcMuxStreamEventCallback on_stream_event = {}); } // namespace Cilium } // namespace Envoy diff --git a/cilium/host_map.cc b/cilium/host_map.cc index ddf67da2f..6c0c912c4 100644 --- a/cilium/host_map.cc +++ b/cilium/host_map.cc @@ -4,15 +4,18 @@ #include #include +#include #include #include #include #include +#include #include #include #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/event/dispatcher.h" #include "envoy/server/factory_context.h" @@ -21,9 +24,11 @@ #include "envoy/thread_local/thread_local.h" #include "envoy/thread_local/thread_local_object.h" +#include "source/common/common/assert.h" #include "source/common/common/logger.h" #include "source/common/common/macros.h" +#include "absl/container/flat_hash_set.h" #include "absl/numeric/int128.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -58,9 +63,18 @@ unsigned int checkPrefix(T addr, bool have_prefix, unsigned int plen, absl::stri } // namespace struct ThreadLocalHostMapInitializer : public PolicyHostMap::ThreadLocalHostMap { -protected: +public: friend class PolicyHostMap; // PolicyHostMap can insert(); + ThreadLocalHostMapInitializer() = default; + + explicit ThreadLocalHostMapInitializer(const PolicyHostMap::ThreadLocalHostMap* host_map) { + if (host_map != nullptr) { + static_cast(*this) = *host_map; + } + } + +protected: // find the map of the given prefix length, insert in the decreasing order if // it does not exist template @@ -159,6 +173,29 @@ struct ThreadLocalHostMapInitializer : public PolicyHostMap::ThreadLocalHostMap fmt::format("NetworkPolicyHosts: Invalid host entry \'{}\' for policy {}", host, policy)); } } + + template + void prunePolicyMapVec(MapVec& maps, const absl::flat_hash_set& nids) { + for (auto vec_it = maps.begin(); vec_it != maps.end();) { + auto& map = vec_it->second; + for (auto map_it = map.begin(); map_it != map.end();) { + auto it = map_it++; + if (nids.contains(it->second)) { + map.erase(it); + } + } + if (map.empty()) { + vec_it = maps.erase(vec_it); + } else { + ++vec_it; + } + } + } + + void remove(const absl::flat_hash_set& removed_nids) { + prunePolicyMapVec(ipv4_to_policy_, removed_nids); + prunePolicyMapVec(ipv6_to_policy_, removed_nids); + } }; uint64_t PolicyHostMap::instance_id_ = 0; @@ -196,22 +233,155 @@ PolicyHostMap::PolicyHostMap(Server::Configuration::CommonFactoryContext& contex } void PolicyHostMap::startSubscription(Server::Configuration::CommonFactoryContext& context, - const envoy::config::core::v3::ConfigSource& config_source) { - if (config_source.config_source_specifier_case() == envoy::config::core::v3::ConfigSource::kAds) { - auto ads_mux = context.xdsManager().adsMux(); - subscription_ = THROW_OR_RETURN_VALUE( - context.clusterManager().subscriptionFactory().subscriptionOverAdsGrpcMux( - ads_mux, config_source, NetworkPolicyHostsTypeUrl, *scope_, *this, - std::make_shared(), {}), - Config::SubscriptionPtr); - } else { - subscription_ = subscribe(NetworkPolicyHostsTypeUrl, config_source, context, *scope_, *this, - std::make_shared()); + const envoy::config::core::v3::ConfigSource& npds_config) { + context_ = &context; + desired_config_source_ = npds_config; + subscribe(); +} + +void PolicyHostMap::setConfigSource(const envoy::config::core::v3::ConfigSource& config_source) { + desired_config_source_ = config_source; + if (context_ != nullptr) { + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/false); + } +} + +bool PolicyHostMap::subscriptionUseDeltaXds() const { + if (!config_source_.has_api_config_source()) { + return false; } + const auto& api_type = config_source_.api_config_source().api_type(); + return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; +} +void PolicyHostMap::subscribe() { + ASSERT(context_ != nullptr); + subscription_connected_ = false; + config_source_ = desired_config_source_; + ++subscription_id_; + + auto on_stream_event = [weak_this = weak_from_this(), + id = subscription_id_](Config::GrpcMuxStreamEvent event) { + if (auto shared_this = weak_this.lock()) { + shared_this->onSubscriptionStreamEvent(id, event); + } + }; + + subscription_ = + Cilium::subscribe(NetworkPolicyHostsTypeUrl, config_source_, *context_, *scope_, *this, + std::make_shared(), std::move(on_stream_event)); subscription_->start({}); } +void PolicyHostMap::onSubscriptionStreamEvent(uint64_t subscription_id, + Config::GrpcMuxStreamEvent event) { + if (subscription_id != subscription_id_) { + return; + } + + switch (event) { + case Config::GrpcMuxStreamEvent::Established: + subscription_connected_ = true; + break; + case Config::GrpcMuxStreamEvent::Closed: + if (!subscription_connected_) { + return; + } + subscription_connected_ = false; + + if (context_ == nullptr) { + return; + } + + context_->mainThreadDispatcher().post( + [weak_this = weak_from_this(), subscription_id = subscription_id_]() { + if (auto shared_this = weak_this.lock()) { + if (subscription_id != shared_this->subscription_id_) { + return; + } + shared_this->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + } + }); + break; + } +} + +void PolicyHostMap::maybeRecreateSubscriptionInDesiredMode(bool transport_closed) { + if (subscription_ && (subscription_connected_ || !transport_closed)) { + if (subscription_connected_ && subscriptionUseDeltaXds()) { + return; + } + if (Protobuf::util::MessageDifferencer::Equals(config_source_, desired_config_source_)) { + return; + } + } + + subscribe(); +} + +absl::Status +PolicyHostMap::onConfigUpdate(const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + const bool is_new_stream = subscription_id_ != accepted_subscription_id_; + ENVOY_LOG( + debug, + "PolicyHostMap::onConfigUpdate({}), {} added_resources, {} removed_resources, version: {}, " + "subscription_id: {}, accepted_subscription_id: {}, is_new_stream: {}", + name_, added_resources.size(), removed_resources.size(), system_version_info, + subscription_id_, accepted_subscription_id_, is_new_stream); + + auto newmap = + std::make_shared(is_new_stream ? nullptr : getHostMap()); + + absl::flat_hash_set to_remove; + to_remove.reserve(added_resources.size() + removed_resources.size()); + + for (const auto& name : removed_resources) { + uint64_t nid = 0; + auto [ptr, ec] = std::from_chars(name.data(), name.data() + name.size(), nid); + if (ec != std::errc{} || ptr != name.data() + name.size()) { + throw EnvoyException(fmt::format("Invalid removed resource name '{}'", name)); + } + ENVOY_LOG(trace, + "Removing NetworkPolicyHosts for policy {} in delta onConfigUpdate() version {}", nid, + system_version_info); + to_remove.insert(nid); + } + for (const auto& resource : added_resources) { + const auto& config = dynamic_cast(resource.get().resource()); + to_remove.insert(config.policy()); + } + newmap->remove(to_remove); + + for (const auto& resource : added_resources) { + const auto& config = dynamic_cast(resource.get().resource()); + ENVOY_LOG(trace, + "Received NetworkPolicyHosts for policy {} in delta onConfigUpdate() version {}", + config.policy(), system_version_info); + newmap->insert(config); + } + + // Force 'this' to be not deleted for as long as the lambda stays + // alive. Note that generally capturing a shared pointer is + // dangerous as it may happen that there is a circular reference + // from 'this' to itself via the lambda capture, leading to 'this' + // never being released. It should happen in this case, though. + std::shared_ptr shared_this = shared_from_this(); + + // Assign the new map to all threads. + tls_->set([shared_this, newmap](Event::Dispatcher&) -> ThreadLocal::ThreadLocalObjectSharedPtr { + UNREFERENCED_PARAMETER(shared_this); + ENVOY_LOG(trace, "PolicyHostMap: Assigning new map"); + return newmap; + }); + logmaps("delta onConfigUpdate"); + accepted_subscription_id_ = subscription_id_; + stats_.update_success_.inc(); + return absl::OkStatus(); +} + absl::Status PolicyHostMap::onConfigUpdate(const std::vector& resources, const std::string& version_info) { diff --git a/cilium/host_map.h b/cilium/host_map.h index af07eec7a..280369121 100644 --- a/cilium/host_map.h +++ b/cilium/host_map.h @@ -15,6 +15,7 @@ #include "envoy/common/exception.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/network/address.h" #include "envoy/protobuf/message_validator.h" @@ -26,7 +27,6 @@ #include "envoy/thread_local/thread_local_object.h" #include "source/common/common/logger.h" -#include "source/common/common/macros.h" #include "source/common/network/utility.h" #include "source/common/protobuf/message_validator_impl.h" #include "source/common/protobuf/protobuf.h" @@ -118,6 +118,8 @@ class PolicyHostMap : public Singleton::Instance, void startSubscription(Server::Configuration::CommonFactoryContext& context, const envoy::config::core::v3::ConfigSource& config_source); + void setConfigSource(const envoy::config::core::v3::ConfigSource& config_source); + // This is used for testing with a file-based subscription void startSubscription(std::unique_ptr&& subscription) { subscription_ = std::move(subscription); @@ -229,22 +231,30 @@ class PolicyHostMap : public Singleton::Instance, const std::string& version_info) override; absl::Status onConfigUpdate(const std::vector& added_resources, const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override { - // NOT IMPLEMENTED YET. - UNREFERENCED_PARAMETER(added_resources); - UNREFERENCED_PARAMETER(removed_resources); - UNREFERENCED_PARAMETER(system_version_info); - return absl::OkStatus(); - } + const std::string& system_version_info) override; void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) override; private: + bool subscriptionUseDeltaXds() const; + void subscribe(); + void onSubscriptionStreamEvent(uint64_t subscription_id, Config::GrpcMuxStreamEvent event); + void maybeRecreateSubscriptionInDesiredMode(bool transport_closed); + ThreadLocal::SlotPtr tls_; std::string name_; Stats::ScopeSharedPtr scope_; Stats::ScopeSharedPtr stats_scope_; std::unique_ptr subscription_; + Server::Configuration::CommonFactoryContext* context_{nullptr}; + // We need a separate desired_config_source_ as it may be set to a pessimistic value via explicit + // BpfMetadata config in CiliumEnvoyConfig CRD, and we should not change to a "worse" (e.g., SotW) + // ConfigSource if "better" (e.g., Delta) is already up-and-running (in config_source_). + envoy::config::core::v3::ConfigSource desired_config_source_; + envoy::config::core::v3::ConfigSource config_source_; + uint64_t subscription_id_{0}; + uint64_t accepted_subscription_id_{0}; + bool subscription_connected_{false}; static uint64_t instance_id_; PolicyHostsStats stats_; }; diff --git a/cilium/network_policy.cc b/cilium/network_policy.cc index cf5d2141a..bdfececfe 100644 --- a/cilium/network_policy.cc +++ b/cilium/network_policy.cc @@ -7,11 +7,13 @@ #include #include +#include #include #include #include #include #include +#include #include #include @@ -22,6 +24,7 @@ #include "envoy/config/core/v3/address.pb.h" #include "envoy/config/core/v3/base.pb.h" #include "envoy/config/core/v3/config_source.pb.h" +#include "envoy/config/grpc_mux.h" #include "envoy/config/subscription.h" #include "envoy/event/dispatcher_thread_deletable.h" #include "envoy/http/header_map.h" @@ -39,7 +42,6 @@ #include "source/common/common/assert.h" #include "source/common/common/logger.h" -#include "source/common/common/macros.h" #include "source/common/common/matchers.h" #include "source/common/common/thread.h" #include "source/common/http/header_utility.h" @@ -49,7 +51,6 @@ #include "source/common/network/utility.h" #include "source/common/protobuf/protobuf.h" #include "source/common/protobuf/utility.h" -#include "source/extensions/config_subscription/grpc/grpc_subscription_impl.h" #include "source/server/transport_socket_config_impl.h" #include "absl/container/btree_map.h" @@ -61,16 +62,29 @@ #include "absl/strings/match.h" #include "absl/strings/str_replace.h" #include "absl/strings/string_view.h" +#include "absl/types/variant.h" #include "cilium/accesslog.h" #include "cilium/api/npds.pb.h" #include "cilium/grpc_subscription.h" #include "cilium/ipcache.h" #include "cilium/secret_watcher.h" +#include "cilium/versioned.h" namespace { static constexpr absl::string_view NetworkPolicyTypeUrl = "type.googleapis.com/cilium.NetworkPolicy"; +static constexpr absl::string_view NetworkPolicyResourceTypeUrl = + "type.googleapis.com/cilium.NetworkPolicyResource"; + +bool configSourceIsDelta(const envoy::config::core::v3::ConfigSource& config_source) { + if (!config_source.has_api_config_source()) { + return false; + } + const auto& api_type = config_source.api_config_source().api_type(); + return api_type == envoy::config::core::v3::ApiConfigSource::DELTA_GRPC || + api_type == envoy::config::core::v3::ApiConfigSource::AGGREGATED_DELTA_GRPC; +} } // namespace @@ -122,53 +136,291 @@ template <> struct formatter { namespace Envoy { namespace Cilium { +// PolicyStreamState is shared by all policies created from one accepted NPDS stream generation. +// Same-stream selector-only updates publish a newer selector version into this object so existing +// policies follow immediately. When the NPDS stream restarts, new policies get a fresh state +// object while old policies keep the old one until the old policy map has quiesced and been +// retired. This allows the new stream to reuse selector resource names so that the xDS server +// need not keep selector resource names in stable storage accross restarts. +class PolicyStreamState { +public: + explicit PolicyStreamState(uint64_t stream_generation, SelectorVersion version = versionMin) + : stream_generation_(stream_generation), version_(version) {} + + uint64_t streamGeneration() const { return stream_generation_; } + + SelectorVersion version() const { return version_.load(std::memory_order_acquire); } + + void publishVersion(SelectorVersion version) { + version_.store(version, std::memory_order_release); + } + +private: + const uint64_t stream_generation_; + std::atomic version_; +}; +using PolicyStreamStateSharedPtr = std::shared_ptr; +using PolicyStreamStateConstSharedPtr = std::shared_ptr; + +// A specific version of a selector used in a policy. Each update yields a new instance. +class SelectorInstance : public VersionedNode, + public absl::flat_hash_set {}; + +// Read-only series of specific selector insteances. +class NamedSelectorReadable : public VersionedReadable { +public: + explicit NamedSelectorReadable(const std::string& name) : name_(name) {} + + const std::string& name() const { return name_; } + +private: + std::string name_; +}; + +// Stable handle on a read-only selector for accessing specific versions of the selector. +using SelectorHandle = std::shared_ptr; + +// Writable series of selector versions for main-thread updates +class NamedSelectorValue : public VersionedValue { +public: + explicit NamedSelectorValue(const std::string& name) + : VersionedValue(name) {} +}; + +// Map of named selectors, keyed with xDS resource name +class SelectorMap : public VersionedMap { +public: + using VersionedMap::VersionedMap; +}; + class PolicyInstanceImpl; +// Stable policy map, usually keyed with endpoint IP (IPv4 and IPv6). using PolicyMapSnapshot = absl::flat_hash_map>; +// variant wrapper for supported resource map keys for delta policy updates +// Delta xDS refers to removed resources by resource name, so we must have a map to +// locate the policy/selector to be removed. +class ResourceKey { +public: + struct PolicyResourceEntry { + std::shared_ptr policy; + }; + + // reference to a policy entry by name + struct PolicyEndpointIpEntry { + std::string policy_name; + }; + + struct SelectorResourceEntry { + SelectorHandle handle; + }; + + static ResourceKey policyResource(const std::shared_ptr& policy) { + return ResourceKey(PolicyResourceEntry{policy}); + } + + static ResourceKey policyEndpointIp(const std::string& name) { + return ResourceKey(PolicyEndpointIpEntry{name}); + } + + static ResourceKey selectorResource(const SelectorHandle& handle) { + return ResourceKey(SelectorResourceEntry{handle}); + } + + const PolicyResourceEntry* policyResourceEntry() const { + return absl::get_if(&value_); + } + + const SelectorResourceEntry* selectorResourceEntry() const { + return absl::get_if(&value_); + } + + const PolicyEndpointIpEntry* policyEndpointIpEntry() const { + return absl::get_if(&value_); + } + + bool isPolicyEndpointIpEntry() const { + return absl::holds_alternative(value_); + } + +private: + explicit ResourceKey(const PolicyResourceEntry& value) : value_(value) {} + explicit ResourceKey(const PolicyEndpointIpEntry& value) : value_(value) {} + explicit ResourceKey(const SelectorResourceEntry& value) : value_(value) {} + + absl::variant value_; +}; + +// Map of Delta xDS resources for name collision and duplicate name detection. +class ResourceMap : public absl::flat_hash_map { +public: + using absl::flat_hash_map::flat_hash_map; + + const ResourceKey* findEntry(const std::string& key) const { + auto it = find(key); + return it != end() ? &it->second : nullptr; + } +}; + +// ResourceMapOverlay lets delta updates stage tentative resource-map removals and insertions on top +// of the current ResourceMap while validation is still in progress. This preserves transactional +// behavior without copying the full map: failed updates can be discarded cheaply, and successful +// ones are applied to the real map only after the whole update has been accepted. +class ResourceMapOverlay { +public: + ResourceMapOverlay() = default; + explicit ResourceMapOverlay(const ResourceMap& base) : base_(&base) {} + + const ResourceKey* findEntry(const std::string& key) const { + auto upsert_it = upserts_.find(key); + if (upsert_it != upserts_.end()) { + return &upsert_it->second; + } + if (removed_.contains(key)) { + return nullptr; + } + return base_ ? base_->findEntry(key) : nullptr; + } + + std::string describeExistingResourceKey(const std::string& key) const; + + SelectorHandle getSelectorHandleOrThrow(const std::string& selector) const { + const auto* entry = findEntry(selector); + if (entry == nullptr) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource rule references missing selector resource '{}'", selector)); + } + const auto* selector_entry = entry->selectorResourceEntry(); + if (selector_entry == nullptr || selector_entry->handle == nullptr) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource rule references non-selector resource '{}'", selector)); + } + return selector_entry->handle; + } + + bool emplace(const std::string& key, ResourceKey&& value) { + if (findEntry(key)) { + return false; + } + removed_.erase(key); + return upserts_.emplace(key, std::move(value)).second; + } + + bool insertOrUpdateSelectorResource(const std::string& key, const SelectorHandle& handle) { + if (upserts_.contains(key)) { + return false; + } + const auto* existing = findEntry(key); + if (existing != nullptr && existing->selectorResourceEntry() == nullptr) { + return false; + } + removed_.erase(key); + return upserts_.emplace(key, ResourceKey::selectorResource(handle)).second; + } + + void erase(const std::string& key) { + upserts_.erase(key); + if (base_ && base_->find(key) != base_->end()) { + removed_.insert(key); + } else { + removed_.erase(key); + } + } + + bool erasePolicyResourceIfPresent(const std::string& resource_name) { + const auto* entry = findEntry(resource_name); + if (entry == nullptr) { + return false; + } + const auto* policy_entry = entry->policyResourceEntry(); + if (policy_entry == nullptr) { + return false; + } + erasePolicyResource(resource_name, policy_entry->policy); + return true; + } + + void erasePolicyResource(const std::string& resource_name, + const std::shared_ptr& policy); + + PolicyMapSnapshot toPolicyMapSnapshot() const; + + void applyTo(ResourceMap& map) && { + if (!upserts_.empty()) { + map.reserve(map.size() + upserts_.size()); + } + for (const auto& key : removed_) { + map.erase(key); + } + for (auto& [key, value] : upserts_) { + map.insert_or_assign(std::move(key), std::move(value)); + } + } + +private: + template void forEachEntry(Visitor visitor) const { + if (base_) { + for (const auto& [key, value] : *base_) { + if (!removed_.contains(key) && !upserts_.contains(key)) { + visitor(key, value); + } + } + } + for (const auto& [key, value] : upserts_) { + visitor(key, value); + } + } + + std::pair> + findPolicyByEndpointIp(const std::string& endpoint_ip) const; + + const ResourceMap* base_{}; + absl::flat_hash_set removed_; + absl::flat_hash_map upserts_; +}; + +// helper for validating resource names. +void validateResourceName(const std::string& resource_name, absl::string_view subject) { + if (resource_name.empty()) { + throw EnvoyException(fmt::format("{} must not be empty", subject)); + } + if (std::ranges::any_of(resource_name, [](unsigned char c) { return absl::ascii_isspace(c); })) { + throw EnvoyException( + fmt::format("{} '{}' must not contain whitespace", subject, resource_name)); + } +} + class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, public Envoy::Event::DispatcherThreadDeletable, - public Logger::Loggable { + public Logger::Loggable, + public std::enable_shared_from_this { public: - NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, + friend class PortNetworkPolicyRule; + NetworkPolicyMapImpl(Server::Configuration::FactoryContext& context, bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source); ~NetworkPolicyMapImpl() override; - void startSubscription(const envoy::config::core::v3::ConfigSource& config_source) { - if (config_source.config_source_specifier_case() == - envoy::config::core::v3::ConfigSource::kAds) { - auto ads_mux = context_.xdsManager().adsMux(); - subscription_ = THROW_OR_RETURN_VALUE( - context_.clusterManager().subscriptionFactory().subscriptionOverAdsGrpcMux( - ads_mux, config_source, NetworkPolicyTypeUrl, *npds_stats_scope_, *this, - std::make_shared(), {}), - Config::SubscriptionPtr); - } else { - subscription_ = subscribe(NetworkPolicyTypeUrl, config_source, context_, *npds_stats_scope_, - *this, std::make_shared()); - } - } + void subscribe(); // This is used for testing with a file-based subscription - void startSubscription(std::unique_ptr&& subscription) { + void subscribe(std::unique_ptr&& subscription) { subscription_ = std::move(subscription); + current_config_ = desired_config_; + subscription_connected_ = false; } - const envoy::config::core::v3::ConfigSource& getConfigSource() const { return config_source_; } + const envoy::config::core::v3::ConfigSource& getConfigSource() const { + return current_config_.config_source_; + } // Config::SubscriptionCallbacks absl::Status onConfigUpdate(const std::vector& resources, const std::string& version_info) override; absl::Status onConfigUpdate(const std::vector& added_resources, const Protobuf::RepeatedPtrField& removed_resources, - const std::string& system_version_info) override { - // NOT IMPLEMENTED YET. - UNREFERENCED_PARAMETER(added_resources); - UNREFERENCED_PARAMETER(removed_resources); - UNREFERENCED_PARAMETER(system_version_info); - return absl::OkStatus(); - } + const std::string& system_version_info) override; void onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason, const EnvoyException* e) override; @@ -181,18 +433,12 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, void tlsWrapperMissingPolicyInc() const { stats_.tls_wrapper_missing_policy_.inc(); } protected: - bool isNewStream() const { - auto sub = dynamic_cast(subscription_.get()); - if (!sub) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcSubscriptionImpl"); - return false; - } - auto mux = dynamic_cast(sub->grpcMux().get()); - if (!mux) { - ENVOY_LOG(error, "Cilium NetworkPolicyMapImpl: Cannot get GrpcMuxImpl"); - return false; - } - return mux->isNewStream(); + uint64_t streamGeneration() const { return subscription_stream_generation_; } + void resetStreamForTest() { subscription_stream_generation_++; } + + void setConfig(bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source) { + desired_config_ = XdsConfig{use_nprds, config_source}; + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/false); } // run the given function after all the threads have scheduled @@ -204,19 +450,92 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, context_.threadLocal().runOnAllWorkerThreads([]() {}, cb); } - std::string resourceName(const cilium::NetworkPolicy& config) { - return fmt::format("{}", config.endpoint_id()); - } - void reopenIpcache(); - std::shared_ptr - createOrReusePolicy(const cilium::NetworkPolicy& config, const PolicyMapSnapshot& old_policy_map); + void validatePolicy(const std::string& resource_name, const cilium::NetworkPolicy& config, + const std::string& version_info); + + std::pair, bool> + createOrReusePolicy(const std::string& resource_name, const cilium::NetworkPolicy& config, + const PolicyStreamStateConstSharedPtr& policy_stream_state, + const ResourceMap& resource_map, bool is_new_stream, + const ResourceMapOverlay* pending_resource_map) const; + + std::pair createOrReuseSelector(const std::string& resource_name, + const cilium::Selector& config, + uint64_t update_version, + const ResourceMap& resource_map); - void installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, - Init::ManagerImpl& version_init_manager, std::string&& version_name); + void installNewPolicyMap(ResourceMapOverlay&& pending_resource_map, + Init::ManagerImpl& version_init_manager, std::string&& version_name, + const PolicyStreamStateSharedPtr& policy_stream_state); private: + void onSubscriptionStreamEvent(uint64_t subscription_id, Config::GrpcMuxStreamEvent event) { + // skip stale notifications for earlier subscriptions + if (subscription_id != subscription_id_) { + return; + } + switch (event) { + case Config::GrpcMuxStreamEvent::Established: + ++subscription_stream_generation_; + subscription_connected_ = true; + break; + case Config::GrpcMuxStreamEvent::Closed: + if (!subscription_connected_) { + return; + } + subscription_connected_ = false; + + // Test code executes synchronously + if (subscription_factory_for_test_) { + maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + return; + } + + // The close callback runs on the subscription object's own stack, so defer any possible + // recreation until after it unwinds to avoid destroying the current subscription + // mid-callback. + context_.mainThreadDispatcher().post( + [weak_this = weak_from_this(), subscription_id = subscription_id_]() { + if (auto shared_this = weak_this.lock()) { + // skip stale callbacks for earlier subscriptions + if (subscription_id != shared_this->subscription_id_) { + return; + } + shared_this->maybeRecreateSubscriptionInDesiredMode(/*transport_closed=*/true); + } + }); + break; + } + } + + void startSubscription() { + ASSERT(subscription_ != nullptr); + subscription_->start({}); + } + + void maybeRecreateSubscriptionInDesiredMode(bool transport_closed) { + // only ever skip subscribe if we have a subscription already, and it is already connected in + // Delta NPRDS or desired mode, or still connecting in desired mode. + if (subscription_ && (subscription_connected_ || !transport_closed)) { + if (subscription_connected_ && configSourceIsDelta(current_config_.config_source_) && + current_config_.use_nprds_) { + // Keep Delta NPRDS on a connected subscription until transport closes. + return; + } + if (current_config_.use_nprds_ == desired_config_.use_nprds_ && + Protobuf::util::MessageDifferencer::Equals(current_config_.config_source_, + desired_config_.config_source_)) { + // Let the current subscription keep going when it is already in the desired mode. + return; + } + } + + // Recreate the subscription in the latest desired mode. + subscribe(); + } + // Helpers for atomic swap of the policy map pointer. // // store() is only used for the initialization of the map during construction. @@ -256,15 +575,61 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, } const PolicyInstance* getPolicyInstanceImpl(const std::string& endpoint_policy_name) const; + PolicyInstanceConstSharedPtr + getPolicyInstanceSharedImpl(const std::string& endpoint_policy_name) const; + uint64_t policySelectorStreamGenerationForTestImpl(const PolicyInstance& policy) const; + SelectorVersion policySelectorVersionForTestImpl(const PolicyInstance& policy) const; void removeInitManager(); static uint64_t instance_id_; + bool subscription_connected_{false}; + uint64_t subscription_id_{0}; + bool subscription_should_start_{false}; + + void scheduleSelectorDeferredDeletion(DeferredDeletion&& deferred); + void scheduleSelectorGCAndDeferredDeletion(uint64_t published_version, + const PolicyMapSnapshot* old_policy_map = nullptr); + void startManagedSubscriptionForTest() { + subscription_should_start_ = true; + subscribe(); + } + void setSubscriptionFactoryForTest(NetworkPolicyMap::SubscriptionFactoryForTest factory) { + subscription_factory_for_test_ = std::move(factory); + } + void onSubscriptionConnectedForTest() { + onSubscriptionStreamEvent(subscription_id_, Config::GrpcMuxStreamEvent::Established); + } + void onSubscriptionTransportCloseForTest() { + onSubscriptionStreamEvent(subscription_id_, Config::GrpcMuxStreamEvent::Closed); + } + bool subscriptionConnectedForTest() const { return subscription_connected_; } + Server::Configuration::ServerFactoryContext& context_; + std::atomic map_ptr_; + SelectorMap selector_map_; + // Policies hold a shared per-stream state object. A freshly installed stream stores its actual + // gRPC stream generation here, so same-stream selector-only updates advance existing policies + // immediately while old policies remain pinned to the latest selector version reached by their + // own stream. + PolicyStreamStateSharedPtr policy_stream_state_{std::make_shared(0)}; + const ResourceMap empty_resource_map_; + ResourceMap resource_map_; Stats::ScopeSharedPtr npds_stats_scope_; Stats::ScopeSharedPtr policy_stats_scope_; + struct XdsConfig { + bool use_nprds_; + envoy::config::core::v3::ConfigSource config_source_; + }; + + // We need a separate desired_config_ as it may be set to a pessimistic value via explicit + // BpfMetadata config in CiliumEnvoyConfig CRD, and we should not change to a "worse" (e.g., SotW) + // ConfigSource if "better" (e.g., Delta NPRDS) is already up-and-running (in current_config_). + XdsConfig desired_config_; + XdsConfig current_config_; + // init target which starts gRPC subscription Init::TargetImpl init_target_; std::shared_ptr @@ -274,8 +639,9 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, // while parked, log and rotate it out before making it active again. std::unique_ptr parked_init_manager_; - envoy::config::core::v3::ConfigSource config_source_; std::unique_ptr subscription_; + static uint64_t subscription_stream_generation_; + NetworkPolicyMap::SubscriptionFactoryForTest subscription_factory_for_test_; ProtobufTypes::MessagePtr dumpNetworkPolicyConfigs(const Matchers::StringMatcher& name_matcher); Server::ConfigTracker::EntryOwnerPtr config_tracker_entry_; @@ -287,6 +653,7 @@ class NetworkPolicyMapImpl : public Envoy::Config::SubscriptionCallbacks, }; uint64_t NetworkPolicyMapImpl::instance_id_ = 0; +uint64_t NetworkPolicyMapImpl::subscription_stream_generation_ = 1; IpAddressPair::IpAddressPair(const cilium::NetworkPolicy& proto) { for (const auto& ip_addr : proto.endpoint_ips()) { @@ -650,7 +1017,8 @@ class PortNetworkPolicyRule : public Logger::Loggable { tier_last_precedence_(0), pass_index_(0), l7_proto_("") {} PortNetworkPolicyRule(const NetworkPolicyMapImpl& parent, - const cilium::PortNetworkPolicyRule& rule) + const cilium::PortNetworkPolicyRule& rule, + const ResourceMapOverlay* resource_map) : name_(rule.name()), verdict_(rule.pass_precedence() ? RuleVerdict::Pass : (rule.deny() ? RuleVerdict::Deny : RuleVerdict::Allow)), @@ -661,10 +1029,26 @@ class PortNetworkPolicyRule : public Logger::Loggable { fmt::format("PortNetworkPolicyRule: pass_precedence {} must be lower than precedence {}", tier_last_precedence_, precedence_)); } - for (const auto& remote : rule.remote_policies()) { - ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", verdict_, - remote, name_); - remotes_.emplace(remote); + if (resource_map) { + if (rule.remote_policies_size()) { + throw EnvoyException( + "NetworkPolicyResource rule must use selectors instead of remote_policies"); + } + selectors_.reserve(rule.selectors_size()); + for (const auto& selector : rule.selectors()) { + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} selector {} by rule: {}", verdict_, + selector, name_); + selectors_.emplace_back(resource_map->getSelectorHandleOrThrow(selector)); + } + } else { + if (rule.selectors_size()) { + throw EnvoyException("NetworkPolicy rule must not use selectors"); + } + for (const auto remote : rule.remote_policies()) { + ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRule(): {} remote {} by rule: {}", verdict_, + remote, name_); + remotes_.emplace(remote); + } } if (rule.has_downstream_tls_context()) { auto config = rule.downstream_tls_context(); @@ -701,29 +1085,41 @@ class PortNetworkPolicyRule : public Logger::Loggable { } } - // inheritpassprecedence bumps up the precedence of a rule in a lower tier to the precedence - // range reserved right after the precedence of the given pass rule. - void inheritPassPrecedence(const PortNetworkPolicyRule& pass_rule) { - precedence_ -= pass_rule.tier_last_precedence_; - precedence_ += pass_rule.precedence_; - } + bool isRemoteWildcard() const { return remotes_.empty() && selectors_.empty(); } - bool isRemoteWildcard() const { return remotes_.empty(); } + bool matchesRemoteId(uint32_t remote_id, const SelectorVersion selector_version) const { + if (isRemoteWildcard()) { + return true; + } + if (!remotes_.empty()) { + return remotes_.contains(remote_id); + } + + for (const auto& selector : selectors_) { + const auto resolved_selector = selector->get(selector_version); + if (resolved_selector && resolved_selector->contains(remote_id)) { + return true; + } + } + return false; + } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id) const { + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, + const SelectorVersion selector_version) const { // proxy_id must match if we have any. if (proxy_id_ && proxy_id != proxy_id_) { return RuleVerdict::None; } // Remote ID must match if we have any. - if (!isRemoteWildcard() && !remotes_.contains(remote_id)) { + if (!matchesRemoteId(remote_id, selector_version)) { return RuleVerdict::None; // no verdict } ASSERT(verdict_ != RuleVerdict::None, "rule must have a verdict"); return verdict_; } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, + const SelectorVersion selector_version) const { // sni must match if we have any if (!allowed_snis_.empty() && (sni.empty() || std::ranges::none_of(allowed_snis_, [&](const auto& pattern) { @@ -731,13 +1127,14 @@ class PortNetworkPolicyRule : public Logger::Loggable { }))) { return RuleVerdict::None; } - return getVerdict(proxy_id, remote_id); + return getVerdict(proxy_id, remote_id, selector_version); } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, Envoy::Http::RequestHeaderMap& headers, - Cilium::AccessLog::Entry& log_entry) const { - auto verdict = getVerdict(proxy_id, remote_id); + Cilium::AccessLog::Entry& log_entry, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (!hasHttpRules() || verdict != RuleVerdict::Allow) { return verdict; } @@ -761,8 +1158,9 @@ class PortNetworkPolicyRule : public Logger::Loggable { return (header_matched) ? RuleVerdict::Allow : RuleVerdict::None; } - RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { - auto verdict = getVerdict(proxy_id, remote_id); + RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (verdict != RuleVerdict::Allow) { return verdict; } @@ -777,8 +1175,9 @@ class PortNetworkPolicyRule : public Logger::Loggable { // Envoy Metadata matcher, called after deny has already been checked for RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, - const envoy::config::core::v3::Metadata& metadata) const { - auto verdict = getVerdict(proxy_id, remote_id); + const envoy::config::core::v3::Metadata& metadata, + const SelectorVersion selector_version) const { + auto verdict = getVerdict(proxy_id, remote_id, selector_version); if (verdict != RuleVerdict::Allow) { return verdict; } @@ -829,8 +1228,18 @@ class PortNetworkPolicyRule : public Logger::Loggable { } void toString(int indent, std::string& res) const { - res.append(indent - 2, ' ').append("- remotes: ["); - res.append(fmt::format("{}", fmt::join(remotes_, ","))); + if (!selectors_.empty()) { + res.append(indent - 2, ' ').append("- selectors: ["); + std::vector quoted_selectors; + quoted_selectors.reserve(selectors_.size()); + for (const auto& selector : selectors_) { + quoted_selectors.emplace_back(fmt::format("\"{}\"", selector->name())); + } + res.append(fmt::format("{}", fmt::join(quoted_selectors, ","))); + } else { + res.append(indent - 2, ' ').append("- remotes: ["); + res.append(fmt::format("{}", fmt::join(remotes_, ","))); + } res.append("]\n"); if (!name_.empty()) { @@ -888,16 +1297,17 @@ class PortNetworkPolicyRule : public Logger::Loggable { bool hasHttpRules() const { return http_rules_ && !http_rules_->empty(); } - std::string name_; + const std::string name_; DownstreamTLSContextSharedPtr server_context_; UpstreamTLSContextSharedPtr client_context_; bool has_headermatches_{false}; const RuleVerdict verdict_; const uint16_t proxy_id_; - uint32_t precedence_; - uint32_t tier_last_precedence_; + const uint32_t precedence_; + const uint32_t tier_last_precedence_; uint32_t pass_index_; absl::btree_set remotes_; + std::vector selectors_; std::vector allowed_snis_; // All SNIs allowed if empty. std::shared_ptr> @@ -951,13 +1361,15 @@ class PortNetworkPolicyRules : public Logger::Loggable { // First call marks 'rules_' as initialized. Of further calls, if either is empty, // we must add a default allow rule to retain the semantics of empty rules. void append(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* selector_resource_map) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (const auto& it : rules) { - rules_.emplace_back(std::make_shared(parent, it)); + rules_.emplace_back( + std::make_shared(parent, it, selector_resource_map)); updateFor(rules_.back()); } initialized_ = true; @@ -967,13 +1379,15 @@ class PortNetworkPolicyRules : public Logger::Loggable { // First call marks 'rules_' as initialized. Of further calls, if either is empty, // we must add a default allow rule to retain the semantics of an empty rules. void prepend(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* resource_map) { if (initialized_ && rules.empty() != rules_.empty()) { // add an explicit allow-all rule to keep the combined semantics rules_.emplace(rules_.begin(), std::make_shared()); } for (const auto& it : rules) { - rules_.emplace(rules_.begin(), std::make_shared(parent, it)); + rules_.emplace(rules_.begin(), + std::make_shared(parent, it, resource_map)); updateFor(rules_.front()); } initialized_ = true; @@ -984,7 +1398,6 @@ class PortNetworkPolicyRules : public Logger::Loggable { // we must add a default allow rule to retain the semantics of the combined rules. void appendRules(const std::vector& rules) { if (initialized_ && rules.empty() != rules_.empty()) { - // add an explicit allow-all rule to keep the combined semantics addDefaultAllowRule(); } for (auto& rule : rules) { @@ -1147,9 +1560,10 @@ class PortNetworkPolicyRules : public Logger::Loggable { RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, Envoy::Http::RequestHeaderMap& headers, - Cilium::AccessLog::Entry& log_entry) const { + Cilium::AccessLog::Entry& log_entry, + const SelectorVersion selector_version) const { auto verdict = forEachRule(can_short_circuit_, [&](const auto& rule) { - return rule.getVerdict(proxy_id, remote_id, headers, log_entry); + return rule.getVerdict(proxy_id, remote_id, headers, log_entry, selector_version); }); ENVOY_LOG(trace, @@ -1158,24 +1572,30 @@ class PortNetworkPolicyRules : public Logger::Loggable { return verdict; } - RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { - auto verdict = forEachRule( - true, [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }); + RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, + const SelectorVersion selector_version) const { + auto verdict = forEachRule(true, [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }); ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRules(proxy_id: {}, remote_id: {}, sni: {}): {}", proxy_id, remote_id, sni, verdict); return verdict; } - RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { - return forEachRule( - true, [&](const auto& rule) { return rule.useProxylib(proxy_id, remote_id, l7_proto); }); + RuleVerdict useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto, + const SelectorVersion selector_version) const { + return forEachRule(true, [&](const auto& rule) { + return rule.useProxylib(proxy_id, remote_id, l7_proto, selector_version); + }); } RuleVerdict getVerdict(uint16_t proxy_id, uint32_t remote_id, - const envoy::config::core::v3::Metadata& metadata) const { - auto verdict = forEachRule( - true, [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, metadata); }); + const envoy::config::core::v3::Metadata& metadata, + const SelectorVersion selector_version) const { + auto verdict = forEachRule(true, [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, metadata, selector_version); + }); ENVOY_LOG(trace, "Cilium L7 PortNetworkPolicyRules(proxy_id: {}, remote_id: {}, metadata: {}): {}", @@ -1185,20 +1605,24 @@ class PortNetworkPolicyRules : public Logger::Loggable { } RuleVerdict getServerTlsContext(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, - Ssl::ContextSharedPtr& tls_ctx, - const Ssl::ContextConfig*& config) const { + Ssl::ContextSharedPtr& tls_ctx, const Ssl::ContextConfig*& config, + const SelectorVersion selector_version) const { tls_ctx = nullptr; return forEachRulePred( - [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }, + [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }, [&](const auto& rule) { return rule.getServerTlsContext(tls_ctx, config); }); } RuleVerdict getClientTlsContext(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni, - Ssl::ContextSharedPtr& tls_ctx, - const Ssl::ContextConfig*& config) const { + Ssl::ContextSharedPtr& tls_ctx, const Ssl::ContextConfig*& config, + const SelectorVersion selector_version) const { tls_ctx = nullptr; return forEachRulePred( - [&](const auto& rule) { return rule.getVerdict(proxy_id, remote_id, sni); }, + [&](const auto& rule) { + return rule.getVerdict(proxy_id, remote_id, sni, selector_version); + }, [&](const auto& rule) { return rule.getClientTlsContext(tls_ctx, config); }); } @@ -1251,7 +1675,7 @@ struct PortRangeCompare { // PolicySnapshot is keyed by port ranges, and contains a list of PortNetworkPolicyRules's // applicable to this range. A list is needed as rules may come from multiple sources (e.g., -// resulting from use of named ports and numbered ports in Cilium Network Policy at the same time). +// resulting from use of named ports and numbered ports in Cilium NetworkPolicy at the same time). class PolicySnapshot : public absl::btree_map { public: using absl::btree_map::btree_map; @@ -1279,13 +1703,14 @@ const PortNetworkPolicyRules* findPortRules(const PolicySnapshot& map, uint16_t } // namespace -PortPolicy::PortPolicy(const PolicySnapshot& map, uint16_t port) +PortPolicy::PortPolicy(const PolicySnapshot& map, uint16_t port, SelectorVersion selector_version) : port_rules_(findPortRules(map, port)), - has_http_rules_(port_rules_ && port_rules_->hasHttpRules()) {} + has_http_rules_(port_rules_ && port_rules_->hasHttpRules()), + selector_version_(selector_version) {} bool PortPolicy::useProxylib(uint16_t proxy_id, uint32_t remote_id, std::string& l7_proto) const { if (port_rules_) { - auto verdict = port_rules_->useProxylib(proxy_id, remote_id, l7_proto); + auto verdict = port_rules_->useProxylib(proxy_id, remote_id, l7_proto, selector_version_); if (verdict == RuleVerdict::Allow) { return true; } @@ -1305,14 +1730,15 @@ bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, headers, log_entry) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, headers, log_entry, selector_version_) == + RuleVerdict::Allow; } bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, absl::string_view sni) const { if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, sni) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, sni, selector_version_) == RuleVerdict::Allow; } bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, @@ -1320,7 +1746,8 @@ bool PortPolicy::allowed(uint16_t proxy_id, uint32_t remote_id, if (!port_rules_) { return false; } - return port_rules_->getVerdict(proxy_id, remote_id, metadata) == RuleVerdict::Allow; + return port_rules_->getVerdict(proxy_id, remote_id, metadata, selector_version_) == + RuleVerdict::Allow; } Ssl::ContextSharedPtr PortPolicy::getServerTlsContext(uint16_t proxy_id, uint32_t remote_id, @@ -1332,7 +1759,8 @@ Ssl::ContextSharedPtr PortPolicy::getServerTlsContext(uint16_t proxy_id, uint32_ config = nullptr; raw_socket_allowed = false; if (port_rules_) { - auto verdict = port_rules_->getServerTlsContext(proxy_id, remote_id, sni, tls_ctx, config); + auto verdict = port_rules_->getServerTlsContext(proxy_id, remote_id, sni, tls_ctx, config, + selector_version_); raw_socket_allowed = verdict == RuleVerdict::Allow && tls_ctx == nullptr && config == nullptr; } return tls_ctx; @@ -1347,7 +1775,8 @@ Ssl::ContextSharedPtr PortPolicy::getClientTlsContext(uint16_t proxy_id, uint32_ config = nullptr; raw_socket_allowed = false; if (port_rules_) { - auto verdict = port_rules_->getClientTlsContext(proxy_id, remote_id, sni, tls_ctx, config); + auto verdict = port_rules_->getClientTlsContext(proxy_id, remote_id, sni, tls_ctx, config, + selector_version_); raw_socket_allowed = verdict == RuleVerdict::Allow && tls_ctx == nullptr && config == nullptr; } return tls_ctx; @@ -1364,7 +1793,8 @@ bool inline rangesOverlap(const PortRange& a, const PortRange& b) { class PortNetworkPolicy : public Logger::Loggable { public: PortNetworkPolicy(const NetworkPolicyMapImpl& parent, - const Protobuf::RepeatedPtrField& rules) { + const Protobuf::RepeatedPtrField& rules, + const ResourceMapOverlay* resource_map) { for (const auto& rule : rules) { // Only TCP supported for HTTP if (rule.protocol() == envoy::config::core::v3::SocketAddress::TCP) { @@ -1523,10 +1953,10 @@ class PortNetworkPolicy : public Logger::Loggable { // so the relative order of rules from this batch is reversed. This // is harmless: equal-precedence rules are evaluated as alternatives // (stable sort only affects presentation/debug ordering). - rules.prepend(parent, rule.rules()); + rules.prepend(parent, rule.rules(), resource_map); } else { // Rules with a non-trivial range go to the back of the list - rules.append(parent, rule.rules()); + rules.append(parent, rule.rules(), resource_map); } } } else { @@ -1578,7 +2008,9 @@ class PortNetworkPolicy : public Logger::Loggable { } } - const PortPolicy findPortPolicy(uint16_t port) const { return PortPolicy(rules_, port); } + const PortPolicy findPortPolicy(uint16_t port, const SelectorVersion selector_version) const { + return PortPolicy(rules_, port, selector_version); + } void toString(int indent, std::string& res) const { if (rules_.empty()) { @@ -1601,11 +2033,15 @@ class PortNetworkPolicy : public Logger::Loggable { // methods. class PolicyInstanceImpl : public PolicyInstance { public: + friend class NetworkPolicyMapImpl; PolicyInstanceImpl(const NetworkPolicyMapImpl& parent, uint64_t hash, - const cilium::NetworkPolicy& proto) + const cilium::NetworkPolicy& proto, + const PolicyStreamStateConstSharedPtr& policy_stream_state, + const ResourceMapOverlay* resource_map) : endpoint_id_(proto.endpoint_id()), hash_(hash), policy_proto_(proto), endpoint_ips_(proto), - parent_(parent), ingress_(parent, policy_proto_.ingress_per_port_policies()), - egress_(parent, policy_proto_.egress_per_port_policies()) {} + parent_(parent), policy_stream_state_(policy_stream_state), + ingress_(parent, policy_proto_.ingress_per_port_policies(), resource_map), + egress_(parent, policy_proto_.egress_per_port_policies(), resource_map) {} bool allowed(bool ingress, uint16_t proxy_id, uint32_t remote_id, uint16_t port, Envoy::Http::RequestHeaderMap& headers, @@ -1624,7 +2060,9 @@ class PolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool ingress, uint16_t port) const override { - return ingress ? ingress_.findPortPolicy(port) : egress_.findPortPolicy(port); + const auto selector_version = policy_stream_state_->version(); + return ingress ? ingress_.findPortPolicy(port, selector_version) + : egress_.findPortPolicy(port, selector_version); } bool useProxylib(bool ingress, uint16_t proxy_id, uint32_t remote_id, uint16_t port, @@ -1656,20 +2094,209 @@ class PolicyInstanceImpl : public PolicyInstance { private: const NetworkPolicyMapImpl& parent_; + const PolicyStreamStateConstSharedPtr policy_stream_state_; const PortNetworkPolicy ingress_; const PortNetworkPolicy egress_; }; +template std::string endpointIpsForLog(const EndpointIps& endpoint_ips) { + std::string formatted = "["; + bool first = true; + for (const auto& endpoint_ip : endpoint_ips) { + if (!first) { + formatted += ", "; + } + formatted += endpoint_ip; + first = false; + } + formatted += "]"; + return formatted; +} + +std::string describePolicyResourceForLog(const std::string& resource_name, + const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource description requires a policy"); + return fmt::format("policy resource '{}' (endpoint_id {}, endpoint_ips {})", resource_name, + policy->endpoint_id_, endpointIpsForLog(policy->policy_proto_.endpoint_ips())); +} + +std::string describePolicyResourceForLog(const std::string& resource_name, + const cilium::NetworkPolicy& policy) { + return fmt::format("policy resource '{}' (endpoint_id {}, endpoint_ips {})", resource_name, + policy.endpoint_id(), endpointIpsForLog(policy.endpoint_ips())); +} + +// pass nullptr pending_resource_map for non-selector based policy! +std::pair, bool> +NetworkPolicyMapImpl::createOrReusePolicy( + const std::string& resource_name, const cilium::NetworkPolicy& config, + const PolicyStreamStateConstSharedPtr& policy_stream_state, const ResourceMap& resource_map, + bool is_new_stream, const ResourceMapOverlay* pending_resource_map) const { + const uint64_t new_hash = MessageUtil::hash(config); + if (!is_new_stream) { + const auto* old_entry = resource_map.findEntry(resource_name); + if (old_entry) { + const auto* old_policy_entry = old_entry->policyResourceEntry(); + if (old_policy_entry) { + const auto& old_policy = old_policy_entry->policy; + if (old_policy && old_policy->hash_ == new_hash && + Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config)) { + return {old_policy, true}; + } + } + } + } + return {std::make_shared(*this, new_hash, config, policy_stream_state, + pending_resource_map), + false}; +} + +std::pair +NetworkPolicyMapImpl::createOrReuseSelector(const std::string& resource_name, + const cilium::Selector& config, uint64_t update_version, + const ResourceMap& resource_map) { + // Compare against the selector visible in the currently prepared update version, not just the + // last published one. Under the single-update-in-flight VersionedMap contract, any selector + // visible in 'update_version' is also the indefinite selector value for that candidate update. + const auto* old_entry = resource_map.findEntry(resource_name); + if (old_entry) { + const auto* old_selector_entry = old_entry->selectorResourceEntry(); + if (old_selector_entry) { + const auto& old_selector_handle = old_selector_entry->handle; + if (old_selector_handle) { + const auto* old_selector = old_selector_handle->get(update_version); + if (old_selector && + old_selector->size() == static_cast(config.remote_identities_size()) && + std::ranges::all_of(config.remote_identities(), [&](const auto remote_identity) { + return old_selector->contains(remote_identity); + })) { + return {old_selector_handle, true}; + } + } + } + } + // otherwise create a new one + auto selector = new SelectorInstance(); + selector->reserve(config.remote_identities_size()); + for (const auto remote_identity : config.remote_identities()) { + selector->emplace(remote_identity); + } + return {selector_map_.insert(std::string(resource_name), selector), false}; +} + +template +void insertPolicyResource(ResourceMapOverlay& pending_resource_map, + const cilium::NetworkPolicy& config, const std::string& version_info, + bool is_new_stream, const std::string& resource_name, + const std::shared_ptr& policy, Logger logger) { + if (!pending_resource_map.emplace(resource_name, ResourceKey::policyResource(policy))) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " + "stream: incoming {} collides with existing {}", + version_info, resource_name, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(resource_name))); + } + for (const auto& endpoint_ip : config.endpoint_ips()) { + logger(endpoint_ip); + if (!pending_resource_map.emplace(endpoint_ip, ResourceKey::policyEndpointIp(resource_name))) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}' on {} " + "stream: incoming {} collides with existing {}", + version_info, endpoint_ip, is_new_stream ? "a new" : "an old", + describePolicyResourceForLog(resource_name, config), + pending_resource_map.describeExistingResourceKey(endpoint_ip))); + } + } +} + +std::pair> +ResourceMapOverlay::findPolicyByEndpointIp(const std::string& endpoint_ip) const { + const auto* key = findEntry(endpoint_ip); + if (key) { + const auto* ip_entry = key->policyEndpointIpEntry(); + if (ip_entry) { + key = findEntry(ip_entry->policy_name); + if (key) { + const auto* policy_entry = key->policyResourceEntry(); + if (policy_entry) { + return {ip_entry->policy_name, policy_entry->policy}; + } + } + } + } + static std::string empty_string; + return {empty_string, nullptr}; +} + +std::string ResourceMapOverlay::describeExistingResourceKey(const std::string& key) const { + const auto* entry = findEntry(key); + if (entry == nullptr) { + return fmt::format("resource key '{}'", key); + } + if (entry->selectorResourceEntry() != nullptr) { + return fmt::format("selector resource '{}'", key); + } + if (const auto* policy_entry = entry->policyResourceEntry(); + policy_entry != nullptr && policy_entry->policy != nullptr) { + return describePolicyResourceForLog(key, policy_entry->policy); + } + if (!entry->isPolicyEndpointIpEntry()) { + return fmt::format("resource key '{}'", key); + } + + auto [resource_name, policy] = findPolicyByEndpointIp(key); + if (policy == nullptr) { + return fmt::format("endpoint IP alias '{}'", key); + } + + if (!resource_name.empty()) { + return fmt::format("endpoint IP alias '{}' owned by {}", key, + describePolicyResourceForLog(resource_name, policy)); + } + + return fmt::format("endpoint IP alias '{}' owned by endpoint_id {} with endpoint_ips {}", key, + policy->endpoint_id_, endpointIpsForLog(policy->policy_proto_.endpoint_ips())); +} + +PolicyMapSnapshot ResourceMapOverlay::toPolicyMapSnapshot() const { + PolicyMapSnapshot policy_map; + forEachEntry([&](const std::string& resource_name, const ResourceKey& resource_key) { + const auto* policy_entry = resource_key.policyResourceEntry(); + if (policy_entry == nullptr || policy_entry->policy == nullptr) { + return; + } + for (const auto& endpoint_ip : policy_entry->policy->policy_proto_.endpoint_ips()) { + const bool inserted = policy_map.emplace(endpoint_ip, policy_entry->policy).second; + RELEASE_ASSERT( + inserted, + fmt::format("duplicate endpoint IP alias '{}' while creating policy map snapshot from {}", + endpoint_ip, + describePolicyResourceForLog(resource_name, policy_entry->policy))); + } + }); + return policy_map; +} + +void ResourceMapOverlay::erasePolicyResource( + const std::string& resource_name, const std::shared_ptr& policy) { + ASSERT(policy != nullptr, "policy resource key must carry a policy"); + for (const auto& endpoint_ip : policy->policy_proto_.endpoint_ips()) { + erase(endpoint_ip); + } + erase(resource_name); +} + // Common base constructor // This is used directly for testing with a file-based subscription -NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& context, +NetworkPolicyMap::NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source, bool subscribe) : context_(context.serverFactoryContext()) { - impl_ = std::make_unique(context, config_source); + impl_ = std::make_shared(context, use_nprds, config_source); if (subscribe) { - impl_->startSubscription(config_source); + impl_->subscribe(); } } @@ -1691,17 +2318,46 @@ NetworkPolicyMap::~NetworkPolicyMap() { ENVOY_LOG(debug, "Cilium L7 NetworkPolicyMap: posting NetworkPolicyMapImpl deletion to main thread"); - context_.mainThreadDispatcher().deleteInDispatcherThread( - Event::DispatcherThreadDeletableConstPtr(impl_.release())); + context_.mainThreadDispatcher().post([impl = std::move(impl_)]() mutable { impl.reset(); }); } bool NetworkPolicyMap::exists(const std::string& endpoint_policy_name) const { - return impl_->getPolicyInstanceImpl(endpoint_policy_name) != nullptr; + return impl_->getPolicyInstanceImpl(endpoint_policy_name); +} + +void NetworkPolicyMap::setConfig(bool use_nprds, + const envoy::config::core::v3::ConfigSource& config_source) { + impl_->setConfig(use_nprds, config_source); } void NetworkPolicyMap::startSubscriptionForTest( std::unique_ptr&& subscription) { - impl_->startSubscription(std::move(subscription)); + impl_->subscribe(std::move(subscription)); +} + +void NetworkPolicyMap::startManagedSubscriptionForTest() { + impl_->startManagedSubscriptionForTest(); +} + +void NetworkPolicyMap::setSubscriptionFactoryForTest(SubscriptionFactoryForTest factory) { + impl_->setSubscriptionFactoryForTest(std::move(factory)); +} + +void NetworkPolicyMap::onSubscriptionConnectedForTest() { impl_->onSubscriptionConnectedForTest(); } + +void NetworkPolicyMap::onSubscriptionTransportCloseForTest() { + impl_->onSubscriptionTransportCloseForTest(); +} + +bool NetworkPolicyMap::subscriptionUseDeltaXdsForTest() const { + return configSourceIsDelta(impl_->getConfigSource()); +} +bool NetworkPolicyMap::desiredUseDeltaXdsForTest() const { + return configSourceIsDelta(impl_->desired_config_.config_source_); +} + +bool NetworkPolicyMap::subscriptionConnectedForTest() const { + return impl_->subscriptionConnectedForTest(); } Envoy::Config::SubscriptionCallbacks& NetworkPolicyMap::subscriptionCallbacksForTest() const { @@ -1710,15 +2366,34 @@ Envoy::Config::SubscriptionCallbacks& NetworkPolicyMap::subscriptionCallbacksFor PolicyStats& NetworkPolicyMap::statsForTest() const { return impl_->stats_; } +void NetworkPolicyMap::resetStreamForTest() { impl_->resetStreamForTest(); } + +PolicyInstanceConstSharedPtr +NetworkPolicyMap::getPolicyInstanceSharedForTest(const std::string& endpoint_policy_name) const { + return impl_->getPolicyInstanceSharedImpl(endpoint_policy_name); +} + +uint64_t +NetworkPolicyMap::policySelectorStreamGenerationForTest(const PolicyInstance& policy) const { + return impl_->policySelectorStreamGenerationForTestImpl(policy); +} + +SelectorVersion NetworkPolicyMap::policySelectorVersionForTest(const PolicyInstance& policy) const { + return impl_->policySelectorVersionForTestImpl(policy); +} + NetworkPolicyMapImpl::NetworkPolicyMapImpl( - Server::Configuration::FactoryContext& context, + Server::Configuration::FactoryContext& context, bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source) : context_(context.serverFactoryContext()), map_ptr_(nullptr), npds_stats_scope_(context_.serverScope().createScope("cilium.npds.")), policy_stats_scope_(context_.serverScope().createScope("cilium.policy.")), - init_target_(fmt::format("Cilium Network Policy subscription start"), + desired_config_(XdsConfig{use_nprds, config_source}), current_config_(desired_config_), + init_target_(fmt::format("Cilium NetworkPolicy subscription start"), [this]() { - subscription_->start({}); + // Production subscription is allowed to start from now on. + subscription_should_start_ = true; + startSubscription(); // Allow listener init to continue before network policy updates are received init_target_.ready(); }), @@ -1727,8 +2402,8 @@ NetworkPolicyMapImpl::NetworkPolicyMapImpl( context_, *npds_stats_scope_, context_.messageValidationContext().dynamicValidationVisitor())), parked_init_manager_(std::make_unique("Cilium NetworkPolicyMap parked")), - config_source_(config_source), - stats_{ALL_CILIUM_POLICY_STATS(POOL_COUNTER(*policy_stats_scope_))} { + stats_{ALL_CILIUM_POLICY_COUNTERS(POOL_COUNTER(*policy_stats_scope_)) + ALL_CILIUM_POLICY_GAUGES(POOL_GAUGE(*policy_stats_scope_))} { // Use listener init manager for subscription initialization context.initManager().add(init_target_); transport_factory_context_->setInitManager(*parked_init_manager_); @@ -1754,49 +2429,81 @@ NetworkPolicyMapImpl::~NetworkPolicyMapImpl() { delete load(); } +void NetworkPolicyMapImpl::subscribe() { + subscription_connected_ = false; + current_config_ = desired_config_; + ++subscription_id_; + + if (subscription_factory_for_test_) { + subscription_ = + subscription_factory_for_test_(configSourceIsDelta(current_config_.config_source_)); + if (subscription_should_start_) { + startSubscription(); + } + return; + } + + auto on_stream_event = [weak_this = weak_from_this(), + id = subscription_id_](Config::GrpcMuxStreamEvent event) { + if (auto shared_this = weak_this.lock()) { + shared_this->onSubscriptionStreamEvent(id, event); + } + }; + + absl::string_view type_url; + Config::OpaqueResourceDecoderSharedPtr decoder; + if (current_config_.use_nprds_) { + type_url = NetworkPolicyResourceTypeUrl; + decoder = std::make_shared(); + } else { + type_url = NetworkPolicyTypeUrl; + decoder = std::make_shared(); + } + + subscription_ = Cilium::subscribe(type_url, current_config_.config_source_, context_, + *npds_stats_scope_, *this, decoder, std::move(on_stream_event)); + + if (subscription_should_start_) { + startSubscription(); + } +} + void NetworkPolicyMapImpl::reopenIpcache() { // Get ipcache singleton only if it was successfully created previously. // Cilium agent re-creates IP cache on restart, and the first accepted update on // the new stream must reopen it before workers enforce refreshed identities. IpCacheSharedPtr ipcache = IpCache::getIpCache(context_); - if (ipcache != nullptr) { + if (ipcache) { ENVOY_LOG(info, "Reopening ipcache on new stream"); ipcache->open(); } } -std::shared_ptr -NetworkPolicyMapImpl::createOrReusePolicy(const cilium::NetworkPolicy& config, - const PolicyMapSnapshot& old_policy_map) { - const uint64_t new_hash = MessageUtil::hash(config); - auto policy_it = old_policy_map.find(config.endpoint_ips()[0]); - if (policy_it != old_policy_map.cend()) { - const auto& old_policy = policy_it->second; - if (old_policy && old_policy->hash_ == new_hash && - Protobuf::util::MessageDifferencer::Equals(old_policy->policy_proto_, config)) { - ENVOY_LOG(trace, "New policy is equal to old one, not updating."); - return old_policy; - } - } - - // May throw - return std::make_shared(*this, new_hash, config); -} - -void NetworkPolicyMapImpl::installNewPolicyMap(PolicyMapSnapshot&& new_policy_map, - Init::ManagerImpl& version_init_manager, - std::string&& version_name) { +void NetworkPolicyMapImpl::installNewPolicyMap( + ResourceMapOverlay&& pending_resource_map, Init::ManagerImpl& version_init_manager, + std::string&& version_name, const PolicyStreamStateSharedPtr& policy_stream_state) { // Initialize SDS secrets. We do not wait for the completion. version_init_manager.initialize(Init::WatcherImpl(std::move(version_name), []() {})); - const auto* old_policy_map = exchange(new PolicyMapSnapshot(std::move(new_policy_map))); - - // Delete the old map once all worker threads have entered their event queues, as this - // is proof that they no longer refer to the old map. - runAfterAllThreads([old_policy_map]() { - // Clean-up in the main thread after all threads have scheduled - delete old_policy_map; - }); + // Publish selector data before publishing the new policy map. New policies created above already + // point at 'policy_stream_state', so any worker that can observe the swapped-in policy map must + // also be able to observe the selector version those policies expect to use. + auto new_version = selector_map_.publishNextVersion(); + if (new_version > 0) { + policy_stream_state->publishVersion(new_version); + } + policy_stream_state_ = policy_stream_state; + stats_.policy_stream_generation_.set(policy_stream_state_->streamGeneration()); + + // Build the immutable lookup snapshot before applyTo() moves staged entries into resource_map_. + auto new_policy_map = pending_resource_map.toPolicyMapSnapshot(); + std::move(pending_resource_map).applyTo(resource_map_); + + // old version can be GC'd once all worker threads have quiesced. + // Delete the old map once all worker threads have entered their event queues, as this is proof + // that they no longer refer to the old map. + scheduleSelectorGCAndDeferredDeletion(new_version, + exchange(new PolicyMapSnapshot(std::move(new_policy_map)))); } // removeInitManager must be called at the end of each policy update @@ -1839,6 +2546,35 @@ void NetworkPolicyMapImpl::removeInitManager() { transport_factory_context_->setInitManager(*parked_init_manager_); } +void NetworkPolicyMapImpl::scheduleSelectorDeferredDeletion( + DeferredDeletion&& deferred) { + if (deferred.empty()) { + return; + } + auto deferred_owner = std::make_shared>(std::move(deferred)); + // The callback exists only to keep the deferred-deletion batch alive until all workers have + // quiesced once more. The batch deletes its nodes from the closure destructor. + runAfterAllThreads([deferred_owner]() {}); +} + +void NetworkPolicyMapImpl::scheduleSelectorGCAndDeferredDeletion( + uint64_t published_version, const PolicyMapSnapshot* old_policy_map) { + if (published_version == 0 && old_policy_map == nullptr) { + return; + } + runAfterAllThreads([shared_this = shared_from_this(), published_version, old_policy_map]() { + // Clean-up in the main thread after all worker threads have scheduled. + // Delete the old policy map before selector GC. Old policies are the only remaining users of + // old-stream selector versions; once the old map is gone after this quiescence point, those + // selector versions may be unlinked and deferred for deletion. + delete old_policy_map; + if (published_version == 0) { + return; + } + shared_this->scheduleSelectorDeferredDeletion(shared_this->selector_map_.gc(published_version)); + }); +} + // onConfigUpdate parses the new network policy resources, allocates a new policy map and atomically // swaps it in place of the old policy map. Throws if any of the 'resources' can not be // parsed. Otherwise an OK status is returned without pausing NPDS gRPC stream, causing a new @@ -1846,58 +2582,500 @@ void NetworkPolicyMapImpl::removeInitManager() { absl::Status NetworkPolicyMapImpl::onConfigUpdate( const std::vector& resources, const std::string& version_info) { - ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}", - instance_id_, resources.size(), version_info); stats_.updates_total_.inc(); + subscription_connected_ = true; - // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, - // and that is also when the old stream terminates and a new one is created. - // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, - // so open it before the workers get a chance to enforce policy on the new IDs. - if (isNewStream()) { - ENVOY_LOG(info, "New NetworkPolicy stream"); + auto stream_generation = streamGeneration(); + // policy_stream_state_ gets updated on first successful update, + // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. + const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); - reopenIpcache(); - } + // Start from an empty resource map for a new stream + const auto& resource_map = is_new_stream ? empty_resource_map_ : resource_map_; + + ENVOY_LOG(debug, "NetworkPolicyMapImpl::onConfigUpdate({}), {} resources, version: {}, stream {}", + instance_id_, resources.size(), version_info, stream_generation); std::string version_name = fmt::format("NetworkPolicyMap version {}", version_info); Init::ManagerImpl version_init_manager(version_name); - // Set the init manager to use via the transport factory context - // Must be set before the new network policy is parsed, as the parsed - // SDS secrets will use this! - transport_factory_context_->setInitManager(version_init_manager); + transport_factory_context_->setInitManager(version_init_manager); // before parsing policies + + ResourceMapOverlay pending_resource_map; + const auto policy_stream_state = + is_new_stream + ? std::make_shared(stream_generation, selector_map_.getVersion()) + : policy_stream_state_; + + try { + // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, + // and that is also when the old stream terminates and a new one is created. + // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, + // so open it before the workers get a chance to enforce policy on the new IDs. + if (is_new_stream) { + ENVOY_LOG(info, "New NetworkPolicy{} stream {}", current_config_.use_nprds_ ? "Resource" : "", + stream_generation); + reopenIpcache(); + } + + // Handle NPRDS and NPDS separately + if (current_config_.use_nprds_) { + const auto selector_update_version = selector_map_.prepareNextVersion(); + absl::flat_hash_set incoming_selector_names; + + // First validate and collect selector resource names + for (const auto& resource : resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicyResource resource name"); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kPolicy: + validatePolicy(resource_name, typed_resource.policy(), version_info); + break; + case cilium::NetworkPolicyResource::kSelector: + ENVOY_LOG(debug, + "Received NetworkPolicyResource selector {} in onConfigUpdate() version {}", + resource_name, version_info); + incoming_selector_names.emplace(resource_name); + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("NetworkPolicyResource has no payload"); + } + } + + // SotW resources are a complete snapshot. Selectors omitted from this snapshot are removed. + for (const auto& [resource_name, resource_key] : resource_map_) { + if (resource_key.selectorResourceEntry() && + !incoming_selector_names.contains(resource_name)) { + selector_map_.clear(resource_name); + } + } + + // Add selectors before policies so policies can resolve selectors from the same snapshot. + for (const auto& resource : resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kSelector) { + continue; + } + const std::string& resource_name = resource.get().name(); + auto [selector_handle, reused] = createOrReuseSelector( + resource_name, typed_resource.selector(), selector_update_version, resource_map); + if (reused) { + ENVOY_LOG(trace, "New selector is equal to old one, not updating."); + } + if (!pending_resource_map.insertOrUpdateSelectorResource(resource_name, selector_handle)) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource update for version {} has duplicate resource key '{}': " + "incoming selector resource '{}' collides with existing {}", + version_info, resource_name, resource_name, + pending_resource_map.describeExistingResourceKey(resource_name))); + } + } + + // Finally add policies + for (const auto& resource : resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kPolicy) { + continue; + } + const std::string& resource_name = resource.get().name(); + const auto& config = typed_resource.policy(); + auto [policy, reused] = + createOrReusePolicy(resource_name, config, policy_stream_state, resource_map, + is_new_stream, &pending_resource_map); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + } + insertPolicyResource( + pending_resource_map, config, version_info, is_new_stream, resource_name, policy, + [](absl::string_view ip) { + ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", ip); + }); + } + } else { + // NPDS just adds policy resources + for (const auto& resource : resources) { + const auto& config = dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicy resource name"); + validatePolicy(resource_name, config, version_info); + + auto [policy, reused] = createOrReusePolicy(resource_name, config, policy_stream_state, + resource_map, is_new_stream, nullptr); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + } + insertPolicyResource( + pending_resource_map, config, version_info, is_new_stream, resource_name, policy, + [](absl::string_view ip) { + ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", ip); + }); + } + } + } catch (const EnvoyException& e) { + ENVOY_LOG(warn, "NetworkPolicy{} update for version {} failed: {}", + current_config_.use_nprds_ ? "Resource" : "", version_info, e.what()); + stats_.updates_rejected_.inc(); + removeInitManager(); + if (selector_map_.resetNextVersionIfNotChanged()) { + scheduleSelectorDeferredDeletion(selector_map_.revert()); + } + throw; // re-throw + } catch (const std::bad_cast&) { + ENVOY_LOG(warn, "NetworkPolicy{} update for version {} failed: invalid resource type", + current_config_.use_nprds_ ? "Resource" : "", version_info); + stats_.updates_rejected_.inc(); + removeInitManager(); + if (selector_map_.resetNextVersionIfNotChanged()) { + scheduleSelectorDeferredDeletion(selector_map_.revert()); + } + throw EnvoyException("Invalid resource type for Cilium NetworkPolicy"); + } + + stats_.update_success_.inc(); + removeInitManager(); + + // SotW update always rewrites the whole resource map + resource_map_.clear(); + + installNewPolicyMap(std::move(pending_resource_map), version_init_manager, + std::move(version_name), policy_stream_state); + return absl::OkStatus(); +} + +void NetworkPolicyMapImpl::validatePolicy(const std::string& resource_name, + const cilium::NetworkPolicy& config, + const std::string& version_info) { + if (config.endpoint_ips().empty()) { + throw EnvoyException("NetworkPolicyResource has no endpoint ips"); + } + ENVOY_LOG(debug, + "Received NetworkPolicy {} for endpoint {}, endpoint_ip {} in " + "onConfigUpdate() version {}", + resource_name, config.endpoint_id(), config.endpoint_ips()[0], version_info); + if (config.endpoint_id() == 0) { + throw EnvoyException("NetworkPolicyResource endpoint_id must be non-zero"); + } +} + +absl::Status NetworkPolicyMapImpl::onConfigUpdate( + const std::vector& added_resources, + const Protobuf::RepeatedPtrField& removed_resources, + const std::string& system_version_info) { + stats_.updates_total_.inc(); + subscription_connected_ = true; + + auto stream_generation = streamGeneration(); + // policy_stream_state_ gets updated on first successful update, + // so 'is_new_stream' remains 'true' as long as the stream has not had a successful update yet. + const bool is_new_stream = stream_generation != policy_stream_state_->streamGeneration(); + + // Start from an empty resource map for a new stream + const auto& resource_map = is_new_stream ? empty_resource_map_ : resource_map_; + + ENVOY_LOG(debug, + "NetworkPolicyMapImpl::onConfigUpdate({}), {} added resources, {} removed resources, " + "version: {}, stream {}", + instance_id_, added_resources.size(), removed_resources.size(), system_version_info, + stream_generation); + + std::string version_name = fmt::format("NetworkPolicyMap version {}", system_version_info); + Init::ManagerImpl version_init_manager(version_name); + transport_factory_context_->setInitManager(version_init_manager); // before parsing policies + + ResourceMapOverlay pending_resource_map = + is_new_stream ? ResourceMapOverlay() : ResourceMapOverlay(resource_map); + const auto policy_stream_state = + is_new_stream + ? std::make_shared(stream_generation, selector_map_.getVersion()) + : policy_stream_state_; - const auto* old_policy_map = load(); - PolicyMapSnapshot new_policy_map; try { - for (const auto& resource : resources) { - const auto& config = dynamic_cast(resource.get().resource()); - if (config.endpoint_ips().empty()) { - throw EnvoyException("Network Policy has no endpoint ips"); + // Reopen IPcache for every new stream. Cilium agent re-creates IP cache on restart, + // and that is also when the old stream terminates and a new one is created. + // New security identities (e.g., for FQDN policies) only get inserted to the new IP cache, + // so open it before the workers get a chance to enforce policy on the new IDs. + if (is_new_stream) { + ENVOY_LOG(info, "New NetworkPolicy{} stream {}", current_config_.use_nprds_ ? "Resource" : "", + stream_generation); + reopenIpcache(); + } + + // Handle NPRDS and NPDS separately + if (current_config_.use_nprds_) { + // First validate and find whether this is a selector-only update. + bool adds_policies = false; + bool removes_policies = false; + bool adds_selectors = false; + bool removes_selectors = false; + for (const auto& removed_resource : removed_resources) { + validateResourceName(removed_resource, "NetworkPolicyResource removed resource name"); + auto resource_it = resource_map.find(removed_resource); + if (resource_it == resource_map.end()) { + continue; + } + if (resource_it->second.selectorResourceEntry()) { + removes_selectors = true; + } else if (resource_it->second.policyResourceEntry()) { + removes_policies = true; + } else { + throw EnvoyException(fmt::format( + "NetworkPolicyResource removed resource '{}' is a policy endpoint IP alias, " + "not a resource name", + removed_resource)); + } + } + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicyResource added resource name"); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kPolicy: + validatePolicy(resource_name, typed_resource.policy(), system_version_info); + adds_policies = true; + break; + case cilium::NetworkPolicyResource::kSelector: + ENVOY_LOG(trace, + "Received NetworkPolicyResource selector {} in onConfigUpdate() " + "version {}", + resource_name, system_version_info); + adds_selectors = true; + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("NetworkPolicyResource has no payload"); + } } + ENVOY_LOG(debug, - "Received Network Policy for endpoint {}, endpoint_ip {} in onConfigUpdate() " - "version {}", - config.endpoint_id(), config.endpoint_ips()[0], version_info); - - auto policy = createOrReusePolicy(config, *old_policy_map); - for (const auto& endpoint_ip : config.endpoint_ips()) { - ENVOY_LOG(trace, "Cilium updating or keeping network policy for endpoint {}", endpoint_ip); - // new_policy_map is not exception safe, policy must be computed separately! - new_policy_map.emplace(endpoint_ip, policy); + "NetworkPolicyMapImpl::onConfigUpdate({}), " + "version: {}, stream {}, updates_selectors: {}, updates_policies: {}", + instance_id_, system_version_info, stream_generation, + removes_selectors || adds_selectors, removes_policies || adds_policies); + + const auto selector_update_version = selector_map_.prepareNextVersion(); + + // Is this a selector only update? + if (!is_new_stream && (removes_selectors || adds_selectors) && + !(removes_policies || adds_policies)) { + for (const auto& resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource selector {}", resource); + const auto* resource_entry = pending_resource_map.findEntry(resource); + if (resource_entry == nullptr) { + ENVOY_LOG( + debug, + "NetworkPolicyResource removed selector name '{}' not found from resource map", + resource); + continue; + } + if (resource_entry->isPolicyEndpointIpEntry()) { + throw EnvoyException(fmt::format("NetworkPolicyResource removed selector name " + "'{}' is a policy endpoint IP alias, " + "not a resource name", + resource)); + } + if (resource_entry->policyResourceEntry()) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource removed selector name '{}' refers to a policy resource", + resource)); + } + selector_map_.clear(resource); + pending_resource_map.erase(resource); + } + + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + const std::string& resource_name = resource.get().name(); + + switch (typed_resource.resource_case()) { + case cilium::NetworkPolicyResource::kSelector: { + ENVOY_LOG(debug, + "Received NetworkPolicyResource selector {} in onConfigUpdate() " + "version {}", + resource_name, system_version_info); + auto [selector_handle, reused] = createOrReuseSelector( + resource_name, typed_resource.selector(), selector_update_version, resource_map); + if (reused) { + ENVOY_LOG(trace, "New selector is equal to old one, not updating."); + } + if (!pending_resource_map.insertOrUpdateSelectorResource(resource_name, + selector_handle)) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource selector update for version {} " + "has duplicate resource key '{}' on {} " + "stream: incoming selector resource '{}' collides with existing {}", + system_version_info, resource_name, is_new_stream ? "a new" : "an old", + resource_name, pending_resource_map.describeExistingResourceKey(resource_name))); + } + break; + } + case cilium::NetworkPolicyResource::kPolicy: + IS_ENVOY_BUG( + "Selector-only NetworkPolicyResource update unexpectedly included a policy"); + break; + case cilium::NetworkPolicyResource::RESOURCE_NOT_SET: + throw EnvoyException("NetworkPolicyResource has no payload"); + } + } + } else { // Policy and selector update path + // First remove removed resources. + for (const auto& removed_resource : removed_resources) { + ENVOY_LOG(trace, "Cilium removing NetworkPolicyResource {}", removed_resource); + const auto* resource_entry = pending_resource_map.findEntry(removed_resource); + if (!resource_entry) { + continue; + } + if (resource_entry->selectorResourceEntry()) { + selector_map_.clear(removed_resource); + pending_resource_map.erase(removed_resource); + } else if (const auto* policy_entry = resource_entry->policyResourceEntry()) { + pending_resource_map.erasePolicyResource(removed_resource, policy_entry->policy); + } + } + // Then remove policies that are being replaced so duplicate endpoint IP detection sees the + // pending state. Selector resources are updated in place below to preserve their stable + // handles. + if (adds_policies) { + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() == cilium::NetworkPolicyResource::kPolicy) { + const std::string& resource_name = resource.get().name(); + pending_resource_map.erasePolicyResourceIfPresent(resource_name); + } + } + } + // Then add selectors so policies can bind to selectors introduced by the same update. + if (adds_selectors) { + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kSelector) { + continue; + } + const std::string& resource_name = resource.get().name(); + auto [selector_handle, reused] = createOrReuseSelector( + resource_name, typed_resource.selector(), selector_update_version, resource_map); + if (reused) { + ENVOY_LOG(trace, "New selector is equal to old one, not updating."); + } + if (!pending_resource_map.insertOrUpdateSelectorResource(resource_name, + selector_handle)) { + throw EnvoyException(fmt::format( + "NetworkPolicyResource selector update for version {} has duplicate resource " + "key '{}' on {} " + "stream: incoming selector resource '{}' collides with existing {}", + system_version_info, resource_name, is_new_stream ? "a new" : "an old", + resource_name, pending_resource_map.describeExistingResourceKey(resource_name))); + } + } + } + // Finally add policies. + if (adds_policies) { + for (const auto& resource : added_resources) { + const auto& typed_resource = + dynamic_cast(resource.get().resource()); + if (typed_resource.resource_case() != cilium::NetworkPolicyResource::kPolicy) { + continue; + } + const std::string& resource_name = resource.get().name(); + const auto& config = typed_resource.policy(); + auto [policy, reused] = + createOrReusePolicy(resource_name, config, policy_stream_state, resource_map, + is_new_stream, &pending_resource_map); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + } + insertPolicyResource(pending_resource_map, config, system_version_info, is_new_stream, + resource_name, policy, [](absl::string_view ip) { + ENVOY_LOG(trace, + "Cilium updating network policy for endpoint {}", ip); + }); + } + } + } + } else { + // Delta NPDS carries only NetworkPolicy resources, so skip the selector scan/update paths. + for (const auto& removed_resource : removed_resources) { + validateResourceName(removed_resource, "NetworkPolicy removed resource name"); + ENVOY_LOG(trace, "Cilium removing NetworkPolicy {}", removed_resource); + const auto* resource_entry = pending_resource_map.findEntry(removed_resource); + if (resource_entry == nullptr) { + continue; + } + if (const auto* policy_entry = resource_entry->policyResourceEntry()) { + pending_resource_map.erasePolicyResource(removed_resource, policy_entry->policy); + continue; + } + if (resource_entry->isPolicyEndpointIpEntry()) { + throw EnvoyException( + fmt::format("NetworkPolicy removed resource '{}' is a policy endpoint " + "IP alias, not a resource name", + removed_resource)); + } + throw EnvoyException(fmt::format( + "NetworkPolicy removed resource '{}' is not a policy resource", removed_resource)); + } + + // First validate and remove policies that are being replaced so duplicate endpoint IP + // detection sees the pending state. + for (const auto& resource : added_resources) { + const std::string& resource_name = resource.get().name(); + validateResourceName(resource_name, "NetworkPolicy added resource name"); + const auto& config = dynamic_cast(resource.get().resource()); + validatePolicy(resource_name, config, system_version_info); + pending_resource_map.erasePolicyResourceIfPresent(resource_name); + } + + for (const auto& resource : added_resources) { + const std::string& resource_name = resource.get().name(); + const auto& config = dynamic_cast(resource.get().resource()); + auto [policy, reused] = createOrReusePolicy(resource_name, config, policy_stream_state, + resource_map, is_new_stream, nullptr); + if (reused) { + ENVOY_LOG(trace, "New policy is equal to old one, not updating."); + } + insertPolicyResource(pending_resource_map, config, system_version_info, is_new_stream, + resource_name, policy, [](absl::string_view ip) { + ENVOY_LOG(trace, "Cilium updating network policy for endpoint {}", + ip); + }); } } } catch (const EnvoyException& e) { - ENVOY_LOG(warn, "NetworkPolicy update for version {} failed: {}", version_info, e.what()); + ENVOY_LOG(warn, "NetworkPolicy{} update for version {} failed: {}", + current_config_.use_nprds_ ? "Resource" : "", system_version_info, e.what()); stats_.updates_rejected_.inc(); removeInitManager(); + if (selector_map_.resetNextVersionIfNotChanged()) { + scheduleSelectorDeferredDeletion(selector_map_.revert()); + } throw; // re-throw + } catch (const std::bad_cast&) { + ENVOY_LOG(warn, "NetworkPolicy{} update for version {} failed: invalid resource type", + current_config_.use_nprds_ ? "Resource" : "", system_version_info); + stats_.updates_rejected_.inc(); + removeInitManager(); + if (selector_map_.resetNextVersionIfNotChanged()) { + scheduleSelectorDeferredDeletion(selector_map_.revert()); + } + throw EnvoyException("Invalid resource type for Cilium NetworkPolicy"); } stats_.update_success_.inc(); removeInitManager(); - installNewPolicyMap(std::move(new_policy_map), version_init_manager, std::move(version_name)); + // Do not carry over any resources from an old stream. + if (is_new_stream) { + resource_map_.clear(); + } + installNewPolicyMap(std::move(pending_resource_map), version_init_manager, + std::move(version_name), policy_stream_state); return absl::OkStatus(); } @@ -1905,7 +3083,8 @@ void NetworkPolicyMapImpl::onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailu const EnvoyException*) { // We need to allow server startup to continue, even if we have a bad // config. - ENVOY_LOG(debug, "Network Policy Update failed, keeping existing policy."); + ENVOY_LOG(debug, "NetworkPolicy update on stream {} failed, keeping existing policy.", + streamGeneration()); } ProtobufTypes::MessagePtr @@ -1949,7 +3128,7 @@ class AllowAllEgressPolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool ingress, uint16_t) const override { - return ingress ? PortPolicy(empty_map_, 0) : PortPolicy(empty_map_, 1); + return ingress ? PortPolicy(empty_map_, 0, versionMin) : PortPolicy(empty_map_, 1, versionMin); } bool useProxylib(bool, uint16_t, uint32_t, uint16_t, std::string&) const override { @@ -1992,7 +3171,7 @@ class DenyAllPolicyInstanceImpl : public PolicyInstance { } const PortPolicy findPortPolicy(bool, uint16_t) const override { - return PortPolicy(empty_map_, 0); + return PortPolicy(empty_map_, 0, versionMin); } bool useProxylib(bool, uint16_t, uint32_t, uint16_t, std::string&) const override { @@ -2030,6 +3209,32 @@ NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) cons return nullptr; } +PolicyInstanceConstSharedPtr +NetworkPolicyMapImpl::getPolicyInstanceSharedImpl(const std::string& endpoint_ip) const { + const auto* map = load(); + auto it = map->find(endpoint_ip); + if (it != map->end()) { + return it->second; + } + return nullptr; +} + +uint64_t NetworkPolicyMapImpl::policySelectorStreamGenerationForTestImpl( + const PolicyInstance& policy) const { + if (const auto* policy_impl = dynamic_cast(&policy)) { + return policy_impl->policy_stream_state_->streamGeneration(); + } + return 0; +} + +SelectorVersion +NetworkPolicyMapImpl::policySelectorVersionForTestImpl(const PolicyInstance& policy) const { + if (const auto* policy_impl = dynamic_cast(&policy)) { + return policy_impl->policy_stream_state_->version(); + } + return versionMin; +} + // getPolicyInstance return a const reference to a policy in the policy map for the given // 'endpoint_ip'. If there is no policy for the given IP, a default policy is returned, // controlled by the 'default_allow_egress' argument as follows: @@ -2044,9 +3249,7 @@ NetworkPolicyMapImpl::getPolicyInstanceImpl(const std::string& endpoint_ip) cons const PolicyInstance& NetworkPolicyMap::getPolicyInstance(const std::string& endpoint_ip, bool default_allow_egress) const { const auto* policy = impl_->getPolicyInstanceImpl(endpoint_ip); - return policy != nullptr ? *policy - : default_allow_egress ? getAllowAllEgressPolicy() - : getDenyAllPolicy(); + return policy ? *policy : default_allow_egress ? getAllowAllEgressPolicy() : getDenyAllPolicy(); } } // namespace Cilium diff --git a/cilium/network_policy.h b/cilium/network_policy.h index ab5621c22..e6d041f6d 100644 --- a/cilium/network_policy.h +++ b/cilium/network_policy.h @@ -3,9 +3,11 @@ #include #include +#include #include #include +#include "envoy/common/exception.h" #include "envoy/common/pure.h" #include "envoy/common/regex.h" #include "envoy/config/core/v3/base.pb.h" @@ -40,6 +42,7 @@ namespace Cilium { class PortNetworkPolicyRules; class PolicySnapshot; +using SelectorVersion = uint64_t; // PortPolicy holds a reference to a set of rules in a policy map that apply to the given port. // Methods then iterate through the set to determine if policy allows or denies. This is needed to @@ -50,7 +53,7 @@ class PortPolicy : public Logger::Loggable { friend class PortNetworkPolicy; friend class DenyAllPolicyInstanceImpl; friend class AllowAllEgressPolicyInstanceImpl; - PortPolicy(const PolicySnapshot& map, uint16_t port); + PortPolicy(const PolicySnapshot& map, uint16_t port, SelectorVersion selector_version); public: // If hasHttpRules() returns false, then HTTP policy enforcement can be skipped, @@ -96,6 +99,7 @@ class PortPolicy : public Logger::Loggable { // rules. const PortNetworkPolicyRules* port_rules_; const bool has_http_rules_; + const SelectorVersion selector_version_; }; class IpAddressPair { @@ -167,51 +171,102 @@ class NetworkPolicyDecoder : public Envoy::Config::OpaqueResourceDecoder { ProtobufMessage::ValidationVisitor& validation_visitor_; }; +// cilium::NetworkPolicyResource does not carry a resource name, but relies on the +// DeltaDiscoveryRespons Resource wrapper to have the name. Hence can not use +// Envoy::Config::OpaqueResourceDecoderImpl +class NetworkPolicyResourceDecoder : public Envoy::Config::OpaqueResourceDecoder { +public: + NetworkPolicyResourceDecoder() + : validation_visitor_(ProtobufMessage::getNullValidationVisitor()) {} + + // Config::OpaqueResourceDecoder + ProtobufTypes::MessagePtr decodeResource(const Protobuf::Any& resource) override { + auto typed_message = std::make_unique(); + // If the Any is a synthetic empty message (e.g. because the resource field + // was not set in Resource, this might be empty, so we shouldn't decode. + if (!resource.type_url().empty()) { + MessageUtil::anyConvertAndValidate(resource, *typed_message, + validation_visitor_); + } + return typed_message; + } + + std::string resourceName(const Protobuf::Message&) override { + throw EnvoyException( + "NetworkPolicyResource does not carry a name and must be wrapped in Resource"); + } + +private: + ProtobufMessage::ValidationVisitor& validation_visitor_; +}; + /** * All Cilium L7 filter stats. @see stats_macros.h */ // clang-format off -#define ALL_CILIUM_POLICY_STATS(COUNTER) \ - COUNTER(updates_total) \ - COUNTER(updates_rejected) \ - COUNTER(tls_wrapper_missing_policy) \ +#define ALL_CILIUM_POLICY_COUNTERS(COUNTER) \ + COUNTER(updates_total) \ + COUNTER(updates_rejected) \ + COUNTER(tls_wrapper_missing_policy) \ COUNTER(update_success) + +#define ALL_CILIUM_POLICY_GAUGES(GAUGE) \ + GAUGE(policy_stream_generation, NeverImport) // clang-format on /** * Struct definition for all policy stats. @see stats_macros.h */ struct PolicyStats { - ALL_CILIUM_POLICY_STATS(GENERATE_COUNTER_STRUCT) + ALL_CILIUM_POLICY_COUNTERS(GENERATE_COUNTER_STRUCT) + ALL_CILIUM_POLICY_GAUGES(GENERATE_GAUGE_STRUCT) }; class NetworkPolicyMapImpl; class NetworkPolicyMap : public Singleton::Instance, public Logger::Loggable { public: - NetworkPolicyMap(Server::Configuration::FactoryContext& context, + NetworkPolicyMap(Server::Configuration::FactoryContext& context, bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source, bool subscribe = false); + ~NetworkPolicyMap() override; bool exists(const std::string& endpoint_policy_name) const; + void setConfig(bool use_nprds, const envoy::config::core::v3::ConfigSource& config_source); + const PolicyInstance& getPolicyInstance(const std::string& endpoint_policy_name, bool allow_egress) const; static PolicyInstance& getDenyAllPolicy(); static PolicyInstance& getAllowAllEgressPolicy(); + using SubscriptionFactoryForTest = + std::function(bool use_delta_xds)>; + protected: friend class CiliumNetworkPolicyTest; friend struct TestHelper; PolicyStats& statsForTest() const; + void resetStreamForTest(); + PolicyInstanceConstSharedPtr + getPolicyInstanceSharedForTest(const std::string& endpoint_policy_name) const; + uint64_t policySelectorStreamGenerationForTest(const PolicyInstance& policy) const; + SelectorVersion policySelectorVersionForTest(const PolicyInstance& policy) const; void startSubscriptionForTest(std::unique_ptr&& subscription); + void startManagedSubscriptionForTest(); + void setSubscriptionFactoryForTest(SubscriptionFactoryForTest factory); + void onSubscriptionConnectedForTest(); + void onSubscriptionTransportCloseForTest(); + bool desiredUseDeltaXdsForTest() const; + bool subscriptionUseDeltaXdsForTest() const; + bool subscriptionConnectedForTest() const; Envoy::Config::SubscriptionCallbacks& subscriptionCallbacksForTest() const; private: Server::Configuration::ServerFactoryContext& context_; - std::unique_ptr impl_; + std::shared_ptr impl_; }; using NetworkPolicyMapSharedPtr = std::shared_ptr; diff --git a/cilium/versioned.h b/cilium/versioned.h new file mode 100644 index 000000000..c41e94d0f --- /dev/null +++ b/cilium/versioned.h @@ -0,0 +1,531 @@ +#pragma once + +// NOLINT(namespace-envoy) + +#include +#include +#include +#include +#include +#include + +#include "source/common/common/assert.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" + +constexpr uint64_t versionMin = 0; +constexpr uint64_t versionMax = std::numeric_limits::max() - 1; +constexpr uint64_t versionNotRemoved = std::numeric_limits::max(); + +// Versioned.h provides a lock-free reader / single-writer versioned-value container, +// based on earlier implementations in OVS (C) and Cilium (Go). +// +// API contract and intended use: +// +// 1. Object model +// - T is a CRTP node type inheriting from VersionedNode. +// - VersionedValue stores a linked list of historical T nodes, newest first. +// - VersionedReadable exposes only the worker-safe read API: get(version). +// - VersionedHandle is a shared_ptr, so handles intentionally +// expose only the readable API to worker/runtime code. Main-thread mutation goes through +// VersionedMap, which keeps mutable shared_ptr internally. +// - The optional readable base R allows a handle to expose extra read-only metadata (for +// example selector names) without exposing VersionedValue mutators. +// +// 2. Threading model +// - There is exactly one mutator thread: the main thread. +// - Worker threads may concurrently call VersionedReadable::get(version) and may traverse nodes +// reached from a handle without additional locking. +// - Any future cross-thread mutation must still preserve the exclusive-writer rule, e.g. by +// holding an exclusive lock around all mutation. +// - VersionedMap itself is main-thread-only. Its name directory (map_), dirty set, and +// transactional state must not be accessed from worker threads. +// +// 3. VersionedMap transaction flow +// - prepareNextVersion() starts a new unpublished update and returns the candidate version to +// use for any main-thread lookups against in-flight state. +// - insert(key, value) reuses the existing stable handle if the key is still present in the +// main-thread directory; otherwise it creates a fresh handle. +// - clear(key) marks the key's current value invisible in the candidate version but does not +// erase the key from the main-thread directory immediately. +// - publishNextVersion() publishes all pending changes, removes keys whose handle has no visible +// value in the published version, and returns the new published version, or 0 if there was +// nothing to publish. +// - revert() discards the unpublished update, restores visibility of older nodes as needed, and +// removes keys whose handle has no value in the published version. +// - find(key) returns the stable readable handle parked under the key in the main-thread +// directory. It does not itself answer whether the key is visible in any specific version; use +// handle->get(version) for that. +// +// 4. Stable-handle semantics +// - Updating an active key writes a new version onto the same stable handle. +// - clear() followed by re-add of the same key in the same unpublished update also reuses that +// same handle; the caller should compare against the candidate version returned from +// prepareNextVersion(), not against the currently published version. +// - Once a key has been fully removed across a published version boundary, the name disappears +// from VersionedMap. A later same-name add creates a fresh stable handle. +// - This is why old published policies can keep using an old cleared handle while refreshed +// policies bind a new one after a later same-name re-add. +// +// 5. Reader semantics +// - get(version) walks from the current head and returns the first node visible in the requested +// version. +// - The walk stops at the first node that was added before the requested version but is no +// longer visible there; all remaining nodes are older and therefore already invisible in that +// version. +// - Querying an older published version after newer publishes is logically stale and may produce +// outdated results, but it must remain memory-safe. +// +// 6. Deferred deletion / grace periods +// - Unpublished nodes removed by revert() were never visible to workers, but are still unlinked +// into a DeferredDeletion batch so that any concurrent traversal that had already stepped into +// them cannot race with delete. +// - gc(published_version) is only the first GC phase. It must be called only after all workers +// have quiesced for that published version, unlinks nodes that are no longer visible to +// anyone, and returns them in a DeferredDeletion batch. +// - The returned DeferredDeletion batch must stay alive until all workers have quiesced once +// more. Destroying the batch performs the actual node deletion. +// - The production pattern is therefore: publish version N, wait for worker quiescence, call +// gc(N), then keep the returned DeferredDeletion alive for one more quiescence round. +// +// 7. High-level memory-model rationale +// - The main thread publishes new list links with 'release' stores to the list node pointers. +// - Workers traverse with 'acquire' loads from the list node pointers, so they either observe +// the old reachable chain or the newly published/unlinked chain, but not torn pointer state. +// - published_version_ is stored with 'release' in publishNextVersion() and loaded with +// 'acquire' by readers, so readers that observe a new published version also observe the +// prior main-thread mutations that made that version reachable. +// - remove_version_ uses 'relaxed' atomic loads/stores because it controls logical visibility, +// not structural reachability. Structural coherence and memory safety come from the +// 'release'/'acquire' ordering on list node pointers plus the extra deferred-deletion grace +// period. +// - Many mutation-side loads/stores are 'relaxed' because mutation is single-threaded on the +// main thread. +// +// 8. DeferredDeletion implementation detail +// - DeferredDeletion reuses VersionedNode::next_ to chain together unlinked nodes awaiting +// destruction. This keeps deferred deletion self-contained without an extra node container. +// - Public code must not rely on any semantics of detached invisible tails beyond memory safety; +// only get(version) is the supported read API. +// +// 9. Test coverage in tests/versioned_test.cc +// - Value-level tests cover visibility rules, shadowing, clear(), revert(), first-phase GC, and +// deferred deletion ownership/move semantics. +// - Map-level tests cover version numbering, publish-without-change, insert/update/clear/revert, +// same-handle reuse for active updates and same-update clear+read, fresh-handle behavior +// after published removal, and dirty-handle retention across multiple future GC runs. +// - Stable validator tests verify consistent published snapshots and post-unlink/pre-deletion +// states. +// - A multithreaded chaos test exercises one writer with multiple busy-loop readers, repeated +// publish/revert/GC cycles, stale-version reads, and traversal safety under deferred deletion. +// +template class VersionedReadable; +template > class VersionedValue; +template class DeferredDeletion; +template > struct VersionedTestAccess; + +// VersionedNode is CRTP type on T, T must inherit from VersionedNode +template class VersionedNode { +public: + template + VersionedNode() + : next_(nullptr), add_version_(versionNotRemoved), remove_version_(versionNotRemoved) {} + +private: + template friend class VersionedReadable; + template friend class VersionedValue; + template friend class DeferredDeletion; + template friend struct VersionedTestAccess; + + bool isUnpublished() const { return add_version_ == versionNotRemoved; } + + void setAddVersion(uint64_t version) { + ASSERT(version < versionNotRemoved); + add_version_ = version; // set before published + } + + void removeInVersion(uint64_t version) { + remove_version_.store(version, std::memory_order_relaxed); + } + + bool isVisibleInVersion(uint64_t version) const { + return add_version_ <= version && version < remove_version_.load(std::memory_order_relaxed); + } + + bool isAddedAfterVersion(uint64_t version) const { return add_version_ > version; } + + bool isAddedBeforeVersion(uint64_t version) const { return add_version_ < version; } + + bool isEventuallyInvisible() const { + return remove_version_.load(std::memory_order_relaxed) < versionNotRemoved; + } + + bool isVisibleOnlyBeforeVersion(uint64_t version) const { + return remove_version_.load(std::memory_order_relaxed) <= version; + } + + bool isRemovedAfterVersion(uint64_t version) const { + const auto remove_version = remove_version_.load(std::memory_order_relaxed); + return remove_version < versionNotRemoved && version < remove_version; + } + + VersionedNode* getNext() const { return next_.load(std::memory_order_acquire); } + + void setNext(VersionedNode* next) { return next_.store(next, std::memory_order_release); } + + VersionedNode* getNextProtected() const { return next_.load(std::memory_order_relaxed); } + std::atomic*>* getNextP() { return &next_; } + + T* value() { return static_cast(this); } + const T* value() const { return static_cast(this); } + + std::atomic*> next_; + + uint64_t add_version_; // Version object was added in. + std::atomic remove_version_; // Version object is removed in. +}; + +template class VersionedReadable { +public: + const T* get(uint64_t version) const { + for (auto node = head_.load(std::memory_order_acquire); node; node = node->getNext()) { + if (node->isVisibleInVersion(version)) { + return node->value(); + } + // stop traveral on first non-visible version that was added before this version, all + // remaining nodes are already invisible. + if (node->isAddedBeforeVersion(version)) { + break; + } + } + return nullptr; // not found + } + +protected: + VersionedReadable() : head_(nullptr) {} + + template friend class VersionedValue; + template friend class DeferredDeletion; + template friend struct VersionedTestAccess; + + std::atomic*> head_; +}; + +// T is the CRTP node type stored in the version chain. R is the read-only interface exposed +// through VersionedHandle: it defaults to VersionedReadable, but callers may provide a richer +// readable base when handles need extra read-side API without exposing the main-thread mutators. +template class VersionedValue : public R { +public: + using Readable = R; + using R::head_; + + VersionedValue() = default; + + template >> + explicit VersionedValue(Args&&... args) : R(std::forward(args)...) {} + + ~VersionedValue() { + VersionedNode* next; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + delete node->value(); + } + } + + // make all versions invisible starting at "version" + void clear(uint64_t version) { + for (auto node = head_.load(std::memory_order_relaxed); node; node = node->getNextProtected()) { + if (!node->isEventuallyInvisible()) { + node->removeInVersion(version); + } + } + } + + void set(uint64_t version, VersionedNode* node) { + // publish a new version that is visible starting at "version" + ASSERT(node->isUnpublished()); // node not pushed into a list yet + node->setAddVersion(version); + node->setNext(head_.load(std::memory_order_relaxed)); + head_.store(node, std::memory_order_release); + + // make all other versions invisible at "version", starting from the first old node. + for (node = node->getNextProtected(); node; node = node->getNextProtected()) { + if (node->isVisibleInVersion(version)) { + node->removeInVersion(version); + } + } + } + + bool empty() const { return head_.load(std::memory_order_relaxed) == nullptr; } + + // Unlink unpublished versions added after 'version' and append them to 'deferred' for deletion + // after an additional grace period. Removed nodes that were still visible in 'version' are + // restored. + void revert(uint64_t version, DeferredDeletion& deferred) { + auto prev = &head_; + VersionedNode* next; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + if (node->isAddedAfterVersion(version)) { + // unlink node by pointing prev to next + prev->store(next, std::memory_order_release); + deferred.push(node); + // prev stays + continue; + } + if (node->isRemovedAfterVersion(version)) { + // visibility was limited, restore + node->removeInVersion(versionNotRemoved); + } + prev = node->getNextP(); + } + } + + // Unlink all versions not visible to anyone after the "version" was published and all readers + // have quiesced, but do not delete them yet. Unlinked nodes are appended to 'deferred' and are + // only deleted after one more worker-thread quiescence round. Any nodes not yet visible at + // "version" must not be unlinked. + // Returns true if there are no future version removals left in this handle after this first + // phase GC run. + bool gcForVersion(uint64_t version, DeferredDeletion& deferred) { + ASSERT(version < versionNotRemoved); + auto prev = &head_; + VersionedNode* next; + bool has_future_gc_work = false; + for (auto node = head_.load(std::memory_order_relaxed); node; node = next) { + next = node->getNextProtected(); + if (node->isVisibleOnlyBeforeVersion(version)) { + // unlink node by pointing prev to next + prev->store(next, std::memory_order_release); + deferred.push(node); + // prev stays + continue; + } + if (node->isEventuallyInvisible()) { + has_future_gc_work = true; + } + prev = node->getNextP(); + } + return !has_future_gc_work; + } + +protected: + friend class DeferredDeletion; + template friend struct VersionedTestAccess; +}; + +template class DeferredDeletion : protected VersionedValue { +public: + // Bring `head_` from the templated base class into this scope so the rest of the code can use + // normal unqualified member access. Without this, dependent-base lookup would require + // `this->head_` everywhere. + using VersionedValue::head_; + + DeferredDeletion() = default; + DeferredDeletion(const DeferredDeletion&) = delete; + DeferredDeletion& operator=(const DeferredDeletion&) = delete; + DeferredDeletion(DeferredDeletion&& other) noexcept { + head_.store(other.head_.exchange(nullptr, std::memory_order_relaxed), + std::memory_order_relaxed); + tail_ = std::exchange(other.tail_, nullptr); + } + DeferredDeletion& operator=(DeferredDeletion&&) noexcept = delete; + ~DeferredDeletion() = default; + + bool empty() const { return head_.load(std::memory_order_relaxed) == nullptr; } + +private: + template friend class VersionedValue; + + void push(VersionedNode* node) { + node->setNext(nullptr); + if (tail_) { + tail_->setNext(node); + } else { + // Only the main thread is accessing the deferred-deletion head, hence relaxed. + head_.store(node, std::memory_order_relaxed); + } + tail_ = node; + } + + VersionedNode* tail_{nullptr}; +}; + +template > +using VersionedHandle = std::shared_ptr; + +// VersionedMap is main-thread / exclusive-writer only: map_, pending_keys_, dirty_values_, and +// all mutation methods are accessed only by the single publishing thread. Worker threads only use +// the published Handle objects and then call the read-only handle API. +template > class VersionedMap { +public: + static_assert(std::is_base_of_v); + static_assert(std::is_base_of_v, typename V::Readable>); + using Handle = VersionedHandle; + + uint64_t getVersion() const { return published_version_.load(std::memory_order_acquire); } + + Handle find(const K& key) const { + auto it = map_.find(key); + if (it != map_.cend()) { + return it->second; + } + return nullptr; + } + + uint64_t prepareNextVersion() { + ASSERT(pending_keys_.empty()); + return ++next_version_; + } + + Handle insert(const K& key, T* value) { + ASSERT(next_version_ > published_version_.load(std::memory_order_relaxed)); + auto [it, inserted] = map_.try_emplace(key); + if (inserted) { + if constexpr (std::is_constructible_v) { + it->second = std::make_shared(key); + } else { + it->second = std::make_shared(); + } + } + auto& handle = it->second; + handle->set(next_version_, value); + pending_keys_.emplace(key); + return handle; + } + + void clear(const K& key) { + ASSERT(next_version_ > published_version_.load(std::memory_order_relaxed)); + auto it = map_.find(key); + if (it != map_.cend()) { + it->second->clear(next_version_); + pending_keys_.emplace(key); + } + } + + // returns true if changes need to be reverted or published. Otherwise + // resets the next version back to current version. + bool resetNextVersionIfNotChanged() { + auto version = published_version_.load(std::memory_order_relaxed); + if (next_version_ <= version || pending_keys_.empty()) { + // no changes, revert back to the published version + next_version_ = version; + pending_keys_.clear(); + return false; + } + return true; + } + + // returns the newly published version, or 0 if there was nothing to publish + uint64_t publishNextVersion() { + auto version = published_version_.load(std::memory_order_relaxed); + if (next_version_ <= version || pending_keys_.empty()) { + // no changes, revert back to the published version + next_version_ = version; + pending_keys_.clear(); + return 0; + } + for (const auto& key : pending_keys_) { + auto it = map_.find(key); + ASSERT(it != map_.end()); + dirty_values_.emplace(it->second); + if (it->second->get(next_version_) == nullptr) { + map_.erase(it); + } + } + published_version_.store(next_version_, std::memory_order_release); + pending_keys_.clear(); + return next_version_; + } + + DeferredDeletion revert() { + auto version = published_version_.load(std::memory_order_relaxed); + DeferredDeletion deferred; + for (const auto& key : pending_keys_) { + auto it = map_.find(key); + ASSERT(it != map_.end()); + it->second->revert(version, deferred); + if (it->second->get(version) == nullptr) { + map_.erase(it); + } + } + pending_keys_.clear(); + next_version_ = version; + return deferred; + } + + // First phase GC after all readers have quiesced for 'published_version'. This unlinks nodes that + // are no longer visible and returns them in a deferred-deletion batch that must survive until all + // readers have quiesced once more before it is destroyed. + DeferredDeletion gc(uint64_t published_version) { + DeferredDeletion deferred; + for (auto it = dirty_values_.begin(); it != dirty_values_.end();) { + const auto& handle = *it; + if (handle->gcForVersion(published_version, deferred)) { + dirty_values_.erase(it++); + continue; + } + ++it; + } + return deferred; + } + +private: + friend struct VersionedTestAccess; + using MutableHandle = std::shared_ptr; + + // Persistent published state. + std::atomic published_version_{versionMin}; + absl::flat_hash_map map_; + absl::flat_hash_set dirty_values_; + + // Transactional state for the currently prepared unpublished update. + // Valid only after prepareNextVersion() and until publishNextVersion() or revert(). + // `next_version_` is the candidate version being built, and `pending_keys_` + // contains the keys touched in that candidate update. The container is reused + // across updates to avoid repeated allocation churn. + uint64_t next_version_{versionMin}; + absl::flat_hash_set pending_keys_; +}; + +template struct VersionedTestAccess { + using Map = VersionedMap; + using Handle = typename Map::Handle; + using MutableHandle = typename Map::MutableHandle; + using Node = VersionedNode; + + static const absl::flat_hash_map& entries(const Map& map) { return map.map_; } + + static const absl::flat_hash_set& dirtyValues(const Map& map) { + return map.dirty_values_; + } + + static const Node* head(const Handle& handle) { + if (handle == nullptr) { + return nullptr; + } + auto value = std::static_pointer_cast(handle); + return value->head_.load(std::memory_order_acquire); + } + + static const Node* next(const Node* node) { + return node ? node->next_.load(std::memory_order_acquire) : nullptr; + } + + static const T* value(const Node* node) { return node ? node->value() : nullptr; } + + static uint64_t addVersion(const Node* node) { return node->add_version_; } + + static uint64_t removeVersion(const Node* node) { + return node->remove_version_.load(std::memory_order_relaxed); + } + + static bool isVisibleInVersion(const Node* node, uint64_t version) { + return node && node->isVisibleInVersion(version); + } + + static bool isAddedAfterVersion(const Node* node, uint64_t version) { + return node && node->isAddedAfterVersion(version); + } +}; diff --git a/go/cilium/api/bpf_metadata.pb.go b/go/cilium/api/bpf_metadata.pb.go index 81281988a..22993b3cd 100644 --- a/go/cilium/api/bpf_metadata.pb.go +++ b/go/cilium/api/bpf_metadata.pb.go @@ -24,6 +24,53 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +// Policy type to use +type BpfMetadata_PolicyType int32 + +const ( + BpfMetadata_NPDS BpfMetadata_PolicyType = 0 // Legacy NPDS (default) + BpfMetadata_NPRDS BpfMetadata_PolicyType = 1 // New NetworkPolicyResource (NPRDS) +) + +// Enum value maps for BpfMetadata_PolicyType. +var ( + BpfMetadata_PolicyType_name = map[int32]string{ + 0: "NPDS", + 1: "NPRDS", + } + BpfMetadata_PolicyType_value = map[string]int32{ + "NPDS": 0, + "NPRDS": 1, + } +) + +func (x BpfMetadata_PolicyType) Enum() *BpfMetadata_PolicyType { + p := new(BpfMetadata_PolicyType) + *p = x + return p +} + +func (x BpfMetadata_PolicyType) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (BpfMetadata_PolicyType) Descriptor() protoreflect.EnumDescriptor { + return file_cilium_api_bpf_metadata_proto_enumTypes[0].Descriptor() +} + +func (BpfMetadata_PolicyType) Type() protoreflect.EnumType { + return &file_cilium_api_bpf_metadata_proto_enumTypes[0] +} + +func (x BpfMetadata_PolicyType) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use BpfMetadata_PolicyType.Descriptor instead. +func (BpfMetadata_PolicyType) EnumDescriptor() ([]byte, []int) { + return file_cilium_api_bpf_metadata_proto_rawDescGZIP(), []int{0, 0} +} + type BpfMetadata struct { state protoimpl.MessageState `protogen:"open.v1"` // File system root for bpf. Bpf will not be used if left empty. @@ -90,6 +137,7 @@ type BpfMetadata struct { // Configuration for the source of Cilium xDS updates. // Used for all cilium-specific xDS protocol, e.g., NPHDS, NPDS, and Secrets (SDS) therein. CiliumConfigSource *v3.ConfigSource `protobuf:"bytes,16,opt,name=cilium_config_source,json=ciliumConfigSource,proto3" json:"cilium_config_source,omitempty"` + PolicyType bool `protobuf:"varint,17,opt,name=policy_type,json=policyType,proto3" json:"policy_type,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -236,11 +284,18 @@ func (x *BpfMetadata) GetCiliumConfigSource() *v3.ConfigSource { return nil } +func (x *BpfMetadata) GetPolicyType() bool { + if x != nil { + return x.PolicyType + } + return false +} + var File_cilium_api_bpf_metadata_proto protoreflect.FileDescriptor const file_cilium_api_bpf_metadata_proto_rawDesc = "" + "\n" + - "\x1dcilium/api/bpf_metadata.proto\x12\x06cilium\x1a(envoy/config/core/v3/config_source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x17validate/validate.proto\"\xea\x06\n" + + "\x1dcilium/api/bpf_metadata.proto\x12\x06cilium\x1a(envoy/config/core/v3/config_source.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x17validate/validate.proto\"\xae\a\n" + "\vBpfMetadata\x12\x19\n" + "\bbpf_root\x18\x01 \x01(\tR\abpfRoot\x12\x1d\n" + "\n" + @@ -259,7 +314,13 @@ const file_cilium_api_bpf_metadata_proto_rawDesc = "" + "\tuse_nphds\x18\r \x01(\bR\buseNphds\x12A\n" + "\x0fcache_entry_ttl\x18\x0e \x01(\v2\x19.google.protobuf.DurationR\rcacheEntryTtl\x12E\n" + "\x11cache_gc_interval\x18\x0f \x01(\v2\x19.google.protobuf.DurationR\x0fcacheGcInterval\x12T\n" + - "\x14cilium_config_source\x18\x10 \x01(\v2\".envoy.config.core.v3.ConfigSourceR\x12ciliumConfigSourceB!\n" + + "\x14cilium_config_source\x18\x10 \x01(\v2\".envoy.config.core.v3.ConfigSourceR\x12ciliumConfigSource\x12\x1f\n" + + "\vpolicy_type\x18\x11 \x01(\bR\n" + + "policyType\"!\n" + + "\n" + + "PolicyType\x12\b\n" + + "\x04NPDS\x10\x00\x12\t\n" + + "\x05NPRDS\x10\x01B!\n" + "\x1f_original_source_so_linger_timeB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( @@ -274,17 +335,19 @@ func file_cilium_api_bpf_metadata_proto_rawDescGZIP() []byte { return file_cilium_api_bpf_metadata_proto_rawDescData } +var file_cilium_api_bpf_metadata_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_cilium_api_bpf_metadata_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_cilium_api_bpf_metadata_proto_goTypes = []any{ - (*BpfMetadata)(nil), // 0: cilium.BpfMetadata - (*durationpb.Duration)(nil), // 1: google.protobuf.Duration - (*v3.ConfigSource)(nil), // 2: envoy.config.core.v3.ConfigSource + (BpfMetadata_PolicyType)(0), // 0: cilium.BpfMetadata.PolicyType + (*BpfMetadata)(nil), // 1: cilium.BpfMetadata + (*durationpb.Duration)(nil), // 2: google.protobuf.Duration + (*v3.ConfigSource)(nil), // 3: envoy.config.core.v3.ConfigSource } var file_cilium_api_bpf_metadata_proto_depIdxs = []int32{ - 1, // 0: cilium.BpfMetadata.policy_update_warning_limit:type_name -> google.protobuf.Duration - 1, // 1: cilium.BpfMetadata.cache_entry_ttl:type_name -> google.protobuf.Duration - 1, // 2: cilium.BpfMetadata.cache_gc_interval:type_name -> google.protobuf.Duration - 2, // 3: cilium.BpfMetadata.cilium_config_source:type_name -> envoy.config.core.v3.ConfigSource + 2, // 0: cilium.BpfMetadata.policy_update_warning_limit:type_name -> google.protobuf.Duration + 2, // 1: cilium.BpfMetadata.cache_entry_ttl:type_name -> google.protobuf.Duration + 2, // 2: cilium.BpfMetadata.cache_gc_interval:type_name -> google.protobuf.Duration + 3, // 3: cilium.BpfMetadata.cilium_config_source:type_name -> envoy.config.core.v3.ConfigSource 4, // [4:4] is the sub-list for method output_type 4, // [4:4] is the sub-list for method input_type 4, // [4:4] is the sub-list for extension type_name @@ -303,13 +366,14 @@ func file_cilium_api_bpf_metadata_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cilium_api_bpf_metadata_proto_rawDesc), len(file_cilium_api_bpf_metadata_proto_rawDesc)), - NumEnums: 0, + NumEnums: 1, NumMessages: 1, NumExtensions: 0, NumServices: 0, }, GoTypes: file_cilium_api_bpf_metadata_proto_goTypes, DependencyIndexes: file_cilium_api_bpf_metadata_proto_depIdxs, + EnumInfos: file_cilium_api_bpf_metadata_proto_enumTypes, MessageInfos: file_cilium_api_bpf_metadata_proto_msgTypes, }.Build() File_cilium_api_bpf_metadata_proto = out.File diff --git a/go/cilium/api/bpf_metadata.pb.validate.go b/go/cilium/api/bpf_metadata.pb.validate.go index 8dc8b53aa..b8f72a926 100644 --- a/go/cilium/api/bpf_metadata.pb.validate.go +++ b/go/cilium/api/bpf_metadata.pb.validate.go @@ -204,6 +204,8 @@ func (m *BpfMetadata) validate(all bool) error { } } + // no validation rules for PolicyType + if m.OriginalSourceSoLingerTime != nil { // no validation rules for OriginalSourceSoLingerTime } diff --git a/go/cilium/api/npds.pb.go b/go/cilium/api/npds.pb.go index 2b09745e8..ad1714305 100644 --- a/go/cilium/api/npds.pb.go +++ b/go/cilium/api/npds.pb.go @@ -75,7 +75,7 @@ func (x HeaderMatch_MatchAction) Number() protoreflect.EnumNumber { // Deprecated: Use HeaderMatch_MatchAction.Descriptor instead. func (HeaderMatch_MatchAction) EnumDescriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5, 0} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7, 0} } type HeaderMatch_MismatchAction int32 @@ -130,7 +130,139 @@ func (x HeaderMatch_MismatchAction) Number() protoreflect.EnumNumber { // Deprecated: Use HeaderMatch_MismatchAction.Descriptor instead. func (HeaderMatch_MismatchAction) EnumDescriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5, 1} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7, 1} +} + +// An NPRDS resource that carries either an endpoint policy or a shared selector. +type NetworkPolicyResource struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Resource: + // + // *NetworkPolicyResource_Policy + // *NetworkPolicyResource_Selector + Resource isNetworkPolicyResource_Resource `protobuf_oneof:"resource"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *NetworkPolicyResource) Reset() { + *x = NetworkPolicyResource{} + mi := &file_cilium_api_npds_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *NetworkPolicyResource) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*NetworkPolicyResource) ProtoMessage() {} + +func (x *NetworkPolicyResource) ProtoReflect() protoreflect.Message { + mi := &file_cilium_api_npds_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use NetworkPolicyResource.ProtoReflect.Descriptor instead. +func (*NetworkPolicyResource) Descriptor() ([]byte, []int) { + return file_cilium_api_npds_proto_rawDescGZIP(), []int{0} +} + +func (x *NetworkPolicyResource) GetResource() isNetworkPolicyResource_Resource { + if x != nil { + return x.Resource + } + return nil +} + +func (x *NetworkPolicyResource) GetPolicy() *NetworkPolicy { + if x != nil { + if x, ok := x.Resource.(*NetworkPolicyResource_Policy); ok { + return x.Policy + } + } + return nil +} + +func (x *NetworkPolicyResource) GetSelector() *Selector { + if x != nil { + if x, ok := x.Resource.(*NetworkPolicyResource_Selector); ok { + return x.Selector + } + } + return nil +} + +type isNetworkPolicyResource_Resource interface { + isNetworkPolicyResource_Resource() +} + +type NetworkPolicyResource_Policy struct { + Policy *NetworkPolicy `protobuf:"bytes,1,opt,name=policy,proto3,oneof"` +} + +type NetworkPolicyResource_Selector struct { + Selector *Selector `protobuf:"bytes,2,opt,name=selector,proto3,oneof"` +} + +func (*NetworkPolicyResource_Policy) isNetworkPolicyResource_Resource() {} + +func (*NetworkPolicyResource_Selector) isNetworkPolicyResource_Resource() {} + +// A shared set of remote identities referenced by selector resource name. +// Unlike the old state-of-the-world remote identity lists, an empty selector +// matches nothing. +type Selector struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The set of numeric remote security IDs selected by this selector. + // If empty, this selector selects no remote identities. + RemoteIdentities []uint32 `protobuf:"varint,1,rep,packed,name=remote_identities,json=remoteIdentities,proto3" json:"remote_identities,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Selector) Reset() { + *x = Selector{} + mi := &file_cilium_api_npds_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Selector) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Selector) ProtoMessage() {} + +func (x *Selector) ProtoReflect() protoreflect.Message { + mi := &file_cilium_api_npds_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Selector.ProtoReflect.Descriptor instead. +func (*Selector) Descriptor() ([]byte, []int) { + return file_cilium_api_npds_proto_rawDescGZIP(), []int{1} +} + +func (x *Selector) GetRemoteIdentities() []uint32 { + if x != nil { + return x.RemoteIdentities + } + return nil } // A network policy that is enforced by a filter on the network flows to/from @@ -161,7 +293,7 @@ type NetworkPolicy struct { func (x *NetworkPolicy) Reset() { *x = NetworkPolicy{} - mi := &file_cilium_api_npds_proto_msgTypes[0] + mi := &file_cilium_api_npds_proto_msgTypes[2] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -173,7 +305,7 @@ func (x *NetworkPolicy) String() string { func (*NetworkPolicy) ProtoMessage() {} func (x *NetworkPolicy) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[0] + mi := &file_cilium_api_npds_proto_msgTypes[2] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -186,7 +318,7 @@ func (x *NetworkPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkPolicy.ProtoReflect.Descriptor instead. func (*NetworkPolicy) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{0} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{2} } func (x *NetworkPolicy) GetEndpointIps() []string { @@ -240,7 +372,7 @@ type PortNetworkPolicy struct { func (x *PortNetworkPolicy) Reset() { *x = PortNetworkPolicy{} - mi := &file_cilium_api_npds_proto_msgTypes[1] + mi := &file_cilium_api_npds_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -252,7 +384,7 @@ func (x *PortNetworkPolicy) String() string { func (*PortNetworkPolicy) ProtoMessage() {} func (x *PortNetworkPolicy) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[1] + mi := &file_cilium_api_npds_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -265,7 +397,7 @@ func (x *PortNetworkPolicy) ProtoReflect() protoreflect.Message { // Deprecated: Use PortNetworkPolicy.ProtoReflect.Descriptor instead. func (*PortNetworkPolicy) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{1} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{3} } func (x *PortNetworkPolicy) GetPort() uint32 { @@ -328,7 +460,7 @@ type TLSContext struct { func (x *TLSContext) Reset() { *x = TLSContext{} - mi := &file_cilium_api_npds_proto_msgTypes[2] + mi := &file_cilium_api_npds_proto_msgTypes[4] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -340,7 +472,7 @@ func (x *TLSContext) String() string { func (*TLSContext) ProtoMessage() {} func (x *TLSContext) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[2] + mi := &file_cilium_api_npds_proto_msgTypes[4] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -353,7 +485,7 @@ func (x *TLSContext) ProtoReflect() protoreflect.Message { // Deprecated: Use TLSContext.ProtoReflect.Descriptor instead. func (*TLSContext) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{2} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{4} } func (x *TLSContext) GetTrustedCa() string { @@ -433,6 +565,11 @@ type PortNetworkPolicyRule struct { // applied on the flow's remote host is contained in this set. // Optional. If not specified, any remote host is matched by this predicate. RemotePolicies []uint32 `protobuf:"varint,7,rep,packed,name=remote_policies,json=remotePolicies,proto3" json:"remote_policies,omitempty"` + // Optional selector resource names that can be resolved to shared remote + // policy sets in delta NPDS. + // Selector references are matched by exact selector resource name. + // Optional. If not specified, any remote host is matched by this predicate. + Selectors []string `protobuf:"bytes,11,rep,name=selectors,proto3" json:"selectors,omitempty"` // Optional downstream TLS context. If present, the incoming connection must // be a TLS connection. DownstreamTlsContext *TLSContext `protobuf:"bytes,3,opt,name=downstream_tls_context,json=downstreamTlsContext,proto3" json:"downstream_tls_context,omitempty"` @@ -473,7 +610,7 @@ type PortNetworkPolicyRule struct { func (x *PortNetworkPolicyRule) Reset() { *x = PortNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[3] + mi := &file_cilium_api_npds_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -485,7 +622,7 @@ func (x *PortNetworkPolicyRule) String() string { func (*PortNetworkPolicyRule) ProtoMessage() {} func (x *PortNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[3] + mi := &file_cilium_api_npds_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -498,7 +635,7 @@ func (x *PortNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use PortNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*PortNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{3} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{5} } func (x *PortNetworkPolicyRule) GetPrecedence() uint32 { @@ -554,6 +691,13 @@ func (x *PortNetworkPolicyRule) GetRemotePolicies() []uint32 { return nil } +func (x *PortNetworkPolicyRule) GetSelectors() []string { + if x != nil { + return x.Selectors + } + return nil +} + func (x *PortNetworkPolicyRule) GetDownstreamTlsContext() *TLSContext { if x != nil { return x.DownstreamTlsContext @@ -679,7 +823,7 @@ type HttpNetworkPolicyRules struct { func (x *HttpNetworkPolicyRules) Reset() { *x = HttpNetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[4] + mi := &file_cilium_api_npds_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -691,7 +835,7 @@ func (x *HttpNetworkPolicyRules) String() string { func (*HttpNetworkPolicyRules) ProtoMessage() {} func (x *HttpNetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[4] + mi := &file_cilium_api_npds_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -704,7 +848,7 @@ func (x *HttpNetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpNetworkPolicyRules.ProtoReflect.Descriptor instead. func (*HttpNetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{4} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{6} } func (x *HttpNetworkPolicyRules) GetHttpRules() []*HttpNetworkPolicyRule { @@ -729,7 +873,7 @@ type HeaderMatch struct { func (x *HeaderMatch) Reset() { *x = HeaderMatch{} - mi := &file_cilium_api_npds_proto_msgTypes[5] + mi := &file_cilium_api_npds_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -741,7 +885,7 @@ func (x *HeaderMatch) String() string { func (*HeaderMatch) ProtoMessage() {} func (x *HeaderMatch) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[5] + mi := &file_cilium_api_npds_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -754,7 +898,7 @@ func (x *HeaderMatch) ProtoReflect() protoreflect.Message { // Deprecated: Use HeaderMatch.ProtoReflect.Descriptor instead. func (*HeaderMatch) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{5} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{7} } func (x *HeaderMatch) GetName() string { @@ -822,7 +966,7 @@ type HttpNetworkPolicyRule struct { func (x *HttpNetworkPolicyRule) Reset() { *x = HttpNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[6] + mi := &file_cilium_api_npds_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -834,7 +978,7 @@ func (x *HttpNetworkPolicyRule) String() string { func (*HttpNetworkPolicyRule) ProtoMessage() {} func (x *HttpNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[6] + mi := &file_cilium_api_npds_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -847,7 +991,7 @@ func (x *HttpNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use HttpNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*HttpNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{6} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{8} } func (x *HttpNetworkPolicyRule) GetHeaders() []*v31.HeaderMatcher { @@ -877,7 +1021,7 @@ type KafkaNetworkPolicyRules struct { func (x *KafkaNetworkPolicyRules) Reset() { *x = KafkaNetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[7] + mi := &file_cilium_api_npds_proto_msgTypes[9] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -889,7 +1033,7 @@ func (x *KafkaNetworkPolicyRules) String() string { func (*KafkaNetworkPolicyRules) ProtoMessage() {} func (x *KafkaNetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[7] + mi := &file_cilium_api_npds_proto_msgTypes[9] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -902,7 +1046,7 @@ func (x *KafkaNetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use KafkaNetworkPolicyRules.ProtoReflect.Descriptor instead. func (*KafkaNetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{7} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{9} } func (x *KafkaNetworkPolicyRules) GetKafkaRules() []*KafkaNetworkPolicyRule { @@ -941,7 +1085,7 @@ type KafkaNetworkPolicyRule struct { func (x *KafkaNetworkPolicyRule) Reset() { *x = KafkaNetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[8] + mi := &file_cilium_api_npds_proto_msgTypes[10] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -953,7 +1097,7 @@ func (x *KafkaNetworkPolicyRule) String() string { func (*KafkaNetworkPolicyRule) ProtoMessage() {} func (x *KafkaNetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[8] + mi := &file_cilium_api_npds_proto_msgTypes[10] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -966,7 +1110,7 @@ func (x *KafkaNetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use KafkaNetworkPolicyRule.ProtoReflect.Descriptor instead. func (*KafkaNetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{8} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{10} } func (x *KafkaNetworkPolicyRule) GetApiVersion() int32 { @@ -1017,7 +1161,7 @@ type L7NetworkPolicyRules struct { func (x *L7NetworkPolicyRules) Reset() { *x = L7NetworkPolicyRules{} - mi := &file_cilium_api_npds_proto_msgTypes[9] + mi := &file_cilium_api_npds_proto_msgTypes[11] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1029,7 +1173,7 @@ func (x *L7NetworkPolicyRules) String() string { func (*L7NetworkPolicyRules) ProtoMessage() {} func (x *L7NetworkPolicyRules) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[9] + mi := &file_cilium_api_npds_proto_msgTypes[11] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1042,7 +1186,7 @@ func (x *L7NetworkPolicyRules) ProtoReflect() protoreflect.Message { // Deprecated: Use L7NetworkPolicyRules.ProtoReflect.Descriptor instead. func (*L7NetworkPolicyRules) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{9} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{11} } func (x *L7NetworkPolicyRules) GetL7AllowRules() []*L7NetworkPolicyRule { @@ -1080,7 +1224,7 @@ type L7NetworkPolicyRule struct { func (x *L7NetworkPolicyRule) Reset() { *x = L7NetworkPolicyRule{} - mi := &file_cilium_api_npds_proto_msgTypes[10] + mi := &file_cilium_api_npds_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1092,7 +1236,7 @@ func (x *L7NetworkPolicyRule) String() string { func (*L7NetworkPolicyRule) ProtoMessage() {} func (x *L7NetworkPolicyRule) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[10] + mi := &file_cilium_api_npds_proto_msgTypes[12] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1105,7 +1249,7 @@ func (x *L7NetworkPolicyRule) ProtoReflect() protoreflect.Message { // Deprecated: Use L7NetworkPolicyRule.ProtoReflect.Descriptor instead. func (*L7NetworkPolicyRule) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{10} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{12} } func (x *L7NetworkPolicyRule) GetName() string { @@ -1140,7 +1284,7 @@ type NetworkPoliciesConfigDump struct { func (x *NetworkPoliciesConfigDump) Reset() { *x = NetworkPoliciesConfigDump{} - mi := &file_cilium_api_npds_proto_msgTypes[11] + mi := &file_cilium_api_npds_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -1152,7 +1296,7 @@ func (x *NetworkPoliciesConfigDump) String() string { func (*NetworkPoliciesConfigDump) ProtoMessage() {} func (x *NetworkPoliciesConfigDump) ProtoReflect() protoreflect.Message { - mi := &file_cilium_api_npds_proto_msgTypes[11] + mi := &file_cilium_api_npds_proto_msgTypes[13] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -1165,7 +1309,7 @@ func (x *NetworkPoliciesConfigDump) ProtoReflect() protoreflect.Message { // Deprecated: Use NetworkPoliciesConfigDump.ProtoReflect.Descriptor instead. func (*NetworkPoliciesConfigDump) Descriptor() ([]byte, []int) { - return file_cilium_api_npds_proto_rawDescGZIP(), []int{11} + return file_cilium_api_npds_proto_rawDescGZIP(), []int{13} } func (x *NetworkPoliciesConfigDump) GetNetworkpolicies() []*NetworkPolicy { @@ -1179,7 +1323,14 @@ var File_cilium_api_npds_proto protoreflect.FileDescriptor const file_cilium_api_npds_proto_rawDesc = "" + "\n" + - "\x15cilium/api/npds.proto\x12\x06cilium\x1a\"envoy/config/core/v3/address.proto\x1a,envoy/config/route/v3/route_components.proto\x1a*envoy/service/discovery/v3/discovery.proto\x1a$envoy/type/matcher/v3/metadata.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"\x95\x02\n" + + "\x15cilium/api/npds.proto\x12\x06cilium\x1a\"envoy/config/core/v3/address.proto\x1a,envoy/config/route/v3/route_components.proto\x1a*envoy/service/discovery/v3/discovery.proto\x1a$envoy/type/matcher/v3/metadata.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"\x84\x01\n" + + "\x15NetworkPolicyResource\x12/\n" + + "\x06policy\x18\x01 \x01(\v2\x15.cilium.NetworkPolicyH\x00R\x06policy\x12.\n" + + "\bselector\x18\x02 \x01(\v2\x10.cilium.SelectorH\x00R\bselectorB\n" + + "\n" + + "\bresource\"7\n" + + "\bSelector\x12+\n" + + "\x11remote_identities\x18\x01 \x03(\rR\x10remoteIdentities\"\x95\x02\n" + "\rNetworkPolicy\x123\n" + "\fendpoint_ips\x18\x01 \x03(\tB\x10\xfaB\r\x92\x01\n" + "\b\x01\x10\x02\"\x04r\x02\x10\x01R\vendpointIps\x12\x1f\n" + @@ -1202,7 +1353,7 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\fserver_names\x18\x04 \x03(\tR\vserverNames\x12A\n" + "\x1dvalidation_context_sds_secret\x18\x05 \x01(\tR\x1avalidationContextSdsSecret\x12$\n" + "\x0etls_sds_secret\x18\x06 \x01(\tR\ftlsSdsSecret\x12%\n" + - "\x0ealpn_protocols\x18\a \x03(\tR\ralpnProtocols\"\xfb\x05\n" + + "\x0ealpn_protocols\x18\a \x03(\tR\ralpnProtocols\"\x99\x06\n" + "\x15PortNetworkPolicyRule\x12\x1e\n" + "\n" + "precedence\x18\n" + @@ -1212,7 +1363,8 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\x04deny\x18\b \x01(\bH\x00R\x04deny\x12$\n" + "\bproxy_id\x18\t \x01(\rB\t\xfaB\x06*\x04\x18\xff\xff\x03R\aproxyId\x12\x12\n" + "\x04name\x18\x05 \x01(\tR\x04name\x12'\n" + - "\x0fremote_policies\x18\a \x03(\rR\x0eremotePolicies\x12H\n" + + "\x0fremote_policies\x18\a \x03(\rR\x0eremotePolicies\x12\x1c\n" + + "\tselectors\x18\v \x03(\tR\tselectors\x12H\n" + "\x16downstream_tls_context\x18\x03 \x01(\v2\x12.cilium.TLSContextR\x14downstreamTlsContext\x12D\n" + "\x14upstream_tls_context\x18\x04 \x01(\v2\x12.cilium.TLSContextR\x12upstreamTlsContext\x12\xa1\x01\n" + "\fserver_names\x18\x06 \x03(\tB~\xfaB{\x92\x01x\"vrt2r^(([*]{1,2}|[*]?[-a-zA-Z0-9_]+([*][-a-zA-Z0-9_]+)*[*]?)[.])*([*]{1,2}|[*]?[-a-zA-Z0-9_]+([*][-a-zA-Z0-9_]+)*[*]?)$R\vserverNames\x12\x19\n" + @@ -1266,11 +1418,16 @@ const file_cilium_api_npds_proto_rawDesc = "" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\\\n" + "\x19NetworkPoliciesConfigDump\x12?\n" + - "\x0fnetworkpolicies\x18\x01 \x03(\v2\x15.cilium.NetworkPolicyR\x0fnetworkpolicies2\xda\x02\n" + - "\x1dNetworkPolicyDiscoveryService\x12z\n" + + "\x0fnetworkpolicies\x18\x01 \x03(\v2\x15.cilium.NetworkPolicyR\x0fnetworkpolicies2\xe0\x03\n" + + "\x1dNetworkPolicyDiscoveryService\x12\x83\x01\n" + + "\x14DeltaNetworkPolicies\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x12z\n" + "\x15StreamNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\x9e\x01\n" + "\x14FetchNetworkPolicies\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\")\x82\xd3\xe4\x93\x02#:\x01*\"\x1e/v3/discovery:network_policies\x1a\x1c\x8a\xa4\x96\xf3\a\x16\n" + - "\x14cilium.NetworkPolicyB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" + "\x14cilium.NetworkPolicy2\xde\x02\n" + + "%NetworkPolicyResourceDiscoveryService\x12\x81\x01\n" + + "\x1cStreamNetworkPolicyResources\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\x8a\x01\n" + + "\x1bDeltaNetworkPolicyResources\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x1a$\x8a\xa4\x96\xf3\a\x1e\n" + + "\x1ccilium.NetworkPolicyResourceB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( file_cilium_api_npds_proto_rawDescOnce sync.Once @@ -1285,59 +1442,71 @@ func file_cilium_api_npds_proto_rawDescGZIP() []byte { } var file_cilium_api_npds_proto_enumTypes = make([]protoimpl.EnumInfo, 2) -var file_cilium_api_npds_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_cilium_api_npds_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_cilium_api_npds_proto_goTypes = []any{ - (HeaderMatch_MatchAction)(0), // 0: cilium.HeaderMatch.MatchAction - (HeaderMatch_MismatchAction)(0), // 1: cilium.HeaderMatch.MismatchAction - (*NetworkPolicy)(nil), // 2: cilium.NetworkPolicy - (*PortNetworkPolicy)(nil), // 3: cilium.PortNetworkPolicy - (*TLSContext)(nil), // 4: cilium.TLSContext - (*PortNetworkPolicyRule)(nil), // 5: cilium.PortNetworkPolicyRule - (*HttpNetworkPolicyRules)(nil), // 6: cilium.HttpNetworkPolicyRules - (*HeaderMatch)(nil), // 7: cilium.HeaderMatch - (*HttpNetworkPolicyRule)(nil), // 8: cilium.HttpNetworkPolicyRule - (*KafkaNetworkPolicyRules)(nil), // 9: cilium.KafkaNetworkPolicyRules - (*KafkaNetworkPolicyRule)(nil), // 10: cilium.KafkaNetworkPolicyRule - (*L7NetworkPolicyRules)(nil), // 11: cilium.L7NetworkPolicyRules - (*L7NetworkPolicyRule)(nil), // 12: cilium.L7NetworkPolicyRule - (*NetworkPoliciesConfigDump)(nil), // 13: cilium.NetworkPoliciesConfigDump - nil, // 14: cilium.L7NetworkPolicyRule.RuleEntry - (v3.SocketAddress_Protocol)(0), // 15: envoy.config.core.v3.SocketAddress.Protocol - (*v31.HeaderMatcher)(nil), // 16: envoy.config.route.v3.HeaderMatcher - (*v32.MetadataMatcher)(nil), // 17: envoy.type.matcher.v3.MetadataMatcher - (*v33.DiscoveryRequest)(nil), // 18: envoy.service.discovery.v3.DiscoveryRequest - (*v33.DiscoveryResponse)(nil), // 19: envoy.service.discovery.v3.DiscoveryResponse + (HeaderMatch_MatchAction)(0), // 0: cilium.HeaderMatch.MatchAction + (HeaderMatch_MismatchAction)(0), // 1: cilium.HeaderMatch.MismatchAction + (*NetworkPolicyResource)(nil), // 2: cilium.NetworkPolicyResource + (*Selector)(nil), // 3: cilium.Selector + (*NetworkPolicy)(nil), // 4: cilium.NetworkPolicy + (*PortNetworkPolicy)(nil), // 5: cilium.PortNetworkPolicy + (*TLSContext)(nil), // 6: cilium.TLSContext + (*PortNetworkPolicyRule)(nil), // 7: cilium.PortNetworkPolicyRule + (*HttpNetworkPolicyRules)(nil), // 8: cilium.HttpNetworkPolicyRules + (*HeaderMatch)(nil), // 9: cilium.HeaderMatch + (*HttpNetworkPolicyRule)(nil), // 10: cilium.HttpNetworkPolicyRule + (*KafkaNetworkPolicyRules)(nil), // 11: cilium.KafkaNetworkPolicyRules + (*KafkaNetworkPolicyRule)(nil), // 12: cilium.KafkaNetworkPolicyRule + (*L7NetworkPolicyRules)(nil), // 13: cilium.L7NetworkPolicyRules + (*L7NetworkPolicyRule)(nil), // 14: cilium.L7NetworkPolicyRule + (*NetworkPoliciesConfigDump)(nil), // 15: cilium.NetworkPoliciesConfigDump + nil, // 16: cilium.L7NetworkPolicyRule.RuleEntry + (v3.SocketAddress_Protocol)(0), // 17: envoy.config.core.v3.SocketAddress.Protocol + (*v31.HeaderMatcher)(nil), // 18: envoy.config.route.v3.HeaderMatcher + (*v32.MetadataMatcher)(nil), // 19: envoy.type.matcher.v3.MetadataMatcher + (*v33.DeltaDiscoveryRequest)(nil), // 20: envoy.service.discovery.v3.DeltaDiscoveryRequest + (*v33.DiscoveryRequest)(nil), // 21: envoy.service.discovery.v3.DiscoveryRequest + (*v33.DeltaDiscoveryResponse)(nil), // 22: envoy.service.discovery.v3.DeltaDiscoveryResponse + (*v33.DiscoveryResponse)(nil), // 23: envoy.service.discovery.v3.DiscoveryResponse } var file_cilium_api_npds_proto_depIdxs = []int32{ - 3, // 0: cilium.NetworkPolicy.ingress_per_port_policies:type_name -> cilium.PortNetworkPolicy - 3, // 1: cilium.NetworkPolicy.egress_per_port_policies:type_name -> cilium.PortNetworkPolicy - 15, // 2: cilium.PortNetworkPolicy.protocol:type_name -> envoy.config.core.v3.SocketAddress.Protocol - 5, // 3: cilium.PortNetworkPolicy.rules:type_name -> cilium.PortNetworkPolicyRule - 4, // 4: cilium.PortNetworkPolicyRule.downstream_tls_context:type_name -> cilium.TLSContext - 4, // 5: cilium.PortNetworkPolicyRule.upstream_tls_context:type_name -> cilium.TLSContext - 6, // 6: cilium.PortNetworkPolicyRule.http_rules:type_name -> cilium.HttpNetworkPolicyRules - 9, // 7: cilium.PortNetworkPolicyRule.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRules - 11, // 8: cilium.PortNetworkPolicyRule.l7_rules:type_name -> cilium.L7NetworkPolicyRules - 8, // 9: cilium.HttpNetworkPolicyRules.http_rules:type_name -> cilium.HttpNetworkPolicyRule - 0, // 10: cilium.HeaderMatch.match_action:type_name -> cilium.HeaderMatch.MatchAction - 1, // 11: cilium.HeaderMatch.mismatch_action:type_name -> cilium.HeaderMatch.MismatchAction - 16, // 12: cilium.HttpNetworkPolicyRule.headers:type_name -> envoy.config.route.v3.HeaderMatcher - 7, // 13: cilium.HttpNetworkPolicyRule.header_matches:type_name -> cilium.HeaderMatch - 10, // 14: cilium.KafkaNetworkPolicyRules.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRule - 12, // 15: cilium.L7NetworkPolicyRules.l7_allow_rules:type_name -> cilium.L7NetworkPolicyRule - 12, // 16: cilium.L7NetworkPolicyRules.l7_deny_rules:type_name -> cilium.L7NetworkPolicyRule - 14, // 17: cilium.L7NetworkPolicyRule.rule:type_name -> cilium.L7NetworkPolicyRule.RuleEntry - 17, // 18: cilium.L7NetworkPolicyRule.metadata_rule:type_name -> envoy.type.matcher.v3.MetadataMatcher - 2, // 19: cilium.NetworkPoliciesConfigDump.networkpolicies:type_name -> cilium.NetworkPolicy - 18, // 20: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 18, // 21: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 19, // 22: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 19, // 23: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 22, // [22:24] is the sub-list for method output_type - 20, // [20:22] is the sub-list for method input_type - 20, // [20:20] is the sub-list for extension type_name - 20, // [20:20] is the sub-list for extension extendee - 0, // [0:20] is the sub-list for field type_name + 4, // 0: cilium.NetworkPolicyResource.policy:type_name -> cilium.NetworkPolicy + 3, // 1: cilium.NetworkPolicyResource.selector:type_name -> cilium.Selector + 5, // 2: cilium.NetworkPolicy.ingress_per_port_policies:type_name -> cilium.PortNetworkPolicy + 5, // 3: cilium.NetworkPolicy.egress_per_port_policies:type_name -> cilium.PortNetworkPolicy + 17, // 4: cilium.PortNetworkPolicy.protocol:type_name -> envoy.config.core.v3.SocketAddress.Protocol + 7, // 5: cilium.PortNetworkPolicy.rules:type_name -> cilium.PortNetworkPolicyRule + 6, // 6: cilium.PortNetworkPolicyRule.downstream_tls_context:type_name -> cilium.TLSContext + 6, // 7: cilium.PortNetworkPolicyRule.upstream_tls_context:type_name -> cilium.TLSContext + 8, // 8: cilium.PortNetworkPolicyRule.http_rules:type_name -> cilium.HttpNetworkPolicyRules + 11, // 9: cilium.PortNetworkPolicyRule.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRules + 13, // 10: cilium.PortNetworkPolicyRule.l7_rules:type_name -> cilium.L7NetworkPolicyRules + 10, // 11: cilium.HttpNetworkPolicyRules.http_rules:type_name -> cilium.HttpNetworkPolicyRule + 0, // 12: cilium.HeaderMatch.match_action:type_name -> cilium.HeaderMatch.MatchAction + 1, // 13: cilium.HeaderMatch.mismatch_action:type_name -> cilium.HeaderMatch.MismatchAction + 18, // 14: cilium.HttpNetworkPolicyRule.headers:type_name -> envoy.config.route.v3.HeaderMatcher + 9, // 15: cilium.HttpNetworkPolicyRule.header_matches:type_name -> cilium.HeaderMatch + 12, // 16: cilium.KafkaNetworkPolicyRules.kafka_rules:type_name -> cilium.KafkaNetworkPolicyRule + 14, // 17: cilium.L7NetworkPolicyRules.l7_allow_rules:type_name -> cilium.L7NetworkPolicyRule + 14, // 18: cilium.L7NetworkPolicyRules.l7_deny_rules:type_name -> cilium.L7NetworkPolicyRule + 16, // 19: cilium.L7NetworkPolicyRule.rule:type_name -> cilium.L7NetworkPolicyRule.RuleEntry + 19, // 20: cilium.L7NetworkPolicyRule.metadata_rule:type_name -> envoy.type.matcher.v3.MetadataMatcher + 4, // 21: cilium.NetworkPoliciesConfigDump.networkpolicies:type_name -> cilium.NetworkPolicy + 20, // 22: cilium.NetworkPolicyDiscoveryService.DeltaNetworkPolicies:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 21, // 23: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 21, // 24: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 21, // 25: cilium.NetworkPolicyResourceDiscoveryService.StreamNetworkPolicyResources:input_type -> envoy.service.discovery.v3.DiscoveryRequest + 20, // 26: cilium.NetworkPolicyResourceDiscoveryService.DeltaNetworkPolicyResources:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 22, // 27: cilium.NetworkPolicyDiscoveryService.DeltaNetworkPolicies:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 23, // 28: cilium.NetworkPolicyDiscoveryService.StreamNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 23, // 29: cilium.NetworkPolicyDiscoveryService.FetchNetworkPolicies:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 23, // 30: cilium.NetworkPolicyResourceDiscoveryService.StreamNetworkPolicyResources:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 22, // 31: cilium.NetworkPolicyResourceDiscoveryService.DeltaNetworkPolicyResources:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 27, // [27:32] is the sub-list for method output_type + 22, // [22:27] is the sub-list for method input_type + 22, // [22:22] is the sub-list for extension type_name + 22, // [22:22] is the sub-list for extension extendee + 0, // [0:22] is the sub-list for field type_name } func init() { file_cilium_api_npds_proto_init() } @@ -1345,7 +1514,11 @@ func file_cilium_api_npds_proto_init() { if File_cilium_api_npds_proto != nil { return } - file_cilium_api_npds_proto_msgTypes[3].OneofWrappers = []any{ + file_cilium_api_npds_proto_msgTypes[0].OneofWrappers = []any{ + (*NetworkPolicyResource_Policy)(nil), + (*NetworkPolicyResource_Selector)(nil), + } + file_cilium_api_npds_proto_msgTypes[5].OneofWrappers = []any{ (*PortNetworkPolicyRule_PassPrecedence)(nil), (*PortNetworkPolicyRule_Deny)(nil), (*PortNetworkPolicyRule_HttpRules)(nil), @@ -1358,9 +1531,9 @@ func file_cilium_api_npds_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_cilium_api_npds_proto_rawDesc), len(file_cilium_api_npds_proto_rawDesc)), NumEnums: 2, - NumMessages: 13, + NumMessages: 15, NumExtensions: 0, - NumServices: 1, + NumServices: 2, }, GoTypes: file_cilium_api_npds_proto_goTypes, DependencyIndexes: file_cilium_api_npds_proto_depIdxs, diff --git a/go/cilium/api/npds.pb.validate.go b/go/cilium/api/npds.pb.validate.go index 418649f39..ea344bc3f 100644 --- a/go/cilium/api/npds.pb.validate.go +++ b/go/cilium/api/npds.pb.validate.go @@ -39,6 +39,294 @@ var ( _ = corev3.SocketAddress_Protocol(0) ) +// Validate checks the field values on NetworkPolicyResource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *NetworkPolicyResource) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on NetworkPolicyResource with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// NetworkPolicyResourceMultiError, or nil if none found. +func (m *NetworkPolicyResource) ValidateAll() error { + return m.validate(true) +} + +func (m *NetworkPolicyResource) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + switch v := m.Resource.(type) { + case *NetworkPolicyResource_Policy: + if v == nil { + err := NetworkPolicyResourceValidationError{ + field: "Resource", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetPolicy()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetPolicy()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return NetworkPolicyResourceValidationError{ + field: "Policy", + reason: "embedded message failed validation", + cause: err, + } + } + } + + case *NetworkPolicyResource_Selector: + if v == nil { + err := NetworkPolicyResourceValidationError{ + field: "Resource", + reason: "oneof value cannot be a typed-nil", + } + if !all { + return err + } + errors = append(errors, err) + } + + if all { + switch v := interface{}(m.GetSelector()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetSelector()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return NetworkPolicyResourceValidationError{ + field: "Selector", + reason: "embedded message failed validation", + cause: err, + } + } + } + + default: + _ = v // ensures v is used + } + + if len(errors) > 0 { + return NetworkPolicyResourceMultiError(errors) + } + + return nil +} + +// NetworkPolicyResourceMultiError is an error wrapping multiple validation +// errors returned by NetworkPolicyResource.ValidateAll() if the designated +// constraints aren't met. +type NetworkPolicyResourceMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m NetworkPolicyResourceMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m NetworkPolicyResourceMultiError) AllErrors() []error { return m } + +// NetworkPolicyResourceValidationError is the validation error returned by +// NetworkPolicyResource.Validate if the designated constraints aren't met. +type NetworkPolicyResourceValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e NetworkPolicyResourceValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e NetworkPolicyResourceValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e NetworkPolicyResourceValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e NetworkPolicyResourceValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e NetworkPolicyResourceValidationError) ErrorName() string { + return "NetworkPolicyResourceValidationError" +} + +// Error satisfies the builtin error interface +func (e NetworkPolicyResourceValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sNetworkPolicyResource.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = NetworkPolicyResourceValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = NetworkPolicyResourceValidationError{} + +// Validate checks the field values on Selector with the rules defined in the +// proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *Selector) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on Selector with the rules defined in +// the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in SelectorMultiError, or nil +// if none found. +func (m *Selector) ValidateAll() error { + return m.validate(true) +} + +func (m *Selector) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SelectorMultiError(errors) + } + + return nil +} + +// SelectorMultiError is an error wrapping multiple validation errors returned +// by Selector.ValidateAll() if the designated constraints aren't met. +type SelectorMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SelectorMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SelectorMultiError) AllErrors() []error { return m } + +// SelectorValidationError is the validation error returned by +// Selector.Validate if the designated constraints aren't met. +type SelectorValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SelectorValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SelectorValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SelectorValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SelectorValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SelectorValidationError) ErrorName() string { return "SelectorValidationError" } + +// Error satisfies the builtin error interface +func (e SelectorValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSelector.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SelectorValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SelectorValidationError{} + // Validate checks the field values on NetworkPolicy with the rules defined in // the proto definition for this message. If any rules are violated, the first // error encountered is returned, or nil if there are no violations. diff --git a/go/cilium/api/npds_grpc.pb.go b/go/cilium/api/npds_grpc.pb.go index e2a51d9ab..edda91e1c 100644 --- a/go/cilium/api/npds_grpc.pb.go +++ b/go/cilium/api/npds_grpc.pb.go @@ -20,6 +20,7 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( + NetworkPolicyDiscoveryService_DeltaNetworkPolicies_FullMethodName = "/cilium.NetworkPolicyDiscoveryService/DeltaNetworkPolicies" NetworkPolicyDiscoveryService_StreamNetworkPolicies_FullMethodName = "/cilium.NetworkPolicyDiscoveryService/StreamNetworkPolicies" NetworkPolicyDiscoveryService_FetchNetworkPolicies_FullMethodName = "/cilium.NetworkPolicyDiscoveryService/FetchNetworkPolicies" ) @@ -29,7 +30,9 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. type NetworkPolicyDiscoveryServiceClient interface { + DeltaNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) StreamNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) FetchNetworkPolicies(ctx context.Context, in *v3.DiscoveryRequest, opts ...grpc.CallOption) (*v3.DiscoveryResponse, error) } @@ -42,9 +45,22 @@ func NewNetworkPolicyDiscoveryServiceClient(cc grpc.ClientConnInterface) Network return &networkPolicyDiscoveryServiceClient{cc} } +func (c *networkPolicyDiscoveryServiceClient) DeltaNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyDiscoveryService_ServiceDesc.Streams[0], NetworkPolicyDiscoveryService_DeltaNetworkPolicies_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyDiscoveryService_DeltaNetworkPoliciesClient = grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + func (c *networkPolicyDiscoveryServiceClient) StreamNetworkPolicies(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) { cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) - stream, err := c.cc.NewStream(ctx, &NetworkPolicyDiscoveryService_ServiceDesc.Streams[0], NetworkPolicyDiscoveryService_StreamNetworkPolicies_FullMethodName, cOpts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyDiscoveryService_ServiceDesc.Streams[1], NetworkPolicyDiscoveryService_StreamNetworkPolicies_FullMethodName, cOpts...) if err != nil { return nil, err } @@ -70,7 +86,9 @@ func (c *networkPolicyDiscoveryServiceClient) FetchNetworkPolicies(ctx context.C // for forward compatibility. // // Each resource name is a network policy identifier. +// Deprecated: This service will be removed when Cilium 1.20 is the oldest supported release. type NetworkPolicyDiscoveryServiceServer interface { + DeltaNetworkPolicies(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error StreamNetworkPolicies(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error FetchNetworkPolicies(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) mustEmbedUnimplementedNetworkPolicyDiscoveryServiceServer() @@ -83,6 +101,9 @@ type NetworkPolicyDiscoveryServiceServer interface { // pointer dereference when methods are called. type UnimplementedNetworkPolicyDiscoveryServiceServer struct{} +func (UnimplementedNetworkPolicyDiscoveryServiceServer) DeltaNetworkPolicies(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method DeltaNetworkPolicies not implemented") +} func (UnimplementedNetworkPolicyDiscoveryServiceServer) StreamNetworkPolicies(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error { return status.Error(codes.Unimplemented, "method StreamNetworkPolicies not implemented") } @@ -111,6 +132,13 @@ func RegisterNetworkPolicyDiscoveryServiceServer(s grpc.ServiceRegistrar, srv Ne s.RegisterService(&NetworkPolicyDiscoveryService_ServiceDesc, srv) } +func _NetworkPolicyDiscoveryService_DeltaNetworkPolicies_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyDiscoveryServiceServer).DeltaNetworkPolicies(&grpc.GenericServerStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyDiscoveryService_DeltaNetworkPoliciesServer = grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + func _NetworkPolicyDiscoveryService_StreamNetworkPolicies_Handler(srv interface{}, stream grpc.ServerStream) error { return srv.(NetworkPolicyDiscoveryServiceServer).StreamNetworkPolicies(&grpc.GenericServerStream[v3.DiscoveryRequest, v3.DiscoveryResponse]{ServerStream: stream}) } @@ -149,6 +177,12 @@ var NetworkPolicyDiscoveryService_ServiceDesc = grpc.ServiceDesc{ }, }, Streams: []grpc.StreamDesc{ + { + StreamName: "DeltaNetworkPolicies", + Handler: _NetworkPolicyDiscoveryService_DeltaNetworkPolicies_Handler, + ServerStreams: true, + ClientStreams: true, + }, { StreamName: "StreamNetworkPolicies", Handler: _NetworkPolicyDiscoveryService_StreamNetworkPolicies_Handler, @@ -158,3 +192,136 @@ var NetworkPolicyDiscoveryService_ServiceDesc = grpc.ServiceDesc{ }, Metadata: "cilium/api/npds.proto", } + +const ( + NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResources_FullMethodName = "/cilium.NetworkPolicyResourceDiscoveryService/StreamNetworkPolicyResources" + NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_FullMethodName = "/cilium.NetworkPolicyResourceDiscoveryService/DeltaNetworkPolicyResources" +) + +// NetworkPolicyResourceDiscoveryServiceClient is the client API for NetworkPolicyResourceDiscoveryService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Policy and selector resource names are exact-match identifiers in NPRDS. +type NetworkPolicyResourceDiscoveryServiceClient interface { + StreamNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) + DeltaNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) +} + +type networkPolicyResourceDiscoveryServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewNetworkPolicyResourceDiscoveryServiceClient(cc grpc.ClientConnInterface) NetworkPolicyResourceDiscoveryServiceClient { + return &networkPolicyResourceDiscoveryServiceClient{cc} +} + +func (c *networkPolicyResourceDiscoveryServiceClient) StreamNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyResourceDiscoveryService_ServiceDesc.Streams[0], NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResources_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DiscoveryRequest, v3.DiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResourcesClient = grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse] + +func (c *networkPolicyResourceDiscoveryServiceClient) DeltaNetworkPolicyResources(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyResourceDiscoveryService_ServiceDesc.Streams[1], NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResourcesClient = grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + +// NetworkPolicyResourceDiscoveryServiceServer is the server API for NetworkPolicyResourceDiscoveryService service. +// All implementations must embed UnimplementedNetworkPolicyResourceDiscoveryServiceServer +// for forward compatibility. +// +// Policy and selector resource names are exact-match identifiers in NPRDS. +type NetworkPolicyResourceDiscoveryServiceServer interface { + StreamNetworkPolicyResources(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error + DeltaNetworkPolicyResources(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error + mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() +} + +// UnimplementedNetworkPolicyResourceDiscoveryServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedNetworkPolicyResourceDiscoveryServiceServer struct{} + +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) StreamNetworkPolicyResources(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method StreamNetworkPolicyResources not implemented") +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) DeltaNetworkPolicyResources(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method DeltaNetworkPolicyResources not implemented") +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() { +} +func (UnimplementedNetworkPolicyResourceDiscoveryServiceServer) testEmbeddedByValue() {} + +// UnsafeNetworkPolicyResourceDiscoveryServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to NetworkPolicyResourceDiscoveryServiceServer will +// result in compilation errors. +type UnsafeNetworkPolicyResourceDiscoveryServiceServer interface { + mustEmbedUnimplementedNetworkPolicyResourceDiscoveryServiceServer() +} + +func RegisterNetworkPolicyResourceDiscoveryServiceServer(s grpc.ServiceRegistrar, srv NetworkPolicyResourceDiscoveryServiceServer) { + // If the following call panics, it indicates UnimplementedNetworkPolicyResourceDiscoveryServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&NetworkPolicyResourceDiscoveryService_ServiceDesc, srv) +} + +func _NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResources_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyResourceDiscoveryServiceServer).StreamNetworkPolicyResources(&grpc.GenericServerStream[v3.DiscoveryRequest, v3.DiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResourcesServer = grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse] + +func _NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyResourceDiscoveryServiceServer).DeltaNetworkPolicyResources(&grpc.GenericServerStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResourcesServer = grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + +// NetworkPolicyResourceDiscoveryService_ServiceDesc is the grpc.ServiceDesc for NetworkPolicyResourceDiscoveryService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var NetworkPolicyResourceDiscoveryService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "cilium.NetworkPolicyResourceDiscoveryService", + HandlerType: (*NetworkPolicyResourceDiscoveryServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "StreamNetworkPolicyResources", + Handler: _NetworkPolicyResourceDiscoveryService_StreamNetworkPolicyResources_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "DeltaNetworkPolicyResources", + Handler: _NetworkPolicyResourceDiscoveryService_DeltaNetworkPolicyResources_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "cilium/api/npds.proto", +} diff --git a/go/cilium/api/nphds.pb.go b/go/cilium/api/nphds.pb.go index f4a2b2dfe..c05898ac0 100644 --- a/go/cilium/api/nphds.pb.go +++ b/go/cilium/api/nphds.pb.go @@ -90,10 +90,11 @@ const file_cilium_api_nphds_proto_rawDesc = "" + "\x16cilium/api/nphds.proto\x12\x06cilium\x1a*envoy/service/discovery/v3/discovery.proto\x1a\x1cgoogle/api/annotations.proto\x1a envoy/annotations/resource.proto\x1a\x17validate/validate.proto\"c\n" + "\x12NetworkPolicyHosts\x12\x16\n" + "\x06policy\x18\x01 \x01(\x04R\x06policy\x125\n" + - "\x0ehost_addresses\x18\x02 \x03(\tB\x0e\xfaB\v\x92\x01\b\x18\x01\"\x04r\x02\x10\x01R\rhostAddresses2\xee\x02\n" + + "\x0ehost_addresses\x18\x02 \x03(\tB\x0e\xfaB\v\x92\x01\b\x18\x01\"\x04r\x02\x10\x01R\rhostAddresses2\xf7\x03\n" + "\"NetworkPolicyHostsDiscoveryService\x12}\n" + "\x18StreamNetworkPolicyHosts\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"\x00(\x010\x01\x12\xa5\x01\n" + - "\x17FetchNetworkPolicyHosts\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"-\x82\xd3\xe4\x93\x02':\x01*\"\"/v2/discovery:network_policy_hosts\x1a!\x8a\xa4\x96\xf3\a\x1b\n" + + "\x17FetchNetworkPolicyHosts\x12,.envoy.service.discovery.v3.DiscoveryRequest\x1a-.envoy.service.discovery.v3.DiscoveryResponse\"-\x82\xd3\xe4\x93\x02':\x01*\"\"/v2/discovery:network_policy_hosts\x12\x86\x01\n" + + "\x17DeltaNetworkPolicyHosts\x121.envoy.service.discovery.v3.DeltaDiscoveryRequest\x1a2.envoy.service.discovery.v3.DeltaDiscoveryResponse\"\x00(\x010\x01\x1a!\x8a\xa4\x96\xf3\a\x1b\n" + "\x19cilium.NetworkPolicyHostsB.Z,github.com/cilium/proxy/go/cilium/api;ciliumb\x06proto3" var ( @@ -110,17 +111,21 @@ func file_cilium_api_nphds_proto_rawDescGZIP() []byte { var file_cilium_api_nphds_proto_msgTypes = make([]protoimpl.MessageInfo, 1) var file_cilium_api_nphds_proto_goTypes = []any{ - (*NetworkPolicyHosts)(nil), // 0: cilium.NetworkPolicyHosts - (*v3.DiscoveryRequest)(nil), // 1: envoy.service.discovery.v3.DiscoveryRequest - (*v3.DiscoveryResponse)(nil), // 2: envoy.service.discovery.v3.DiscoveryResponse + (*NetworkPolicyHosts)(nil), // 0: cilium.NetworkPolicyHosts + (*v3.DiscoveryRequest)(nil), // 1: envoy.service.discovery.v3.DiscoveryRequest + (*v3.DeltaDiscoveryRequest)(nil), // 2: envoy.service.discovery.v3.DeltaDiscoveryRequest + (*v3.DiscoveryResponse)(nil), // 3: envoy.service.discovery.v3.DiscoveryResponse + (*v3.DeltaDiscoveryResponse)(nil), // 4: envoy.service.discovery.v3.DeltaDiscoveryResponse } var file_cilium_api_nphds_proto_depIdxs = []int32{ 1, // 0: cilium.NetworkPolicyHostsDiscoveryService.StreamNetworkPolicyHosts:input_type -> envoy.service.discovery.v3.DiscoveryRequest 1, // 1: cilium.NetworkPolicyHostsDiscoveryService.FetchNetworkPolicyHosts:input_type -> envoy.service.discovery.v3.DiscoveryRequest - 2, // 2: cilium.NetworkPolicyHostsDiscoveryService.StreamNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 2, // 3: cilium.NetworkPolicyHostsDiscoveryService.FetchNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse - 2, // [2:4] is the sub-list for method output_type - 0, // [0:2] is the sub-list for method input_type + 2, // 2: cilium.NetworkPolicyHostsDiscoveryService.DeltaNetworkPolicyHosts:input_type -> envoy.service.discovery.v3.DeltaDiscoveryRequest + 3, // 3: cilium.NetworkPolicyHostsDiscoveryService.StreamNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 3, // 4: cilium.NetworkPolicyHostsDiscoveryService.FetchNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DiscoveryResponse + 4, // 5: cilium.NetworkPolicyHostsDiscoveryService.DeltaNetworkPolicyHosts:output_type -> envoy.service.discovery.v3.DeltaDiscoveryResponse + 3, // [3:6] is the sub-list for method output_type + 0, // [0:3] is the sub-list for method input_type 0, // [0:0] is the sub-list for extension type_name 0, // [0:0] is the sub-list for extension extendee 0, // [0:0] is the sub-list for field type_name diff --git a/go/cilium/api/nphds_grpc.pb.go b/go/cilium/api/nphds_grpc.pb.go index 144a3a815..63cd6c3fa 100644 --- a/go/cilium/api/nphds_grpc.pb.go +++ b/go/cilium/api/nphds_grpc.pb.go @@ -22,6 +22,7 @@ const _ = grpc.SupportPackageIsVersion9 const ( NetworkPolicyHostsDiscoveryService_StreamNetworkPolicyHosts_FullMethodName = "/cilium.NetworkPolicyHostsDiscoveryService/StreamNetworkPolicyHosts" NetworkPolicyHostsDiscoveryService_FetchNetworkPolicyHosts_FullMethodName = "/cilium.NetworkPolicyHostsDiscoveryService/FetchNetworkPolicyHosts" + NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_FullMethodName = "/cilium.NetworkPolicyHostsDiscoveryService/DeltaNetworkPolicyHosts" ) // NetworkPolicyHostsDiscoveryServiceClient is the client API for NetworkPolicyHostsDiscoveryService service. @@ -32,6 +33,7 @@ const ( type NetworkPolicyHostsDiscoveryServiceClient interface { StreamNetworkPolicyHosts(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DiscoveryRequest, v3.DiscoveryResponse], error) FetchNetworkPolicyHosts(ctx context.Context, in *v3.DiscoveryRequest, opts ...grpc.CallOption) (*v3.DiscoveryResponse, error) + DeltaNetworkPolicyHosts(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) } type networkPolicyHostsDiscoveryServiceClient struct { @@ -65,6 +67,19 @@ func (c *networkPolicyHostsDiscoveryServiceClient) FetchNetworkPolicyHosts(ctx c return out, nil } +func (c *networkPolicyHostsDiscoveryServiceClient) DeltaNetworkPolicyHosts(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &NetworkPolicyHostsDiscoveryService_ServiceDesc.Streams[1], NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHostsClient = grpc.BidiStreamingClient[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + // NetworkPolicyHostsDiscoveryServiceServer is the server API for NetworkPolicyHostsDiscoveryService service. // All implementations must embed UnimplementedNetworkPolicyHostsDiscoveryServiceServer // for forward compatibility. @@ -73,6 +88,7 @@ func (c *networkPolicyHostsDiscoveryServiceClient) FetchNetworkPolicyHosts(ctx c type NetworkPolicyHostsDiscoveryServiceServer interface { StreamNetworkPolicyHosts(grpc.BidiStreamingServer[v3.DiscoveryRequest, v3.DiscoveryResponse]) error FetchNetworkPolicyHosts(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) + DeltaNetworkPolicyHosts(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error mustEmbedUnimplementedNetworkPolicyHostsDiscoveryServiceServer() } @@ -89,6 +105,9 @@ func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) StreamNetworkPolicy func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) FetchNetworkPolicyHosts(context.Context, *v3.DiscoveryRequest) (*v3.DiscoveryResponse, error) { return nil, status.Error(codes.Unimplemented, "method FetchNetworkPolicyHosts not implemented") } +func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) DeltaNetworkPolicyHosts(grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]) error { + return status.Error(codes.Unimplemented, "method DeltaNetworkPolicyHosts not implemented") +} func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) mustEmbedUnimplementedNetworkPolicyHostsDiscoveryServiceServer() { } func (UnimplementedNetworkPolicyHostsDiscoveryServiceServer) testEmbeddedByValue() {} @@ -136,6 +155,13 @@ func _NetworkPolicyHostsDiscoveryService_FetchNetworkPolicyHosts_Handler(srv int return interceptor(ctx, in, info, handler) } +func _NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(NetworkPolicyHostsDiscoveryServiceServer).DeltaNetworkPolicyHosts(&grpc.GenericServerStream[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHostsServer = grpc.BidiStreamingServer[v3.DeltaDiscoveryRequest, v3.DeltaDiscoveryResponse] + // NetworkPolicyHostsDiscoveryService_ServiceDesc is the grpc.ServiceDesc for NetworkPolicyHostsDiscoveryService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -155,6 +181,12 @@ var NetworkPolicyHostsDiscoveryService_ServiceDesc = grpc.ServiceDesc{ ServerStreams: true, ClientStreams: true, }, + { + StreamName: "DeltaNetworkPolicyHosts", + Handler: _NetworkPolicyHostsDiscoveryService_DeltaNetworkPolicyHosts_Handler, + ServerStreams: true, + ClientStreams: true, + }, }, Metadata: "cilium/api/nphds.proto", } diff --git a/patches/0007-config-add-grpc-mux-stream-event-callback.patch b/patches/0007-config-add-grpc-mux-stream-event-callback.patch new file mode 100644 index 000000000..e92fff4dd --- /dev/null +++ b/patches/0007-config-add-grpc-mux-stream-event-callback.patch @@ -0,0 +1,448 @@ +diff --git a/envoy/config/BUILD b/envoy/config/BUILD +--- a/envoy/config/BUILD ++++ b/envoy/config/BUILD +@@ -66,6 +66,7 @@ envoy_cc_library( + name = "grpc_mux_interface", + hdrs = ["grpc_mux.h"], + deps = [ ++ "//envoy/common:callback", + ":eds_resources_cache_interface", + ":subscription_interface", + "//envoy/stats:stats_macros", +diff --git a/envoy/config/grpc_mux.h b/envoy/config/grpc_mux.h +--- a/envoy/config/grpc_mux.h ++++ b/envoy/config/grpc_mux.h +@@ -1,8 +1,10 @@ + #pragma once + ++#include + #include + + #include "envoy/common/backoff_strategy.h" ++#include "envoy/common/callback.h" + #include "envoy/common/exception.h" + #include "envoy/common/pure.h" + #include "envoy/config/custom_config_validators.h" +@@ -17,6 +19,11 @@ namespace Envoy { + namespace Config { + + using ScopedResume = std::unique_ptr; ++ ++enum class GrpcMuxStreamEvent { Established, Closed }; ++ ++using GrpcMuxStreamEventCallback = std::function; ++ + /** + * All control plane related stats. @see stats_macros.h + */ +@@ -61,6 +67,18 @@ public: + */ + virtual void start() PURE; + ++ /** ++ * Add a callback to observe gRPC mux stream lifecycle events. ++ * @return a handle that unregisters the callback when destroyed. ++ */ ++ virtual Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) PURE; ++ ++ /** ++ * @return true if the mux currently has an established gRPC stream. ++ */ ++ virtual bool grpcStreamConnected() const PURE; ++ + /** + * Pause discovery requests for a given API type. This is useful when we're processing an update + * for LDS or CDS and don't want a flood of updates for RDS or EDS respectively. Discovery +diff --git a/source/common/config/BUILD b/source/common/config/BUILD +--- a/source/common/config/BUILD ++++ b/source/common/config/BUILD +@@ -86,10 +86,19 @@ envoy_cc_library( + ], + ) +- ++envoy_cc_library( ++ name = "grpc_mux_stream_event_tracker_lib", ++ hdrs = ["grpc_mux_stream_event_tracker.h"], ++ deps = [ ++ "//envoy/config:grpc_mux_interface", ++ "//source/common/common:callback_impl_lib", ++ ], ++) ++ + envoy_cc_library( + name = "null_grpc_mux_lib", + hdrs = ["null_grpc_mux_impl.h"], + deps = [ ++ ":grpc_mux_stream_event_tracker_lib", + "//envoy/config:grpc_mux_interface", + ], + ) +diff --git a/source/common/config/null_grpc_mux_impl.h b/source/common/config/null_grpc_mux_impl.h +--- a/source/common/config/null_grpc_mux_impl.h ++++ b/source/common/config/null_grpc_mux_impl.h +@@ -1,6 +1,7 @@ + #pragma once + + #include "envoy/config/grpc_mux.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + + namespace Envoy { + namespace Config { +@@ -11,5 +12,11 @@ class NullGrpcMuxImpl : public GrpcMux, + public: + void start() override {} ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return false; } + ScopedResume pause(const std::string&) override { + return std::make_unique([] {}); + } +@@ -40,6 +48,9 @@ public: + void onEstablishmentFailure(bool) override {} + void onDiscoveryResponse(std::unique_ptr&&, + ControlPlaneStats&) override {} ++ ++private: ++ GrpcMuxStreamEventTracker stream_event_tracker_; + }; + + } // namespace Config +diff --git a/source/extensions/config_subscription/grpc/BUILD b/source/extensions/config_subscription/grpc/BUILD +--- a/source/extensions/config_subscription/grpc/BUILD ++++ b/source/extensions/config_subscription/grpc/BUILD +@@ -33,6 +33,7 @@ envoy_cc_extension( + ":eds_resources_cache_lib", + ":grpc_mux_context_lib", + ":grpc_mux_failover_lib", ++ "//source/common/config:grpc_mux_stream_event_tracker_lib", + ":grpc_stream_lib", + ":xds_source_id_lib", + "//envoy/config:custom_config_validators_interface", +@@ -65,6 +66,7 @@ envoy_cc_library( + ":grpc_mux_context_lib", + ":grpc_mux_failover_lib", + ":grpc_stream_lib", ++ "//source/common/config:grpc_mux_stream_event_tracker_lib", + ":pausable_ack_queue_lib", + ":watch_map_lib", + "//envoy/config:custom_config_validators_interface", +diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +--- a/source/extensions/config_subscription/grpc/grpc_mux_impl.cc ++++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.cc +@@ -569,6 +569,7 @@ void GrpcMuxImpl::onStreamEstablished() { + for (const auto& type_url : subscriptions_) { + queueDiscoveryRequest(type_url); + } ++ stream_event_tracker_.onStreamEstablished(); + } + + void GrpcMuxImpl::onEstablishmentFailure(bool) { +@@ -589,6 +590,7 @@ void GrpcMuxImpl::onEstablishmentFailure(bool) { + api_state.second->previously_fetched_data_ = true; + } + } ++ stream_event_tracker_.onStreamClosed(); + } + + void GrpcMuxImpl::queueDiscoveryRequest(absl::string_view queue_item) { +diff --git a/source/extensions/config_subscription/grpc/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/grpc_mux_impl.h +--- a/source/extensions/config_subscription/grpc/grpc_mux_impl.h ++++ b/source/extensions/config_subscription/grpc/grpc_mux_impl.h +@@ -3,6 +3,7 @@ + #include + #include + #include ++#include + + #include "envoy/common/random_generator.h" + #include "envoy/common/time.h" +@@ -29,5 +30,6 @@ + #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + + #include "absl/container/node_hash_map.h" + #include "xds/core/v3/resource_name.pb.h" +@@ -57,6 +59,13 @@ public: + + void start() override; + ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + // GrpcMux + ScopedResume pause(const std::string& type_url) override; + ScopedResume pause(const std::vector type_urls) override; +@@ -290,6 +299,7 @@ private: + const bool skip_subsequent_node_; + CustomConfigValidatorsPtr config_validators_; + XdsConfigTrackerOptRef xds_config_tracker_; ++ GrpcMuxStreamEventTracker stream_event_tracker_; + XdsResourcesDelegateOptRef xds_resources_delegate_; + EdsResourcesCachePtr eds_resources_cache_; + const std::string target_xds_authority_; +diff --git a/source/common/config/grpc_mux_stream_event_tracker.h b/source/common/config/grpc_mux_stream_event_tracker.h +new file mode 100644 +--- /dev/null ++++ b/source/common/config/grpc_mux_stream_event_tracker.h +@@ -0,0 +1,36 @@ ++#pragma once ++ ++#include ++ ++#include "envoy/config/grpc_mux.h" ++ ++#include "source/common/common/callback_impl.h" ++ ++namespace Envoy { ++namespace Config { ++ ++class GrpcMuxStreamEventTracker { ++public: ++ Common::CallbackHandlePtr addStreamEventCallback(GrpcMuxStreamEventCallback callback) { ++ return callbacks_.add(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const { return grpc_stream_connected_; } ++ ++ void onStreamEstablished() { ++ grpc_stream_connected_ = true; ++ callbacks_.runCallbacks(GrpcMuxStreamEvent::Established); ++ } ++ ++ void onStreamClosed() { ++ grpc_stream_connected_ = false; ++ callbacks_.runCallbacks(GrpcMuxStreamEvent::Closed); ++ } ++ ++private: ++ bool grpc_stream_connected_{false}; ++ Common::CallbackManager callbacks_; ++}; ++ ++} // namespace Config ++} // namespace Envoy +diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc +--- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc ++++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.cc +@@ -194,6 +194,7 @@ void NewGrpcMuxImpl::onStreamEstablished() { + } + pausable_ack_queue_.clear(); + trySendDiscoveryRequests(); ++ stream_event_tracker_.onStreamEstablished(); + } + + void NewGrpcMuxImpl::onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) { +@@ -215,6 +216,7 @@ void NewGrpcMuxImpl::onEstablishmentFailure(bool next_attempt_may_send_initial_resource_version) { + } + } while (all_subscribed.size() != subscriptions_.size()); + should_send_initial_resource_versions_ = next_attempt_may_send_initial_resource_version; ++ stream_event_tracker_.onStreamClosed(); + } + + void NewGrpcMuxImpl::onWriteable() { trySendDiscoveryRequests(); } +diff --git a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +--- a/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h ++++ b/source/extensions/config_subscription/grpc/new_grpc_mux_impl.h +@@ -1,6 +1,7 @@ + #pragma once + + #include ++#include + + #include "envoy/common/random_generator.h" + #include "envoy/common/token_bucket.h" +@@ -20,6 +21,7 @@ + #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" + #include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + #include "source/extensions/config_subscription/grpc/watch_map.h" + + namespace Envoy { +@@ -50,6 +52,13 @@ public: + static void shutdownAll(); + + void shutdown() { shutdown_ = true; } ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + + GrpcMuxWatchPtr addWatch(const std::string& type_url, + const absl::flat_hash_set& resources, +@@ -217,6 +226,7 @@ private: + + const LocalInfo::LocalInfo& local_info_; + CustomConfigValidatorsPtr config_validators_; ++ GrpcMuxStreamEventTracker stream_event_tracker_; + Common::CallbackHandlePtr dynamic_update_callback_handle_; + XdsConfigTrackerOptRef xds_config_tracker_; + EdsResourcesCachePtr eds_resources_cache_; +diff --git a/source/extensions/config_subscription/grpc/xds_mux/BUILD b/source/extensions/config_subscription/grpc/xds_mux/BUILD +--- a/source/extensions/config_subscription/grpc/xds_mux/BUILD ++++ b/source/extensions/config_subscription/grpc/xds_mux/BUILD +@@ -72,6 +72,7 @@ envoy_cc_extension( + "//source/extensions/config_subscription/grpc:eds_resources_cache_lib", + "//source/extensions/config_subscription/grpc:grpc_mux_context_lib", + "//source/extensions/config_subscription/grpc:grpc_mux_failover_lib", ++ "//source/common/config:grpc_mux_stream_event_tracker_lib", + "//source/extensions/config_subscription/grpc:grpc_stream_lib", + "//source/extensions/config_subscription/grpc:pausable_ack_queue_lib", + "//source/extensions/config_subscription/grpc:watch_map_lib", +diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +--- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc ++++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.cc +@@ -322,6 +322,7 @@ void GrpcMuxImpl::handleEstablishedStream() { + maybeUpdateQueueSizeStat(0); + pausable_ack_queue_.clear(); + trySendDiscoveryRequests(); ++ stream_event_tracker_.onStreamEstablished(); + } + + template +@@ -346,6 +347,7 @@ void GrpcMuxImpl::handleStreamEstablishmentFailure( + } + } while (all_subscribed.size() != subscriptions_.size()); + should_send_initial_resource_versions_ = next_attempt_may_send_initial_resource_version; ++ stream_event_tracker_.onStreamClosed(); + } + + template +diff --git a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +--- a/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h ++++ b/source/extensions/config_subscription/grpc/xds_mux/grpc_mux_impl.h +@@ -3,6 +3,7 @@ + #include + #include + #include ++#include + + #include "envoy/common/random_generator.h" + #include "envoy/common/time.h" +@@ -23,6 +24,7 @@ + #include "source/common/grpc/common.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_context.h" + #include "source/extensions/config_subscription/grpc/grpc_mux_failover.h" ++#include "source/common/config/grpc_mux_stream_event_tracker.h" + #include "source/extensions/config_subscription/grpc/pausable_ack_queue.h" + #include "source/extensions/config_subscription/grpc/watch_map.h" + #include "source/extensions/config_subscription/grpc/xds_mux/delta_subscription_state.h" +@@ -82,6 +84,13 @@ public: + SubscriptionCallbacks& callbacks, + OpaqueResourceDecoderSharedPtr resource_decoder, + const SubscriptionOptions& options) override; ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + void updateWatch(const std::string& type_url, Watch* watch, + const absl::flat_hash_set& resources, + const SubscriptionOptions& options); +@@ -239,6 +248,7 @@ private: + // this one is up to GrpcMux. + const LocalInfo::LocalInfo& local_info_; + Common::CallbackHandlePtr dynamic_update_callback_handle_; ++ GrpcMuxStreamEventTracker stream_event_tracker_; + CustomConfigValidatorsPtr config_validators_; + XdsConfigTrackerOptRef xds_config_tracker_; + XdsResourcesDelegateOptRef xds_resources_delegate_; +@@ -291,6 +301,13 @@ private: + class NullGrpcMuxImpl : public GrpcMux { + public: + void start() override {} ++ Common::CallbackHandlePtr ++ addStreamEventCallback(GrpcMuxStreamEventCallback callback) override { ++ return stream_event_tracker_.addStreamEventCallback(std::move(callback)); ++ } ++ ++ bool grpcStreamConnected() const override { return stream_event_tracker_.grpcStreamConnected(); } ++ + + ScopedResume pause(const std::string&) override { + return std::make_unique([]() {}); +@@ -313,6 +330,9 @@ public: + void requestOnDemandUpdate(const std::string&, const absl::flat_hash_set&) override { + ENVOY_BUG(false, "unexpected request for on demand update"); + } + + EdsResourcesCacheOptRef edsResourcesCache() override { return {}; } ++ ++private: ++ GrpcMuxStreamEventTracker stream_event_tracker_; + }; +diff --git a/test/common/config/grpc_subscription_impl_test.cc b/test/common/config/grpc_subscription_impl_test.cc +--- a/test/common/config/grpc_subscription_impl_test.cc ++++ b/test/common/config/grpc_subscription_impl_test.cc +@@ -64,6 +64,42 @@ TEST_P(GrpcSubscriptionImplTest, RemoteStreamClose) { + timer_->invokeCallback(); + EXPECT_TRUE(statsAre(2, 0, 0, 1, 0, 0, 0, "")); + } + ++TEST_P(GrpcSubscriptionImplTest, StreamEventCallbacks) { ++ uint32_t established = 0; ++ uint32_t closed = 0; ++ auto callback_handle = mux_->addStreamEventCallback([&](GrpcMuxStreamEvent event) { ++ if (event == GrpcMuxStreamEvent::Established) { ++ ++established; ++ } else { ++ ++closed; ++ } ++ }); ++ ++ EXPECT_FALSE(mux_->grpcStreamConnected()); ++ startSubscription({"cluster0", "cluster1"}); ++ EXPECT_TRUE(mux_->grpcStreamConnected()); ++ EXPECT_EQ(1, established); ++ EXPECT_EQ(0, closed); ++ ++ EXPECT_CALL(callbacks_, ++ onConfigUpdateFailed(Envoy::Config::ConfigUpdateFailureReason::ConnectionFailure, _)) ++ .Times(0); ++ EXPECT_CALL(*timer_, enableTimer(_, _)); ++ EXPECT_CALL(random_, random()); ++ onRemoteClose(); ++ EXPECT_FALSE(mux_->grpcStreamConnected()); ++ EXPECT_EQ(1, established); ++ EXPECT_EQ(1, closed); ++ ++ callback_handle.reset(); ++ EXPECT_CALL(*async_client_, startRaw(_, _, _, _)).WillOnce(Return(&async_stream_)); ++ expectSendMessage({"cluster0", "cluster1"}, "", true); ++ timer_->invokeCallback(); ++ EXPECT_TRUE(mux_->grpcStreamConnected()); ++ EXPECT_EQ(1, established); ++ EXPECT_EQ(1, closed); ++} ++ + // Validate that When the management server gets multiple requests for the same version, it can + // ignore later ones. This allows the nonce to be used. + TEST_P(GrpcSubscriptionImplTest, RepeatedNonce) { +diff --git a/test/mocks/config/mocks.h b/test/mocks/config/mocks.h +--- a/test/mocks/config/mocks.h ++++ b/test/mocks/config/mocks.h +@@ -116,6 +116,10 @@ public: + MOCK_METHOD(ScopedResume, pause, (const std::string& type_url), (override)); + MOCK_METHOD(ScopedResume, pause, (const std::vector type_urls), (override)); + ++ MOCK_METHOD(Common::CallbackHandlePtr, addStreamEventCallback, ++ (GrpcMuxStreamEventCallback callback), (override)); ++ MOCK_METHOD(bool, grpcStreamConnected, (), (const, override)); ++ + MOCK_METHOD(void, addSubscription, + (const absl::flat_hash_set& resources, const std::string& type_url, + SubscriptionCallbacks& callbacks, SubscriptionStats& stats, diff --git a/tests/BUILD b/tests/BUILD index 1f6db5971..c084fdf34 100644 --- a/tests/BUILD +++ b/tests/BUILD @@ -122,6 +122,15 @@ envoy_cc_test( ], ) +envoy_cc_test( + name = "versioned_test", + srcs = ["versioned_test.cc"], + repository = "@envoy", + deps = [ + "//cilium:versioned_lib", + ], +) + envoy_cc_test( name = "bpf_metadata_config_test", srcs = ["bpf_metadata_config_test.cc"], diff --git a/tests/bpf_metadata.cc b/tests/bpf_metadata.cc index 84bd5e87a..5528694c7 100644 --- a/tests/bpf_metadata.cc +++ b/tests/bpf_metadata.cc @@ -39,10 +39,10 @@ namespace Envoy { std::string host_map_config = "version_info: \"0\""; -std::shared_ptr hostmap{nullptr}; // Keep reference to singleton +std::shared_ptr hostmap{nullptr}; // Keep reference to singleton Network::Address::InstanceConstSharedPtr original_dst_address; -std::shared_ptr npmap{nullptr}; // Keep reference to singleton +std::shared_ptr npmap{nullptr}; // Keep reference to singleton std::string policy_config = "version_info: \"0\""; std::string policy_path = ""; @@ -51,10 +51,10 @@ std::vector> sds_configs{}; namespace Cilium { -std::shared_ptr +std::shared_ptr TestHelper::createHostMap(const std::string& config, Server::Configuration::ListenerFactoryContext& context) { - return context.serverFactoryContext().singletonManager().getTyped( + return context.serverFactoryContext().singletonManager().getTyped( "cilium_host_map_singleton", [&config, &context] { std::string path = TestEnvironment::writeStringToFileForTest("host_map.yaml", config); ENVOY_LOG_MISC(debug, "Loading Cilium Host Map from file \'{}\' instead of using gRPC", @@ -75,11 +75,11 @@ TestHelper::createHostMap(const std::string& config, }); } -std::shared_ptr +std::shared_ptr TestHelper::createPolicyMap(const std::string& config, const std::vector>& secret_configs, Server::Configuration::FactoryContext& context) { - return context.serverFactoryContext().singletonManager().getTyped( + return context.serverFactoryContext().singletonManager().getTyped( "cilium_network_policy_singleton", [&config, &secret_configs, &context] { if (!secret_configs.empty()) { for (const auto& sds_pair : secret_configs) { @@ -112,8 +112,8 @@ TestHelper::createPolicyMap(const std::string& config, policy_path, context.serverFactoryContext().api())); Envoy::Config::SubscriptionStats stats = Envoy::Config::Utility::generateStats(context.scope()); - auto map = - std::make_shared(context, Cilium::CILIUM_XDS_API_CONFIG); + auto map = std::make_shared(context, false, + Cilium::CILIUM_XDS_API_CONFIG); auto subscription = std::make_unique( context.serverFactoryContext().mainThreadDispatcher(), Envoy::Config::makePathConfigSource(policy_path), map->subscriptionCallbacksForTest(), diff --git a/tests/bpf_metadata.h b/tests/bpf_metadata.h index ecdf80e22..995a805ee 100644 --- a/tests/bpf_metadata.h +++ b/tests/bpf_metadata.h @@ -18,10 +18,10 @@ namespace Envoy { extern std::string host_map_config; -extern std::shared_ptr hostmap; +extern std::shared_ptr hostmap; extern Network::Address::InstanceConstSharedPtr original_dst_address; -extern std::shared_ptr npmap; +extern std::shared_ptr npmap; extern std::string policy_config; extern std::string policy_path; @@ -30,9 +30,9 @@ extern std::vector> sds_configs; namespace Cilium { struct TestHelper { - static std::shared_ptr + static std::shared_ptr createHostMap(const std::string& config, Server::Configuration::ListenerFactoryContext&); - static std::shared_ptr + static std::shared_ptr createPolicyMap(const std::string& config, const std::vector>& secret_configs, Server::Configuration::FactoryContext&); diff --git a/tests/bpf_metadata_integration_test.cc b/tests/bpf_metadata_integration_test.cc index 271ba7123..885bca028 100644 --- a/tests/bpf_metadata_integration_test.cc +++ b/tests/bpf_metadata_integration_test.cc @@ -1,6 +1,10 @@ #include +#include +#include +#include #include +#include #include #include "envoy/config/bootstrap/v3/bootstrap.pb.h" @@ -9,10 +13,11 @@ #include "envoy/config/listener/v3/listener.pb.h" #include "envoy/grpc/status.h" #include "envoy/http/codec.h" -#include "envoy/network/address.h" #include "envoy/service/discovery/v3/discovery.pb.h" #include "source/common/common/assert.h" +#include "source/common/common/logger.h" +#include "source/common/network/utility.h" #include "source/common/protobuf/utility.h" #include "test/common/grpc/grpc_client_integration.h" @@ -22,18 +27,29 @@ #include "test/test_common/resources.h" #include "test/test_common/utility.h" +#include "absl/strings/string_view.h" +#include "absl/synchronization/mutex.h" #include "cilium/api/bpf_metadata.pb.h" #include "cilium/api/npds.pb.h" #include "cilium/api/nphds.pb.h" +#include "cilium/host_map.h" +#include "cilium/network_policy.h" +#include "cilium/policy_id.h" #include "gtest/gtest.h" namespace Envoy { namespace { const std::string NetworkPolicyTypeUrl = "type.googleapis.com/cilium.NetworkPolicy"; - +const std::string NetworkPolicyResourceTypeUrl = "type.googleapis.com/cilium.NetworkPolicyResource"; const std::string NetworkPolicyHostsTypeUrl = "type.googleapis.com/cilium.NetworkPolicyHosts"; +struct NetworkPolicyResourceConfig { + std::string name; + std::string version; + std::string yaml; +}; + const std::string policy1 = R"EOF( endpoint_ips: - '10.1.1.1' @@ -56,6 +72,88 @@ const std::string policy2 = R"EOF( - remote_policies: [ 111 ] )EOF"; +const std::string invalid_policy = R"EOF( + endpoint_id: 8192 +)EOF"; + +const NetworkPolicyResourceConfig selector1_resource = {"selector-1", "1", R"EOF( + selector: + remote_identities: [ 43 ] +)EOF"}; + +const NetworkPolicyResourceConfig selector2_resource = {"selector-2", "1", R"EOF( + selector: + remote_identities: [ 44 ] +)EOF"}; + +const NetworkPolicyResourceConfig selector3_resource = {"selector-3", "1", R"EOF( + selector: + remote_identities: [ 45 ] +)EOF"}; + +const NetworkPolicyResourceConfig policy42 = {"policy-42", "1", R"EOF( + endpoint_ips: + - '10.1.2.3' + endpoint_id: 42 + egress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 222 ] +)EOF"}; + +const NetworkPolicyResourceConfig policy42_resource = {"policy-42", "1", R"EOF( + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy43 = {"policy-43", "1", R"EOF( + endpoint_ips: + - '10.2.3.4' + endpoint_id: 43 + ingress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 111 ] +)EOF"}; + +const NetworkPolicyResourceConfig policy43_resource = {"policy-43", "1", R"EOF( + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy42_new_stream = {"policy-42", "2", R"EOF( + endpoint_ips: + - '10.1.2.3' + endpoint_id: 42 + egress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 222 ] +)EOF"}; + +const NetworkPolicyResourceConfig policy42_new_stream_resource = {"policy-42", "2", R"EOF( + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF"}; + const std::string policy_host1 = R"EOF( policy: 111 host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] @@ -66,6 +164,21 @@ const std::string policy_host2 = R"EOF( host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] )EOF"; +const NetworkPolicyResourceConfig policy_host1_resource = {"111", "1", R"EOF( + policy: 111 + host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy_host2_resource = {"222", "1", R"EOF( + policy: 222 + host_addresses: [ "10.2.2.2", "f00d::2:2:2" ] +)EOF"}; + +const NetworkPolicyResourceConfig policy_host1_new_stream_resource = {"111", "2", R"EOF( + policy: 111 + host_addresses: [ "10.1.1.1", "f00d::1:1:1" ] +)EOF"}; + class BpfMetadataIntegrationTest : public BaseIntegrationTest, public Grpc::GrpcClientIntegrationParamTest { public: @@ -80,30 +193,69 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, cluster: cluster_0 )EOF") { skip_tag_extraction_rule_check_ = true; + use_lds_ = false; // skip built in listener setup, we do it explicitly via xDS + setUpstreamCount(1); // same as default + defer_listener_finalization_ = true; + +#if 0 + for (Logger::Logger& logger : Logger::Registry::loggers()) { + logger.setLevel(spdlog::level::trace); + } +#endif } ~BpfMetadataIntegrationTest() override { resetConnections(); } - void setGrpcServiceHelper(envoy::config::core::v3::GrpcService& grpc_service, - const std::string& cluster_name, - Network::Address::InstanceConstSharedPtr address) { - setGrpcService(grpc_service, cluster_name, address); + void setUpGrpcLds(bool use_ads = true) { + config_helper_.addConfigModifier( + [this, use_ads](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + listener_config_.Swap(bootstrap.mutable_static_resources()->mutable_listeners(0)); + listener_config_.set_name(listener_name_); + bootstrap.mutable_static_resources()->mutable_listeners()->Clear(); + + auto* lds_config_source = bootstrap.mutable_dynamic_resources()->mutable_lds_config(); + lds_config_source->Clear(); + lds_config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + if (use_ads) { + lds_config_source->mutable_ads(); + } else { + setGrpcApiConfigSource(*lds_config_source); + } + }); } - void setUpGrpcLds() { - config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { - listener_config_.Swap(bootstrap.mutable_static_resources()->mutable_listeners(0)); - listener_config_.set_name(listener_name_); - bootstrap.mutable_static_resources()->mutable_listeners()->Clear(); + void setGrpcApiConfigSource(envoy::config::core::v3::ConfigSource& config_source, + envoy::config::core::v3::ApiConfigSource::ApiType api_type = + envoy::config::core::v3::ApiConfigSource::GRPC) { + config_source.set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + auto* api_config_source = config_source.mutable_api_config_source(); + api_config_source->set_set_node_on_first_message_only(true); + api_config_source->set_api_type(api_type); + api_config_source->set_transport_api_version(envoy::config::core::v3::ApiVersion::V3); + api_config_source->add_grpc_services()->mutable_envoy_grpc()->set_cluster_name( + "xds-grpc-cilium"); + } - auto* lds_config_source = bootstrap.mutable_dynamic_resources()->mutable_lds_config(); - lds_config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); - lds_config_source->mutable_ads(); - }); + void setBpfMetadataNpdsConfig(::cilium::BpfMetadata& bpf_config, bool use_ads, + envoy::config::core::v3::ApiConfigSource::ApiType api_type = + envoy::config::core::v3::ApiConfigSource::GRPC, + bool use_nprds = false) { + auto* config_source = bpf_config.mutable_cilium_config_source(); + config_source->Clear(); + if (use_ads) { + config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); + config_source->mutable_ads(); + } else { + setGrpcApiConfigSource(*config_source, api_type); + } + bpf_config.set_policy_type(use_nprds); } // Inject the cilium.bpf_metadata listener filter with config_source into the listener. - void addBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, bool) { + void addBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, bool use_ads, + envoy::config::core::v3::ApiConfigSource::ApiType api_type = + envoy::config::core::v3::ApiConfigSource::GRPC, + bool use_nprds = false) { auto* listener_filter = listener.add_listener_filters(); listener_filter->set_name("cilium.bpf_metadata"); @@ -111,18 +263,30 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, bpf_config.set_is_ingress(false); bpf_config.set_use_nphds(true); - auto* config_source = bpf_config.mutable_cilium_config_source(); - config_source->set_resource_api_version(envoy::config::core::v3::ApiVersion::V3); - config_source->mutable_ads(); + setBpfMetadataNpdsConfig(bpf_config, use_ads, api_type, use_nprds); listener_filter->mutable_typed_config()->PackFrom(bpf_config); } - void initialize() override { - use_lds_ = false; - setUpstreamCount(1); - defer_listener_finalization_ = true; + void updateBpfMetadataListenerFilter(envoy::config::listener::v3::Listener& listener, + envoy::config::core::v3::ApiConfigSource::ApiType api_type, + bool use_nprds = false) { + for (auto& listener_filter : *listener.mutable_listener_filters()) { + if (listener_filter.name() != "cilium.bpf_metadata") { + continue; + } + + ::cilium::BpfMetadata bpf_config; + RELEASE_ASSERT(listener_filter.typed_config().UnpackTo(&bpf_config), + "failed to unpack cilium.bpf_metadata listener filter"); + setBpfMetadataNpdsConfig(bpf_config, /*use_ads=*/false, api_type, use_nprds); + listener_filter.mutable_typed_config()->PackFrom(bpf_config); + return; + } + RELEASE_ASSERT(false, "cilium.bpf_metadata listener filter not found"); + } + void initializeAds() { config_helper_.addConfigModifier([](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { // Add the ADS gRPC cluster. auto* ads_cluster = bootstrap.mutable_static_resources()->add_clusters(); @@ -149,44 +313,140 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, BaseIntegrationTest::initialize(); } + void initializeSotw() { + config_helper_.addConfigModifier([this](envoy::config::bootstrap::v3::Bootstrap& bootstrap) { + auto* xds_cluster = bootstrap.mutable_static_resources()->add_clusters(); + xds_cluster->MergeFrom(bootstrap.static_resources().clusters()[0]); + xds_cluster->set_name("xds-grpc-cilium"); + ConfigHelper::setHttp2(*xds_cluster); + + auto* cds_config = bootstrap.mutable_dynamic_resources()->mutable_cds_config(); + setGrpcApiConfigSource(*cds_config); + }); + + // Must be last modifier — it removes static listeners. + setUpGrpcLds(/*use_ads=*/false); + + BaseIntegrationTest::initialize(); + } + void createUpstreams() override { BaseIntegrationTest::createUpstreams(); - // ADS upstream (fake_upstreams_[1]). + // ADS or SotW upstream (fake_upstreams_[1]). addFakeUpstream(Envoy::Http::CodecType::HTTP2); } - FakeUpstream& getAdsFakeUpstream() const { return *fake_upstreams_[1]; } + FakeUpstream& getFakeUpstream() const { return *fake_upstreams_[1]; } - void createAdsStream() { - AssertionResult result = - getAdsFakeUpstream().waitForHttpConnection(*dispatcher_, ads_connection_); + void createXdsConnection() { + if (xds_connection_ == nullptr) { + AssertionResult result = + getFakeUpstream().waitForHttpConnection(*dispatcher_, xds_connection_); + RELEASE_ASSERT(result, result.message()); + } + } + + void createXdsStream(FakeStreamPtr& stream) { + auto result = xds_connection_->waitForNewStream(*dispatcher_, stream); + RELEASE_ASSERT(result, result.message()); + result = stream->waitForHeadersComplete(); RELEASE_ASSERT(result, result.message()); - auto result2 = ads_connection_->waitForNewStream(*dispatcher_, ads_stream_); - RELEASE_ASSERT(result2, result2.message()); - ads_stream_->startGrpcStream(); + stream->startGrpcStream(); } - void sendCdsResponse(const std::string& version) { + void createAdsStream() { + createXdsConnection(); + createXdsStream(ads_stream_); + } + + void createStreamsUntil(const std::string& response_version, absl::string_view type_url, + bool expect_delta = false) { + createXdsConnection(); + + for (int i = 0; i < 8; i++) { + FakeStreamPtr stream; + createXdsStream(stream); + + std::string request_type_url; + const bool is_delta = + stream->headers().getPathValue().find("/Delta") != absl::string_view::npos; + if (is_delta) { + envoy::service::discovery::v3::DeltaDiscoveryRequest request; + auto result = stream->waitForGrpcMessage(*dispatcher_, request); + RELEASE_ASSERT(result, result.message()); + request_type_url = request.type_url(); + } else { + envoy::service::discovery::v3::DiscoveryRequest request; + auto result = stream->waitForGrpcMessage(*dispatcher_, request); + RELEASE_ASSERT(result, result.message()); + request_type_url = request.type_url(); + } + + if (request_type_url == NetworkPolicyResourceTypeUrl) { + ENVOY_LOG_MISC(info, "GOT NPRDS STREAM"); + nprds_stream_ = std::move(stream); + if (request_type_url == type_url && is_delta == expect_delta) { + return; + } + } else if (request_type_url == NetworkPolicyTypeUrl) { + ENVOY_LOG_MISC(info, "GOT NPDS STREAM"); + npds_stream_ = std::move(stream); + if (request_type_url == type_url && is_delta == expect_delta) { + return; + } + } else if (request_type_url == Envoy::Config::TestTypeUrl::get().Listener) { + ENVOY_LOG_MISC(info, "GOT LDS STREAM"); + lds_stream_ = std::move(stream); + sendLdsResponse(*lds_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, + response_version); + } else if (request_type_url == Envoy::Config::TestTypeUrl::get().Cluster) { + ENVOY_LOG_MISC(info, "GOT CDS STREAM"); + cds_stream_ = std::move(stream); + sendCdsResponse(*cds_stream_, response_version); + } else if (request_type_url == NetworkPolicyHostsTypeUrl) { + ENVOY_LOG_MISC(info, "GOT NPHDS STREAM"); + nphds_stream_ = std::move(stream); + if (!is_delta) { + sendNphdsResponse(*nphds_stream_, response_version); + } + if (type_url == NetworkPolicyHostsTypeUrl && is_delta == expect_delta) { + return; + } + } + } + + RELEASE_ASSERT(false, fmt::format("{} stream was not established", type_url)); + } + + void createSotWStreams(const std::string& response_version) { + createStreamsUntil(response_version, NetworkPolicyTypeUrl); + if (nphds_stream_ == nullptr) { + createStreamsUntil(response_version, NetworkPolicyHostsTypeUrl); + } + } + + void sendCdsResponse(FakeStream& stream, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(Envoy::Config::TestTypeUrl::get().Cluster); - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); } - void sendLdsResponse(const std::vector& listener_configs, + void sendLdsResponse(FakeStream& stream, + const std::vector& listener_configs, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(Envoy::Config::TestTypeUrl::get().Listener); for (const auto& listener_config : listener_configs) { response.add_resources()->PackFrom(listener_config); } - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); } - void sendLdsResponse(const std::vector& listener_configs, + void sendLdsResponse(FakeStream& stream, const std::vector& listener_configs, const std::string& version) { std::vector proto_configs; proto_configs.reserve(listener_configs.size()); @@ -194,26 +454,49 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, proto_configs.emplace_back( TestUtility::parseYaml(listener_blob)); } - sendLdsResponse(proto_configs, version); + sendLdsResponse(stream, proto_configs, version); } - void sendNpdsResponse(const std::string& version) { + void sendNpdsResponse(FakeStream& stream, const std::string& version, + const std::vector& policy_configs = {policy1, policy2}) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(NetworkPolicyTypeUrl); std::vector proto_configs; - proto_configs.emplace_back(TestUtility::parseYaml(policy1)); - proto_configs.emplace_back(TestUtility::parseYaml(policy2)); + proto_configs.reserve(policy_configs.size()); + for (const auto& policy_config : policy_configs) { + proto_configs.emplace_back(TestUtility::parseYaml(policy_config)); + } for (const auto& policy_config : proto_configs) { response.add_resources()->PackFrom(policy_config); } - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); + } + + void sendNprdsResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs = { + selector1_resource, selector2_resource, policy42_resource, + policy43_resource}) { + envoy::service::discovery::v3::DiscoveryResponse response; + response.set_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyResourceTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource resource; + resource.set_name(resource_config.name); + resource.set_version(resource_config.version); + resource.mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + response.add_resources()->PackFrom(resource); + } + stream.sendGrpcMessage(response); } - void sendNphdsResponse(const std::string& version) { + void sendNphdsResponse(FakeStream& stream, const std::string& version) { envoy::service::discovery::v3::DiscoveryResponse response; response.set_version_info(version); + response.set_nonce(version); response.set_type_url(NetworkPolicyHostsTypeUrl); std::vector proto_configs; proto_configs.emplace_back(TestUtility::parseYaml(policy_host1)); @@ -221,24 +504,179 @@ class BpfMetadataIntegrationTest : public BaseIntegrationTest, for (const auto& policy_host_config : proto_configs) { response.add_resources()->PackFrom(policy_host_config); } - ASSERT_NE(nullptr, ads_stream_); - ads_stream_->sendGrpcMessage(response); + stream.sendGrpcMessage(response); + } + + void sendNpdsDeltaResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs, + const std::vector& removed_resources = {}) { + envoy::service::discovery::v3::DeltaDiscoveryResponse response; + response.set_system_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource* resource = response.add_resources(); + resource->set_name(resource_config.name); + resource->set_version(resource_config.version); + resource->mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + } + for (const auto& removed_resource : removed_resources) { + response.add_removed_resources(removed_resource); + } + stream.sendGrpcMessage(response); + } + + void sendNprdsDeltaResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs, + const std::vector& removed_resources = {}) { + envoy::service::discovery::v3::DeltaDiscoveryResponse response; + response.set_system_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyResourceTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource* resource = response.add_resources(); + resource->set_name(resource_config.name); + resource->set_version(resource_config.version); + resource->mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + } + for (const auto& removed_resource : removed_resources) { + response.add_removed_resources(removed_resource); + } + stream.sendGrpcMessage(response); + } + + void sendNphdsDeltaResponse(FakeStream& stream, const std::string& version, + const std::vector& resource_configs, + const std::vector& removed_resources = {}) { + envoy::service::discovery::v3::DeltaDiscoveryResponse response; + response.set_system_version_info(version); + response.set_nonce(version); + response.set_type_url(NetworkPolicyHostsTypeUrl); + for (const auto& resource_config : resource_configs) { + envoy::service::discovery::v3::Resource* resource = response.add_resources(); + resource->set_name(resource_config.name); + resource->set_version(resource_config.version); + resource->mutable_resource()->PackFrom( + TestUtility::parseYaml(resource_config.yaml)); + } + for (const auto& removed_resource : removed_resources) { + response.add_removed_resources(removed_resource); + } + stream.sendGrpcMessage(response); + } + + AssertionResult compareNpdsAck() { + return compareDeltaDiscoveryRequest(NetworkPolicyTypeUrl, {}, {}, npds_stream_.get(), + Grpc::Status::WellKnownGrpcStatus::Ok, "", + /*expect_node=*/false); + } + + AssertionResult compareNprdsAck() { + return compareDeltaDiscoveryRequest(NetworkPolicyResourceTypeUrl, {}, {}, nprds_stream_.get(), + Grpc::Status::WellKnownGrpcStatus::Ok, "", + /*expect_node=*/false); + } + + AssertionResult compareNphdsAck() { + return compareDeltaDiscoveryRequest(NetworkPolicyHostsTypeUrl, {}, {}, nphds_stream_.get(), + Grpc::Status::WellKnownGrpcStatus::Ok, "", + /*expect_node=*/false); + } + + void retireStream(FakeStreamPtr& stream) { + if (stream != nullptr) { + retired_streams_.push_back(std::move(stream)); + } + } + + void resetGrpcStream(FakeStreamPtr& stream) { + stream->encodeResetStream(); + AssertionResult result = stream->waitForReset(*dispatcher_); + RELEASE_ASSERT(result, result.message()); + retireStream(stream); } + void resetNpdsStream() { resetGrpcStream(npds_stream_); } + + void resetNprdsStream() { resetGrpcStream(nprds_stream_); } + + void resetNphdsStream() { resetGrpcStream(nphds_stream_); } + void resetConnections() { - if (ads_connection_ != nullptr) { - AssertionResult result = ads_connection_->close(); + if (xds_connection_ != nullptr) { + AssertionResult result = xds_connection_->close(); RELEASE_ASSERT(result, result.message()); - result = ads_connection_->waitForDisconnect(); + result = xds_connection_->waitForDisconnect(); RELEASE_ASSERT(result, result.message()); - ads_connection_.reset(); + xds_connection_.reset(); } + ads_stream_.reset(); + lds_stream_.reset(); + cds_stream_.reset(); + npds_stream_.reset(); + nphds_stream_.reset(); + nprds_stream_.reset(); + retired_streams_.clear(); + } + + uint64_t policyStreamGeneration() const { + return test_server_->gauge("cilium.policy.policy_stream_generation")->value(); + } + + uint64_t waitForPolicyStreamGenerationAfter(uint64_t previous_generation) { + test_server_->waitForGaugeGe("cilium.policy.policy_stream_generation", previous_generation + 1); + const uint64_t generation = policyStreamGeneration(); + EXPECT_GT(generation, previous_generation); + return generation; + } + + std::shared_ptr networkPolicyMap() const { + auto map = test_server_->server().singletonManager().getTyped( + "cilium_network_policy_singleton"); + RELEASE_ASSERT(map != nullptr, "Cilium NetworkPolicyMap singleton was not created"); + return map; + } + + uint64_t resolveHostPolicyId(const std::string& address) const { + auto parsed_address = Network::Utility::parseInternetAddressNoThrow(address); + RELEASE_ASSERT(parsed_address != nullptr, + fmt::format("failed to parse host address {}", address)); + auto map = test_server_->server().singletonManager().getTyped( + "cilium_host_map_singleton"); + RELEASE_ASSERT(map != nullptr, "Cilium PolicyHostMap singleton was not created"); + + absl::Mutex lock; + bool resolved = false; + uint64_t policy_id = Cilium::ID::UNKNOWN; + + // PolicyHostMap lookups must run on an Envoy thread that has thread-local storage registered. + // The gtest/integration thread calling this helper is not such a thread, so dereferencing the + // TLS-backed host map directly here is unsafe. Posting the resolve to the server dispatcher + // keeps the lookup on a valid Envoy TLS thread while still letting the test read the result. + test_server_->server().dispatcher().post([&, map, parsed_address]() { + policy_id = map->resolve(parsed_address->ip()); + lock.Lock(); + resolved = true; + lock.Unlock(); + }); + + lock.LockWhen(absl::Condition(&resolved)); + lock.Unlock(); + return policy_id; } envoy::config::listener::v3::Listener listener_config_; std::string listener_name_{"testing-listener-0"}; - FakeHttpConnectionPtr ads_connection_; + FakeHttpConnectionPtr xds_connection_; FakeStreamPtr ads_stream_; + FakeStreamPtr lds_stream_; + FakeStreamPtr cds_stream_; + FakeStreamPtr npds_stream_; + FakeStreamPtr nphds_stream_; + FakeStreamPtr nprds_stream_; + std::vector retired_streams_; }; INSTANTIATE_TEST_SUITE_P(IpVersionsAndGrpcTypes, BpfMetadataIntegrationTest, @@ -251,21 +689,311 @@ TEST_P(BpfMetadataIntegrationTest, BpfMetadataWithNpdsAndNpdhsViaAds) { EXPECT_TRUE(compareDiscoveryRequest( Config::TestTypeUrl::get().Cluster, "", {}, {}, {}, /*expect_node=*/true, Envoy::Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); - sendCdsResponse("1"); + sendCdsResponse(*ads_stream_, "1"); EXPECT_TRUE(compareDiscoveryRequest( Config::TestTypeUrl::get().Listener, "", {}, {}, {}, /*expect_node=*/false, Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); - sendLdsResponse({MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); + sendLdsResponse(*ads_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); }; - initialize(); + initializeAds(); test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); EXPECT_EQ(test_server_->server().listenerManager().listeners().size(), 1); - sendNpdsResponse("1"); + sendNpdsResponse(*ads_stream_, "1"); test_server_->waitForCounterGe("cilium.policy.update_success", 1); - sendNphdsResponse("1"); + sendNphdsResponse(*ads_stream_, "1"); test_server_->waitForCounterGe("cilium.hostmap.update_success", 1); } +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedAdsGrpcStreams) { + on_server_init_function_ = [&]() { + createAdsStream(); + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/true); + EXPECT_TRUE(compareDiscoveryRequest( + Config::TestTypeUrl::get().Cluster, "", {}, {}, {}, + /*expect_node=*/true, Envoy::Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); + sendCdsResponse(*ads_stream_, "1"); + EXPECT_TRUE(compareDiscoveryRequest( + Config::TestTypeUrl::get().Listener, "", {}, {}, {}, /*expect_node=*/false, + Grpc::Status::WellKnownGrpcStatus::Ok, "", ads_stream_.get())); + sendLdsResponse(*ads_stream_, {MessageUtil::getYamlStringFromMessage(listener_config_)}, "1"); + }; + initializeAds(); + + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + EXPECT_EQ(policyStreamGeneration(), 0); + + sendNpdsResponse(*ads_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(0); + + sendNpdsResponse(*ads_stream_, "2"); + test_server_->waitForCounterGe("cilium.policy.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + resetConnections(); + EXPECT_EQ(policyStreamGeneration(), first_generation); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedSotwGrpcStreams) { + on_server_init_function_ = [&]() { + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createStreamsUntil("1", NetworkPolicyTypeUrl); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + EXPECT_EQ(policyStreamGeneration(), 0); + + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(0); + + sendNpdsResponse(*npds_stream_, "2"); + test_server_->waitForCounterGe("cilium.policy.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + resetConnections(); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + createStreamsUntil("2", NetworkPolicyTypeUrl); + sendNpdsResponse(*npds_stream_, "3", {invalid_policy}); + // The invalid policy is rejected by the real gRPC subscription decoder/validator before + // NetworkPolicyMapImpl::onConfigUpdate() runs, so this increments NPDS subscription stats + // rather than cilium.policy.updates_rejected. + test_server_->waitForCounterGe("cilium.npds.update_rejected", 1); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + sendNpdsResponse(*npds_stream_, "4"); + test_server_->waitForCounterGe("cilium.policy.update_success", 3); + waitForPolicyStreamGenerationAfter(first_generation); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedSotwGrpcNprdsStreams) { + on_server_init_function_ = [&]() { + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false, + envoy::config::core::v3::ApiConfigSource::GRPC, + /*use_nprds=*/true); + createStreamsUntil("1", NetworkPolicyResourceTypeUrl); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + + sendNprdsResponse(*nprds_stream_, "1", + {selector1_resource, selector2_resource, policy42_resource, policy43_resource}); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + sendNprdsResponse(*nprds_stream_, "2", {selector3_resource, policy42_new_stream_resource}); + test_server_->waitForCounterGe("cilium.policy.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_FALSE(policy_map->exists("10.2.3.4")); + + resetConnections(); + EXPECT_EQ(policyStreamGeneration(), first_generation); + + createStreamsUntil("2", NetworkPolicyResourceTypeUrl); + sendNprdsResponse(*nprds_stream_, "3", {selector3_resource, policy42_new_stream_resource}); + test_server_->waitForCounterGe("cilium.policy.update_success", 3); + waitForPolicyStreamGenerationAfter(first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_FALSE(policy_map->exists("10.2.3.4")); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNpdsStreams) { + on_server_init_function_ = [&]() { + // Step 1: establish the initial SotW LDS, CDS, NPHDS, and NPDS streams. + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createSotWStreams("1"); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + + // Step 2: accept a real SotW NPDS response so the starting mode has installed policy. + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t sotw_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.1.1")); + EXPECT_TRUE(policy_map->exists("10.2.2.2")); + + // Step 3: update the BpfMetadata config source; this is evidence that Delta NPDS is available. + updateBpfMetadataListenerFilter(listener_config_, + envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); + sendLdsResponse(*lds_stream_, {listener_config_}, "2"); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 2); + + // Step 4: observe the immediate switch to Delta NPDS without advancing accepted policy state. + createStreamsUntil("2", NetworkPolicyTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + + // Step 5: accept the first Delta NPDS update and retire the prior SotW policy resources. + sendNpdsDeltaResponse(*npds_stream_, "1", {policy42, policy43}); + EXPECT_TRUE(compareNpdsAck()); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(sotw_generation); + EXPECT_FALSE(policy_map->exists("10.1.1.1")); + EXPECT_FALSE(policy_map->exists("10.2.2.2")); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 6: accept a same-stream Delta update; stream generation and omitted resources stay put. + sendNpdsDeltaResponse(*npds_stream_, "2", {policy42_new_stream}); + EXPECT_TRUE(compareNpdsAck()); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 7: reset the Delta NPDS stream. + resetNpdsStream(); + + // Step 8: open the replacement Delta stream; reconnect alone must not advance policy state. + createStreamsUntil("3", NetworkPolicyTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 9: accept the first update on the new stream and retire resources from the old stream. + sendNpdsDeltaResponse(*npds_stream_, "3", {policy42_new_stream}); + waitForPolicyStreamGenerationAfter(first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_FALSE(policy_map->exists("10.2.3.4")); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNprdsStreams) { + on_server_init_function_ = [&]() { + // Step 1: establish the initial SotW LDS, CDS, NPHDS, and NPDS streams. + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createSotWStreams("1"); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + + // Step 2: accept a real SotW NPDS response so the starting mode has installed policy. + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t sotw_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.1.1")); + EXPECT_TRUE(policy_map->exists("10.2.2.2")); + + // Step 3: update the BpfMetadata config source; this is evidence that Delta NPRDS is available. + updateBpfMetadataListenerFilter(listener_config_, + envoy::config::core::v3::ApiConfigSource::DELTA_GRPC, + /*use_nprds=*/true); + sendLdsResponse(*lds_stream_, {listener_config_}, "2"); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 2); + + // Step 4: observe the immediate switch to Delta NPRDS without advancing accepted policy state. + createStreamsUntil("2", NetworkPolicyResourceTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + + // Step 5: accept the first Delta NPRDS update and retire the prior SotW policy resources. + sendNprdsDeltaResponse( + *nprds_stream_, "1", + {selector1_resource, selector2_resource, policy42_resource, policy43_resource}); + EXPECT_TRUE(compareNprdsAck()); + const uint64_t first_generation = waitForPolicyStreamGenerationAfter(sotw_generation); + EXPECT_FALSE(policy_map->exists("10.1.1.1")); + EXPECT_FALSE(policy_map->exists("10.2.2.2")); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 6: accept a same-stream Delta update; stream generation and omitted resources stay put. + sendNprdsDeltaResponse(*nprds_stream_, "2", {selector3_resource, policy42_new_stream_resource}); + EXPECT_TRUE(compareNprdsAck()); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 7: reset the Delta NPRDS stream. + resetNprdsStream(); + + // Step 8: open the replacement Delta stream; reconnect alone must not advance policy state. + createStreamsUntil("3", NetworkPolicyResourceTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), first_generation); + EXPECT_TRUE(policy_map->exists("10.2.3.4")); + + // Step 9: accept the first update on the new stream and retire resources from the old stream. + sendNprdsDeltaResponse(*nprds_stream_, "3", {selector3_resource, policy42_new_stream_resource}); + waitForPolicyStreamGenerationAfter(first_generation); + EXPECT_TRUE(policy_map->exists("10.1.2.3")); + EXPECT_FALSE(policy_map->exists("10.2.3.4")); +} + +TEST_P(BpfMetadataIntegrationTest, PolicyStreamGenerationTracksAcceptedDeltaNphdsStreams) { + on_server_init_function_ = [&]() { + // Step 1: establish the initial SotW LDS, CDS, NPHDS, and NPDS streams. + addBpfMetadataListenerFilter(listener_config_, /*use_ads=*/false); + createSotWStreams("1"); + }; + initializeSotw(); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 1); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 1); + + auto policy_map = networkPolicyMap(); + EXPECT_EQ(policyStreamGeneration(), 0); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 2: accept a real SotW NPDS response so the starting mode has installed policy. + sendNpdsResponse(*npds_stream_, "1"); + test_server_->waitForCounterGe("cilium.policy.update_success", 1); + const uint64_t sotw_generation = waitForPolicyStreamGenerationAfter(0); + EXPECT_TRUE(policy_map->exists("10.1.1.1")); + EXPECT_TRUE(policy_map->exists("10.2.2.2")); + + // Step 3: update the BpfMetadata config source; this is evidence that Delta NPHDS is available. + updateBpfMetadataListenerFilter(listener_config_, + envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); + sendLdsResponse(*lds_stream_, {listener_config_}, "2"); + test_server_->waitForCounterGe("listener_manager.lds.update_success", 2); + + // Step 4: observe the immediate switch to Delta NPHDS without advancing accepted policy state. + createStreamsUntil("2", NetworkPolicyHostsTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 5: accept the first Delta NPHDS update. This should not change policy stream generation. + sendNphdsDeltaResponse(*nphds_stream_, "1", {policy_host1_resource, policy_host2_resource}); + EXPECT_TRUE(compareNphdsAck()); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 2); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 6: accept a same-stream Delta update; omitted resources stay present on the same stream. + sendNphdsDeltaResponse(*nphds_stream_, "2", {policy_host1_new_stream_resource}); + EXPECT_TRUE(compareNphdsAck()); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 3); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 7: reset the Delta NPHDS stream. + resetNphdsStream(); + + // Step 8: open the replacement Delta stream; reconnect alone must not advance policy state. + createStreamsUntil("3", NetworkPolicyHostsTypeUrl, /*expect_delta=*/true); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), 222); + + // Step 9: accept the first update on the new stream and retire resources from the old stream. + sendNphdsDeltaResponse(*nphds_stream_, "3", {policy_host1_new_stream_resource}); + EXPECT_TRUE(compareNphdsAck()); + test_server_->waitForCounterGe("cilium.hostmap.update_success", 4); + EXPECT_EQ(policyStreamGeneration(), sotw_generation); + EXPECT_EQ(resolveHostPolicyId("10.1.1.1"), 111); + EXPECT_EQ(resolveHostPolicyId("10.2.2.2"), Cilium::ID::UNKNOWN); +} + } // namespace } // namespace Envoy diff --git a/tests/cilium_network_policy_test.cc b/tests/cilium_network_policy_test.cc index bf54669b9..86a8c573c 100644 --- a/tests/cilium_network_policy_test.cc +++ b/tests/cilium_network_policy_test.cc @@ -3,10 +3,12 @@ #include #include +#include #include #include #include #include +#include #include "envoy/common/exception.h" #include "envoy/common/optref.h" @@ -33,6 +35,7 @@ #include "test/mocks/server/factory_context.h" #include "test/test_common/utility.h" +#include "absl/container/flat_hash_set.h" #include "absl/strings/string_view.h" #include "cilium/accesslog.h" #include "cilium/network_policy.h" @@ -71,6 +74,42 @@ extern const envoy::config::core::v3::ConfigSource CILIUM_XDS_API_CONFIG; return secret_provider; \ })) +namespace { + +envoy::config::core::v3::ConfigSource configSourceForUseDeltaXds(bool use_delta_xds) { + auto config_source = CILIUM_XDS_API_CONFIG; + config_source.mutable_api_config_source()->set_api_type( + use_delta_xds ? envoy::config::core::v3::ApiConfigSource::DELTA_GRPC + : envoy::config::core::v3::ApiConfigSource::GRPC); + return config_source; +} + +struct FakeSubscriptionState { + int start_calls_{0}; + std::vector> start_resources_; +}; + +class FakeSubscription : public Envoy::Config::Subscription { +public: + explicit FakeSubscription(std::shared_ptr state) + : state_(std::move(state)) {} + + void start(const absl::flat_hash_set& resource_names) override { + ++state_->start_calls_; + auto& started = state_->start_resources_.emplace_back(); + started.insert(started.end(), resource_names.begin(), resource_names.end()); + std::sort(started.begin(), started.end()); + } + + void updateResourceInterest(const absl::flat_hash_set&) override {} + void requestOnDemandUpdate(const absl::flat_hash_set&) override {} + +private: + std::shared_ptr state_; +}; + +} // namespace + class CiliumNetworkPolicyTest : public ::testing::Test { protected: CiliumNetworkPolicyTest() { @@ -92,8 +131,10 @@ class CiliumNetworkPolicyTest : public ::testing::Test { ON_CALL_SDS_SECRET_PROVIDER(secret_manager_, TlsSessionTicketKeysContext, TlsSessionTicketKeys); ON_CALL_SDS_SECRET_PROVIDER(secret_manager_, GenericSecret, GenericSecret); - policy_map_ = - std::make_shared(factory_context_, Cilium::CILIUM_XDS_API_CONFIG); + bool use_delta = useDeltaXds(); + bool use_nprds = use_delta; + policy_map_ = std::make_shared(factory_context_, use_nprds, + configSourceForUseDeltaXds(use_delta)); } void TearDown() override { @@ -101,6 +142,8 @@ class CiliumNetworkPolicyTest : public ::testing::Test { policy_map_.reset(); } + virtual bool useDeltaXds() const { return false; } + Envoy::Config::SubscriptionCallbacks& subscriptionCallbacks() const { return policy_map_->subscriptionCallbacksForTest(); } @@ -120,6 +163,23 @@ class CiliumNetworkPolicyTest : public ::testing::Test { return message.version_info(); } + std::string deltaUpdateFromYaml(const std::string& config) { + envoy::service::discovery::v3::DeltaDiscoveryResponse message; + MessageUtil::loadFromYaml(config, message, ProtobufMessage::getNullValidationVisitor()); + NetworkPolicyResourceDecoder network_policy_resource_decoder; + auto decoded_resources = std::make_unique(); + for (const auto& resource : message.resources()) { + decoded_resources->pushBack( + Config::DecodedResourceImpl::fromResource(network_policy_resource_decoder, resource)); + } + + EXPECT_TRUE(subscriptionCallbacks() + .onConfigUpdate(decoded_resources->refvec_, message.removed_resources(), + message.system_version_info()) + .ok()); + return message.system_version_info(); + } + testing::AssertionResult validate(const std::string& pod_ip, const std::string& expected) { const auto& policy = policy_map_->getPolicyInstance(pod_ip, false); auto str = policy.string(); @@ -239,6 +299,34 @@ class CiliumNetworkPolicyTest : public ::testing::Test { return policy_map_->statsForTest().updates_rejected_.name(); } + PolicyInstanceConstSharedPtr policyInstanceShared(const std::string& pod_ip) const { + return policy_map_->getPolicyInstanceSharedForTest(pod_ip); + } + + uint64_t selectorStreamGenerationForTest(const PolicyInstance& policy) const { + return policy_map_->policySelectorStreamGenerationForTest(policy); + } + + SelectorVersion selectorVersionForTest(const PolicyInstance& policy) const { + return policy_map_->policySelectorVersionForTest(policy); + } + + void resetStreamForTest() { policy_map_->resetStreamForTest(); } + bool configuredUseDeltaXds() const { return policy_map_->desiredUseDeltaXdsForTest(); } + void setPolicyConfig(bool use_delta_xds, bool use_nprds) const { + policy_map_->setConfig(use_nprds, configSourceForUseDeltaXds(use_delta_xds)); + } + void startManagedSubscriptionForTest() { policy_map_->startManagedSubscriptionForTest(); } + void setSubscriptionFactoryForTest(NetworkPolicyMap::SubscriptionFactoryForTest factory) { + policy_map_->setSubscriptionFactoryForTest(std::move(factory)); + } + void onSubscriptionConnectedForTest() { policy_map_->onSubscriptionConnectedForTest(); } + void onSubscriptionTransportCloseForTest() { policy_map_->onSubscriptionTransportCloseForTest(); } + bool subscriptionUseDeltaXdsForTest() const { + return policy_map_->subscriptionUseDeltaXdsForTest(); + } + bool subscriptionConnectedForTest() const { return policy_map_->subscriptionConnectedForTest(); } + NiceMock factory_context_; NiceMock secret_manager_; std::shared_ptr policy_map_; @@ -246,10 +334,379 @@ class CiliumNetworkPolicyTest : public ::testing::Test { uint16_t proxy_id_ = 42; }; +class CiliumNetworkPolicyDeltaTest : public CiliumNetworkPolicyTest { +protected: + bool useDeltaXds() const override { return true; } +}; + TEST_F(CiliumNetworkPolicyTest, UpdatesRejectedStatName) { EXPECT_EQ("cilium.policy.updates_rejected", updatesRejectedStatName()); } +TEST_F(CiliumNetworkPolicyTest, ManagedSubscriptionColdStartUsesConfiguredSotwMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + + ASSERT_EQ(created_modes.size(), 1); + EXPECT_FALSE(created_modes.front()); + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 1); + EXPECT_TRUE(state->start_resources_.front().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, ManagedSubscriptionColdStartUsesConfiguredDeltaMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + + ASSERT_EQ(created_modes.size(), 1); + EXPECT_TRUE(created_modes.front()); + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 1); + EXPECT_TRUE(state->start_resources_.front().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipFromSotwToDeltaOnHealthySubscriptionRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + + setPolicyConfig(true, true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipFromDeltaToSotwOnHealthySubscriptionWaitsForClose) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_FALSE(subscriptionUseDeltaXdsForTest()); + + setPolicyConfig(true, true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); + + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setPolicyConfig(false, false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_TRUE(subscriptionConnectedForTest()); + // Once we have an established delta subscription, keep it until transport close even if the + // configured desired mode flips back to SotW. + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + + onSubscriptionTransportCloseForTest(); + + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true, false)); + EXPECT_EQ(state->start_calls_, 3); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, FlagFlipWhileDisconnectedRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + ASSERT_FALSE(subscriptionConnectedForTest()); + + setPolicyConfig(true, true); + + EXPECT_TRUE(configuredUseDeltaXds()); + EXPECT_TRUE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, true)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, FlagFlipFromDisconnectedDeltaToSotwRecreatesImmediately) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setPolicyConfig(false, false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, false)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DowngradeFromConnectedDeltaRecreatesDisconnectedRetryToSotw) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + // Agent restart/downgrade drops the established delta transport. While the desired mode is still + // delta we recreate immediately and begin retrying delta. + onSubscriptionTransportCloseForTest(); + + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + ASSERT_EQ(state->start_calls_, 2); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true)); + EXPECT_TRUE(state->start_resources_.back().empty()); + + // When listener metadata later reveals the downgraded agent no longer supports delta, flip to + // SotW immediately rather than letting the disconnected delta retry loop forever. + setPolicyConfig(false, false); + + EXPECT_FALSE(configuredUseDeltaXds()); + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true, false)); + EXPECT_EQ(state->start_calls_, 3); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyTest, TransportCloseWithoutFlagFlipRecreatesInCurrentMode) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + + onSubscriptionTransportCloseForTest(); + + EXPECT_FALSE(subscriptionConnectedForTest()); + EXPECT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(false, false)); + EXPECT_EQ(state->start_calls_, 2); + EXPECT_TRUE(state->start_resources_.back().empty()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPoliciesUseDeltaSdsForGenericSecrets) { + EXPECT_CALL(secret_manager_, findOrCreateGenericSecretProvider(_, "nonexisting-sds-secret", _, _)) + .WillOnce(Invoke([](const envoy::config::core::v3::ConfigSource& sds_config_source, + const std::string& config_name, + Server::Configuration::ServerFactoryContext& server_context, + Init::Manager& init_manager) { + EXPECT_TRUE(sds_config_source.has_api_config_source()); + EXPECT_EQ(envoy::config::core::v3::ApiConfigSource::DELTA_GRPC, + sds_config_source.api_config_source().api_type()); + auto secret_provider = Secret::GenericSecretSdsApi::create( + server_context, sds_config_source, config_name, []() {}, false); + init_manager.add(*secret_provider->initTarget()); + return secret_provider; + })); + + std::string version; + EXPECT_NO_THROW(version = deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-43" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' + header_matches: + - name: 'bearer-token' + value_sds_secret: 'nonexisting-sds-secret' + mismatch_action: REPLACE_ON_MISMATCH +)EOF")); + EXPECT_EQ(version, "1"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, SotwDowngrade) { + auto state = std::make_shared(); + std::vector created_modes; + setSubscriptionFactoryForTest( + [state, &created_modes](bool use_delta_xds) -> std::unique_ptr { + created_modes.push_back(use_delta_xds); + return std::make_unique(state); + }); + + using ApiType = envoy::config::core::v3::ApiConfigSource::ApiType; + auto expect_tls_sds_mode = [this](ApiType api_type) { + EXPECT_CALL(secret_manager_, findOrCreateTlsCertificateProvider(_, "secret1", _, _, _)) + .WillOnce(Invoke([api_type](const envoy::config::core::v3::ConfigSource& sds_config_source, + const std::string& config_name, + Server::Configuration::ServerFactoryContext& server_context, + OptRef init_manager, bool /*warm*/) { + EXPECT_TRUE(sds_config_source.has_api_config_source()); + EXPECT_EQ(api_type, sds_config_source.api_config_source().api_type()); + auto secret_provider = Secret::TlsCertificateSdsApi::create( + server_context, sds_config_source, config_name, []() {}, false); + if (init_manager.has_value()) { + init_manager->add(*secret_provider->initTarget()); + } + return secret_provider; + })) + .RetiresOnSaturation(); + EXPECT_CALL(secret_manager_, + findOrCreateCertificateValidationContextProvider(_, "cacerts", _, _)) + .WillOnce(Invoke([api_type](const envoy::config::core::v3::ConfigSource& sds_config_source, + const std::string& config_name, + Server::Configuration::ServerFactoryContext& server_context, + Init::Manager& init_manager) { + EXPECT_TRUE(sds_config_source.has_api_config_source()); + EXPECT_EQ(api_type, sds_config_source.api_config_source().api_type()); + auto secret_provider = Secret::CertificateValidationContextSdsApi::create( + server_context, sds_config_source, config_name, []() {}, false); + init_manager.add(*secret_provider->initTarget()); + return secret_provider; + })) + .RetiresOnSaturation(); + }; + + startManagedSubscriptionForTest(); + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + expect_tls_sds_mode(envoy::config::core::v3::ApiConfigSource::DELTA_GRPC); + std::string version; + EXPECT_NO_THROW(version = deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + egress_per_port_policies: + - port: 80 + rules: + - server_names: [ "example.com" ] + downstream_tls_context: + tls_sds_secret: "secret1" + upstream_tls_context: + validation_context_sds_secret: "cacerts" +)EOF")); + EXPECT_EQ(version, "1"); + + // Simulate xDS server restart and downgrade to SotW mode + onSubscriptionTransportCloseForTest(); + + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_TRUE(subscriptionUseDeltaXdsForTest()); + + setPolicyConfig(false, false); + + ASSERT_FALSE(subscriptionConnectedForTest()); + ASSERT_FALSE(subscriptionUseDeltaXdsForTest()); + EXPECT_THAT(created_modes, testing::ElementsAre(true, true, false)); + + onSubscriptionConnectedForTest(); + ASSERT_TRUE(subscriptionConnectedForTest()); + ASSERT_FALSE(subscriptionUseDeltaXdsForTest()); + + // Same NetworkPolicy, but it must be updated for the secret watcher to also use SotW + expect_tls_sds_mode(envoy::config::core::v3::ApiConfigSource::GRPC); + EXPECT_NO_THROW(version = updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + egress_per_port_policies: + - port: 80 + rules: + - server_names: [ "example.com" ] + downstream_tls_context: + tls_sds_secret: "secret1" + upstream_tls_context: + validation_context_sds_secret: "cacerts" +)EOF")); + EXPECT_EQ(version, "1"); +} + TEST_F(CiliumNetworkPolicyTest, EmptyPolicyUpdate) { EXPECT_TRUE(subscriptionCallbacks().onConfigUpdate({}, "1").ok()); EXPECT_FALSE(validate("10.1.2.3", "")); // Policy not found @@ -263,6 +720,1493 @@ TEST_F(CiliumNetworkPolicyTest, SimplePolicyUpdate) { EXPECT_FALSE(validate("10.1.2.3", "")); // Policy not found } +TEST_F(CiliumNetworkPolicyTest, RejectsWhitespaceInSotwWrappedResourceName) { + EXPECT_THROW_WITH_MESSAGE(updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/envoy.service.discovery.v3.Resource + name: "policy 42" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF"), + EnvoyException, + "NetworkPolicy resource name 'policy 42' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaIncrementalPolicyUpdate) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 8080)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSameStreamKeepsUntouchedResources) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 8080)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovesPolicyByResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + - "f00d::1" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("f00d::1")); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("f00d::1", 43, 80)); + + EXPECT_TRUE(policy_map_->exists("10.2.3.4")); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "policy-42" +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_FALSE(policy_map_->exists("f00d::1")); + EXPECT_FALSE(validate("10.1.2.3", "")); + EXPECT_FALSE(validate("f00d::1", "")); + + EXPECT_TRUE(policy_map_->exists("10.2.3.4")); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSelectorOnlyUpdateTakesEffectImmediately) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPolicyUpdateRejectsMissingSelectorResource) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-2" ] +)EOF"), + EnvoyException, + "NetworkPolicyResource rule references missing selector resource " + "'selector-2'"); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovedAndReaddedSelectorNameDoesNotRebindOldPolicy) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "selector-1" +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "3" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_EQ(old_policy.get(), policyInstanceShared("10.1.2.3").get()); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "4" +resources: +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_EQ(old_policy.get(), policyInstanceShared("10.1.2.3").get()); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovedAndReaddedSelectorNameInSameUpdateActsAsUpdate) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "selector-1" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectedSelectorUpdateKeepsPublishedBehavior) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-42'.*endpoint_id 42.*10\.1\.2\.3)"); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaPassUsesCurrentSelectorMembership) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaWildcardPortPassIsMergedToExactPortRules) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 0 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - port: 80 + rules: + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSamePrecedenceDenyWinsOverPass) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 1000 + deny: true + selectors: [ "selector-1" ] + - precedence: 500 + selectors: [ "selector-1" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaSelectorOnlyUpdateChangesPassBehaviorImmediately) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43, 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 501 + selectors: [ "selector-1" ] + - precedence: 900 + deny: true + - precedence: 500 + selectors: [ "selector-2" ] + http_rules: + http_rules: + - headers: + - name: ':path' + exact_match: '/allowed' +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80, {{":path", "/allowed"}})); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80, {{":path", "/allowed"}})); +} + +TEST_F(CiliumNetworkPolicyTest, SotwRejectsSelectorsInRules) { + EXPECT_THROW(updateFromYaml(R"EOF(version_info: "1" +resources: +- "@type": type.googleapis.com/cilium.NetworkPolicy + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitrarySelectorResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "7" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "7" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEmbeddedRemotePoliciesInRules) { + EXPECT_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - remote_policies: [ 43 ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsAddedResourceNamesWithWhitespace) { + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector 1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + "NetworkPolicyResource added resource name 'selector 1' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovedResourceNamesWithWhitespace) { + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +removed_resources: +- "selector 1" +)EOF"), + EnvoyException, + "NetworkPolicyResource removed resource name 'selector 1' must not contain whitespace"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicatePolicyResourceNamesInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming policy resource 'shared-name'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesThatDoNotMatchEndpointId) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsResourceNameEndpointIpCollisionsInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsDuplicateSelectorResourceNamesInSameUpdate) { + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-selector" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44, 45 ] +- name: "shared-selector" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 46, 47 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key 'shared-selector'.*incoming selector resource 'shared-selector'.*existing selector resource 'shared-selector')"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsArbitraryPolicyResourceNamesWithHyphens) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-qualified-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRemovesPoliciesByArbitraryResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-qualified-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "42" +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsEndpointIpCollisionsWithExistingPolicies) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.3.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsPolicyResourceNameCollidingWithExistingEndpointIp) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming policy resource '10\.1\.2\.3'.*endpoint_id 43.*10\.1\.2\.4.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsEndpointIpCollidingWithExistingPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF"), + EnvoyException, + R"(NetworkPolicyResource update for version [0-9]+ has duplicate resource key '10\.1\.2\.4'.*incoming policy resource 'policy-b'.*endpoint_id 43.*10\.1\.2\.4.*existing policy resource '10\.1\.2\.4'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsSelectorResourceNameCollidingWithExistingPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "shared-name" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key 'shared-name'.*incoming selector resource 'shared-name'.*existing policy resource 'shared-name'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, + DeltaRejectsSelectorResourceNameCollidingWithExistingEndpointIp) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_REGEX( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "10.1.2.3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +)EOF"), + EnvoyException, + R"(NetworkPolicyResource .*update for version [0-9]+ has duplicate resource key '10\.1\.2\.3'.*incoming selector resource '10\.1\.2\.3'.*existing endpoint IP alias '10\.1\.2\.3' owned by policy resource 'policy-a'.*endpoint_id 42.*10\.1\.2\.3)"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsRemovingPolicyEndpointIpAlias) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-a" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_THROW_WITH_MESSAGE( + deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "10.1.2.3" +)EOF"), + EnvoyException, + "NetworkPolicyResource removed resource '10.1.2.3' is a policy endpoint IP alias, not a " + "resource name"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAllowsEndpointIpReusingRemovedPolicyResourceName) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "10.1.2.4" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +)EOF")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +removed_resources: +- "10.1.2.4" +resources: +- name: "policy-b" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaAcceptsPolicyResourceNamesWithNumericSuffixes) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-042" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 +- name: "policy-0" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.4" + endpoint_id: 43 +)EOF")); + + EXPECT_TRUE(policy_map_->exists("10.1.2.3")); + EXPECT_TRUE(policy_map_->exists("10.1.2.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsZeroEndpointIdRegardlessOfResourceName) { + EXPECT_THROW_WITH_MESSAGE(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "policy-0" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 0 +)EOF"), + EnvoyException, "NetworkPolicyResource endpoint_id must be non-zero"); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaRejectsInconsistentPassPrecedence) { + EXPECT_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - precedence: 1000 + pass_precedence: 100 + selectors: [ "selector-1" ] + - precedence: 900 + pass_precedence: 200 + selectors: [ "selector-2" ] +)EOF"), + EnvoyException); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaNewStreamReplacesStateWithFullSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 43, 80)); + EXPECT_TRUE(ingressAllowed("10.2.3.4", 44, 81)); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 45, 8080)); + EXPECT_FALSE(policy_map_->exists("10.2.3.4")); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, DeltaNewStreamClearsStaleResourceNamesFromResourceMap) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.2.3.4" + endpoint_id: 43 + ingress_per_port_policies: + - port: 81 + rules: + - selectors: [ "selector-2" ] +)EOF")); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-3" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 8080 + rules: + - selectors: [ "selector-3" ] +)EOF")); + + EXPECT_FALSE(policy_map_->exists("10.2.3.4")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "3" +resources: +- name: "policy-43" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 46 ] +)EOF")); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "4" +resources: +- name: "policy-42" + version: "3" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 9090 + rules: + - selectors: [ "policy-43" ] +)EOF")); + + EXPECT_TRUE(ingressAllowed("10.1.2.3", 46, 9090)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, SameStreamSelectorOnlyUpdateUsesLatestSelectorSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + const auto initial_stream_generation = selectorStreamGenerationForTest(*old_policy); + + EXPECT_GT(initial_stream_generation, 0); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(1, selectorVersionForTest(*old_policy)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +)EOF")); + + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); +} + +TEST_F(CiliumNetworkPolicyDeltaTest, NewStreamKeepsOldPolicyPinnedToOldSelectorSnapshot) { + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "selector-2" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "policy-42" + version: "1" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto old_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, old_policy); + const auto initial_stream_generation = selectorStreamGenerationForTest(*old_policy); + + EXPECT_GT(initial_stream_generation, 0); + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(1, selectorVersionForTest(*old_policy)); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "1" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 45 ] +)EOF")); + + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + + resetStreamForTest(); + + EXPECT_NO_THROW(deltaUpdateFromYaml(R"EOF(system_version_info: "2" +resources: +- name: "selector-1" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 44 ] +- name: "selector-2" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + selector: + remote_identities: [ 43 ] +- name: "policy-42" + version: "2" + resource: + "@type": type.googleapis.com/cilium.NetworkPolicyResource + policy: + endpoint_ips: + - "10.1.2.3" + endpoint_id: 42 + ingress_per_port_policies: + - port: 80 + rules: + - selectors: [ "selector-1" ] +)EOF")); + + const auto new_policy = policyInstanceShared("10.1.2.3"); + ASSERT_NE(nullptr, new_policy); + EXPECT_NE(old_policy.get(), new_policy.get()); + + EXPECT_EQ(initial_stream_generation, selectorStreamGenerationForTest(*old_policy)); + EXPECT_EQ(2, selectorVersionForTest(*old_policy)); + EXPECT_EQ(initial_stream_generation + 1, selectorStreamGenerationForTest(*new_policy)); + EXPECT_EQ(3, selectorVersionForTest(*new_policy)); + EXPECT_TRUE(ingressAllowed("10.1.2.3", 44, 80)); + EXPECT_FALSE(ingressAllowed("10.1.2.3", 43, 80)); +} + TEST_F(CiliumNetworkPolicyTest, OverlappingPortRange) { EXPECT_NO_THROW(updateFromYaml(R"EOF(version_info: "1" resources: diff --git a/tests/cilium_tcp_integration.cc b/tests/cilium_tcp_integration.cc index 40ea6f1a1..049a6e352 100644 --- a/tests/cilium_tcp_integration.cc +++ b/tests/cilium_tcp_integration.cc @@ -26,6 +26,7 @@ const std::string TCP_POLICY_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_tcp_integration_test.cc b/tests/cilium_tcp_integration_test.cc index 24ca3817c..b7be8f2b2 100644 --- a/tests/cilium_tcp_integration_test.cc +++ b/tests/cilium_tcp_integration_test.cc @@ -343,6 +343,7 @@ const std::string TCP_POLICY_LINEPARSER_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: 1 @@ -573,6 +574,7 @@ const std::string TCP_POLICY_BLOCKPARSER_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: 1 diff --git a/tests/cilium_tls_http_integration_test.cc b/tests/cilium_tls_http_integration_test.cc index 040ada472..9d277c945 100644 --- a/tests/cilium_tls_http_integration_test.cc +++ b/tests/cilium_tls_http_integration_test.cc @@ -184,6 +184,7 @@ const std::string BASIC_TLS_POLICY_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_tls_tcp_integration_test.cc b/tests/cilium_tls_tcp_integration_test.cc index 05999a877..d3c42bd41 100644 --- a/tests/cilium_tls_tcp_integration_test.cc +++ b/tests/cilium_tls_tcp_integration_test.cc @@ -273,6 +273,7 @@ const std::string TCP_POLICY_UPSTREAM_TLS_fmt = R"EOF(version_info: "0" - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_websocket_codec_integration_test.cc b/tests/cilium_websocket_codec_integration_test.cc index 05eb64d13..c4d1f70bf 100644 --- a/tests/cilium_websocket_codec_integration_test.cc +++ b/tests/cilium_websocket_codec_integration_test.cc @@ -98,6 +98,7 @@ class CiliumWebSocketIntegrationTest : public CiliumTcpIntegrationTest { - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_websocket_decap_integration_test.cc b/tests/cilium_websocket_decap_integration_test.cc index d7a7055f4..46f9411fe 100644 --- a/tests/cilium_websocket_decap_integration_test.cc +++ b/tests/cilium_websocket_decap_integration_test.cc @@ -101,6 +101,7 @@ class CiliumWebSocketIntegrationTest : public CiliumHttpIntegrationTest { - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/cilium_websocket_policy_integration_test.cc b/tests/cilium_websocket_policy_integration_test.cc index c456a9312..17b682c5a 100644 --- a/tests/cilium_websocket_policy_integration_test.cc +++ b/tests/cilium_websocket_policy_integration_test.cc @@ -231,6 +231,7 @@ class CiliumWebSocketIntegrationTest : public CiliumTcpIntegrationTest { - "@type": type.googleapis.com/cilium.NetworkPolicy endpoint_ips: - '{{ ntop_ip_loopback_address }}' + endpoint_id: 42 policy: 3 ingress_per_port_policies: - port: {0} diff --git a/tests/versioned_test.cc b/tests/versioned_test.cc new file mode 100644 index 000000000..be90c9ec5 --- /dev/null +++ b/tests/versioned_test.cc @@ -0,0 +1,1165 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "source/common/common/lock_guard.h" +#include "source/common/common/thread.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "cilium/versioned.h" + +// NOLINT(namespace-envoy) +namespace { + +class TrackedValue : public VersionedNode { +public: + explicit TrackedValue(int id) : TrackedValue(id, id) {} + + TrackedValue(int key_id, int generation) + : key_id_(key_id), generation_(generation), unique_id_(next_unique_id_++) { + ++constructed_count_; + ++live_count_; + } + + ~TrackedValue() { + ++destroyed_count_; + --live_count_; + ++destroyed_by_id_[generation_]; + } + + int id() const { return generation_; } + int keyId() const { return key_id_; } + int generation() const { return generation_; } + uint64_t uniqueId() const { return unique_id_; } + + static void resetCounters() { + constructed_count_ = 0; + destroyed_count_ = 0; + live_count_ = 0; + destroyed_by_id_.clear(); + next_unique_id_ = 1; + } + + static int constructedCount() { return constructed_count_; } + static int destroyedCount() { return destroyed_count_; } + static int liveCount() { return live_count_; } + + static int destroyedCountFor(int id) { + const auto it = destroyed_by_id_.find(id); + return it == destroyed_by_id_.end() ? 0 : it->second; + } + +private: + int key_id_; + int generation_; + uint64_t unique_id_; + + static inline int constructed_count_ = 0; + static inline int destroyed_count_ = 0; + static inline int live_count_ = 0; + static inline uint64_t next_unique_id_ = 1; + static inline std::map destroyed_by_id_; +}; + +using TrackedHandle = VersionedHandle; +using TrackedDeferredDeletion = DeferredDeletion; +using TrackedMap = VersionedMap; +using TrackedAccess = VersionedTestAccess; + +const TrackedValue* activeValue(const TrackedMap& map, const std::string& key, + uint64_t version = versionMax) { + auto handle = map.find(key); + return handle != nullptr ? handle->get(version) : nullptr; +} + +struct PublishedHandles { + uint64_t snapshot_id; + uint64_t version; + std::vector> handles; +}; + +struct ObservedNode { + const void* node; + const void* next; + const TrackedValue* value; + uint64_t add_version; + uint64_t remove_version; +}; + +enum class ValidationMode { + StrictConsistentView, + ConcurrentSafeInvisibleTail, +}; + +std::string describeValue(const TrackedValue* value) { + if (value == nullptr) { + return "nullptr"; + } + return testing::PrintToString( + std::make_tuple(value->keyId(), value->generation(), value->uniqueId())); +} + +std::string describeObservedChain(const std::vector& nodes) { + std::ostringstream out; + for (size_t i = 0; i < nodes.size(); ++i) { + if (i > 0) { + out << " -> "; + } + const auto& node = nodes[i]; + out << "{node=" << node.node << ", next=" << node.next + << ", value=" << describeValue(node.value) << ", add=" << node.add_version + << ", remove=" << node.remove_version << "}"; + } + if (nodes.empty()) { + out << ""; + } + return out.str(); +} + +std::string validateTraversalAtVersion(const std::string& label, + const VersionedNode* initial_head, + uint64_t version, ValidationMode mode, + const TrackedValue* expected = nullptr, + const void* handle = nullptr) { + const bool require_consistent_view = mode == ValidationMode::StrictConsistentView; + const TrackedValue* visible_value = nullptr; + size_t visible_count = 0; + uint64_t previous_add_version = versionNotRemoved; + bool in_invisible_tail = false; + absl::flat_hash_map visited_nodes; + std::vector observed_nodes; + for (const auto* node = initial_head; node != nullptr;) { + const auto* next = TrackedAccess::next(node); + observed_nodes.push_back(ObservedNode{node, next, TrackedAccess::value(node), + TrackedAccess::addVersion(node), + TrackedAccess::removeVersion(node)}); + const auto add_version = observed_nodes.back().add_version; + const auto [visited_it, inserted] = visited_nodes.emplace(node, add_version); + if (!inserted) { + if (visited_it->second == add_version) { + return "cycle detected for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + visited_it->second = add_version; + } + + const auto remove_version = observed_nodes.back().remove_version; + if (add_version >= versionNotRemoved) { + return "unpublished node observed for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + if ((require_consistent_view || !in_invisible_tail) && add_version > previous_add_version) { + return "add_version increased while following next links for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " previous_add=" + std::to_string(previous_add_version) + + " current_add=" + std::to_string(add_version) + + " chain=" + describeObservedChain(observed_nodes); + } + if (remove_version < versionNotRemoved && add_version > remove_version) { + return "invalid node visibility window for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + + if (add_version <= version && version < remove_version) { + ++visible_count; + visible_value = TrackedAccess::value(node); + if (require_consistent_view && visible_count > 1) { + return "multiple visible nodes at version " + std::to_string(version) + " for '" + label + + "' handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + } + // Once traversal reaches the first node that is already stale for this version, production + // readers no longer depend on per-handle ordering. First-phase GC may have already rewired the + // rest of the tail into a mixed deferred-deletion list, but those nodes must remain safe to + // walk and invisible for this version. + if (!require_consistent_view && in_invisible_tail && + TrackedAccess::isVisibleInVersion(node, version)) { + return "visible node observed after entering invisible tail for '" + label + "' at version " + + std::to_string(version) + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " chain=" + describeObservedChain(observed_nodes); + } + if (!TrackedAccess::isVisibleInVersion(node, version) && + !TrackedAccess::isAddedAfterVersion(node, version)) { + in_invisible_tail = true; + } + previous_add_version = add_version; + node = next; + } + + if (require_consistent_view && expected != visible_value) { + return "handle->get mismatch for '" + label + "' at version " + std::to_string(version) + + " handle=" + testing::PrintToString(handle) + + " initial_head=" + testing::PrintToString(initial_head) + + " expected=" + describeValue(expected) + ", traversed=" + describeValue(visible_value) + + ", chain=" + describeObservedChain(observed_nodes); + } + + return ""; +} + +std::string validateHandleAtVersion(const std::string& label, const TrackedHandle& handle, + uint64_t version, + ValidationMode mode = ValidationMode::StrictConsistentView) { + if (handle == nullptr) { + return "snapshot contains null handle for '" + label + "'"; + } + + return validateTraversalAtVersion( + label, TrackedAccess::head(handle), version, mode, + mode == ValidationMode::StrictConsistentView ? handle->get(version) : nullptr, handle.get()); +} + +std::string +validateNodeAtVersion(const std::string& label, const VersionedNode* node, + uint64_t version, + ValidationMode mode = ValidationMode::ConcurrentSafeInvisibleTail) { + if (node == nullptr) { + return "null node supplied for '" + label + "'"; + } + return validateTraversalAtVersion(label, node, version, mode); +} + +std::string +validatePublishedSnapshotAtVersion(const PublishedHandles& snapshot, uint64_t version, + ValidationMode mode = ValidationMode::StrictConsistentView) { + for (const auto& [label, handle] : snapshot.handles) { + auto error = validateHandleAtVersion(label, handle, version, mode); + if (!error.empty()) { + return error; + } + } + return ""; +} + +std::string validatePublishedSnapshot(const PublishedHandles& snapshot) { + return validatePublishedSnapshotAtVersion(snapshot, snapshot.version); +} + +std::shared_ptr makePublishedHandles(const TrackedMap& map, + uint64_t snapshot_id = 0) { + auto snapshot = std::make_shared(); + snapshot->snapshot_id = snapshot_id; + snapshot->version = map.getVersion(); + + absl::flat_hash_set seen_handles; + for (const auto& [key, handle] : TrackedAccess::entries(map)) { + if (handle != nullptr && seen_handles.insert(handle.get()).second) { + snapshot->handles.emplace_back(key, handle); + } + } + size_t dirty_index = 0; + for (const auto& handle : TrackedAccess::dirtyValues(map)) { + if (handle != nullptr && seen_handles.insert(handle.get()).second) { + snapshot->handles.emplace_back("dirty:" + std::to_string(dirty_index++), handle); + } + } + + return snapshot; +} + +class VersionedTest : public testing::Test { +protected: + void SetUp() override { TrackedValue::resetCounters(); } + + void TearDown() override { + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::constructedCount(), TrackedValue::destroyedCount()); + } +}; + +TEST_F(VersionedTest, VersionedValueFirstInsertedValueIsVisibleInPublishedVersion) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + + EXPECT_EQ(value.get(0), nullptr); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); +} + +TEST_F(VersionedTest, VersionedValueNewerValueShadowsOlderValueFromNewVersionOnward) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + EXPECT_EQ(TrackedValue::liveCount(), 2); +} + +TEST_F(VersionedTest, VersionedValueClearHidesOnlyFromThatVersionOnward) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.clear(2); + + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + EXPECT_EQ(value.get(2), nullptr); +} + +TEST_F(VersionedTest, VersionedValueRevertRestoresPendingClear) { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.clear(2); + + EXPECT_EQ(value.get(2), nullptr); + + value.revert(1, deferred); + + EXPECT_TRUE(deferred.empty()); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 1); +} + +TEST_F(VersionedTest, VersionedValueRevertDeletesUnpublishedReplacement) { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + value.revert(1, deferred); + + EXPECT_FALSE(deferred.empty()); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); +} + +TEST_F(VersionedTest, VersionedValueDeferredRevertDeletesReplacementOnBatchDestruction) { + { + VersionedValue value; + TrackedDeferredDeletion deferred; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + value.revert(1, deferred); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +TEST_F(VersionedTest, VersionedValueGcDefersDeletionUntilBatchDestruction) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + TrackedDeferredDeletion deferred; + EXPECT_FALSE(value.gcForVersion(1, deferred)); + + EXPECT_TRUE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + ASSERT_NE(value.get(1), nullptr); + EXPECT_EQ(value.get(1)->id(), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + } + + { + TrackedDeferredDeletion deferred; + EXPECT_TRUE(value.gcForVersion(2, deferred)); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + ASSERT_NE(value.get(2), nullptr); + EXPECT_EQ(value.get(2)->id(), 2); +} + +TEST_F(VersionedTest, DeferredDeletionMoveTransfersDeletionOwnership) { + VersionedValue value; + + value.set(1, new TrackedValue(1)); + value.set(2, new TrackedValue(2)); + + TrackedDeferredDeletion deferred; + EXPECT_TRUE(value.gcForVersion(2, deferred)); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + TrackedDeferredDeletion moved = std::move(deferred); + EXPECT_FALSE(moved.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, FirstPhaseGcKeepsCurrentAndHistoricalLookupsMemorySafe) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(validateHandleAtVersion("a", handle, version1), ""); + EXPECT_EQ(validateHandleAtVersion("a", handle, version2), ""); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, ConcurrentValidatorAcceptsMixedDeferredTailTraversal) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1, 1)); + map.insert("b", new TrackedValue(2, 1)); + map.publishNextVersion(); + + auto handle_a = map.find("a"); + auto handle_b = map.find("b"); + ASSERT_NE(handle_a, nullptr); + ASSERT_NE(handle_b, nullptr); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1, 2)); + map.insert("b", new TrackedValue(2, 2)); + const auto version2 = map.publishNextVersion(); + + const auto* stale_a = TrackedAccess::next(TrackedAccess::head(handle_a)); + const auto* stale_b = TrackedAccess::next(TrackedAccess::head(handle_b)); + ASSERT_NE(stale_a, nullptr); + ASSERT_NE(stale_b, nullptr); + + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + + EXPECT_EQ(validateHandleAtVersion("a", handle_a, version2), ""); + EXPECT_EQ(validateHandleAtVersion("b", handle_b, version2), ""); + EXPECT_EQ(validateNodeAtVersion("stale-a", stale_a, version2), ""); + EXPECT_EQ(validateNodeAtVersion("stale-b", stale_b, version2), ""); + } +} + +TEST_F(VersionedTest, VersionedMapGcRetainsDirtyHandleUntilAllFutureRemovalsAreReclaimed) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(3)); + const auto version3 = map.publishNextVersion(); + + EXPECT_EQ(TrackedValue::liveCount(), 3); + + { + auto deferred = map.gc(version2); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 3); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + ASSERT_NE(activeValue(map, "a", version2), nullptr); + EXPECT_EQ(activeValue(map, "a", version2)->id(), 2); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); + } + + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + + { + auto deferred = map.gc(version3); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); + ASSERT_NE(activeValue(map, "a", version3), nullptr); + EXPECT_EQ(activeValue(map, "a", version3)->id(), 3); +} + +TEST_F(VersionedTest, VersionedMapVersionStartsAtMin) { + TrackedMap map; + + EXPECT_EQ(map.getVersion(), versionMin); +} + +TEST_F(VersionedTest, VersionedMapPublishWithNoPendingChangesReturnsZero) { + TrackedMap map; + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); + + map.prepareNextVersion(); + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); +} + +TEST_F(VersionedTest, VersionedMapInsertNewKeyThenRevertRemovesItEntirely) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + + ASSERT_NE(activeValue(map, "a", 1), nullptr); + + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(map.find("a"), nullptr); +} + +TEST_F(VersionedTest, VersionedMapDeferredRevertDeletesNewKeyAfterBatchDestruction) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(map.find("a"), nullptr); + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); +} + +TEST_F(VersionedTest, VersionedMapUpdateExistingKeyAndRevertRestoresPublishedValue) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + EXPECT_EQ(map.getVersion(), version1); + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); +} + +TEST_F(VersionedTest, VersionedMapDeferredRevertDeletesReplacementAfterBatchDestruction) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + auto deferred = map.revert(); + + EXPECT_FALSE(deferred.empty()); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +TEST_F(VersionedTest, VersionedMapClearExistingKeyAndRevertRestoresPublishedValue) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + const auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.clear("a"); + + EXPECT_EQ(activeValue(map, "a"), nullptr); + + auto deferred = map.revert(); + + EXPECT_TRUE(deferred.empty()); + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a"), nullptr); + EXPECT_EQ(activeValue(map, "a")->id(), 1); +} + +TEST_F(VersionedTest, VersionedMapClearMissingKeyIsNoOp) { + TrackedMap map; + + map.prepareNextVersion(); + map.clear("missing"); + + EXPECT_EQ(map.publishNextVersion(), 0); + EXPECT_EQ(map.getVersion(), versionMin); + EXPECT_EQ(TrackedValue::liveCount(), 0); +} + +TEST_F(VersionedTest, VersionedMapSnapshotsIsolateKeysAndVersions) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.insert("b", new TrackedValue(10)); + auto published_version = map.publishNextVersion(); + const auto version1 = map.getVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + published_version = map.publishNextVersion(); + const auto version2 = map.getVersion(); + + ASSERT_NE(activeValue(map, "a", version1), nullptr); + EXPECT_EQ(activeValue(map, "a", version1)->id(), 1); + ASSERT_NE(activeValue(map, "a", version2), nullptr); + EXPECT_EQ(activeValue(map, "a", version2)->id(), 2); + ASSERT_NE(activeValue(map, "b", version1), nullptr); + EXPECT_EQ(activeValue(map, "b", version1)->id(), 10); + ASSERT_NE(activeValue(map, "b", version2), nullptr); + EXPECT_EQ(activeValue(map, "b", version2)->id(), 10); + + map.gc(published_version); +} + +TEST_F(VersionedTest, VersionedMapActiveUpdateReusesStableHandle) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + ASSERT_NE(handle->get(version2), nullptr); + EXPECT_EQ(handle->get(version2)->id(), 2); + + map.gc(version2); +} + +TEST_F(VersionedTest, VersionedMapClearAndReinsertSameKeyBeforePublishReusesStableHandle) { + TrackedMap map; + + map.prepareNextVersion(); + auto handle = map.insert("a", new TrackedValue(1)); + const auto version1 = map.publishNextVersion(); + + ASSERT_NE(handle, nullptr); + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + + map.prepareNextVersion(); + map.clear("a"); + map.insert("a", new TrackedValue(2)); + const auto version2 = map.publishNextVersion(); + + ASSERT_EQ(map.find("a"), handle); + ASSERT_NE(handle->get(version1), nullptr); + EXPECT_EQ(handle->get(version1)->id(), 1); + ASSERT_NE(handle->get(version2), nullptr); + EXPECT_EQ(handle->get(version2)->id(), 2); + + map.gc(version2); +} + +TEST_F(VersionedTest, VersionedMapRemovedHandleStaysEmptyAfterSameNameReAdd) { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + map.publishNextVersion(); + + auto old_handle = map.find("a"); + ASSERT_NE(old_handle, nullptr); + + map.prepareNextVersion(); + map.clear("a"); + const auto version2 = map.publishNextVersion(); + map.gc(version2); + + EXPECT_EQ(map.find("a"), nullptr); + EXPECT_EQ(old_handle->get(version2), nullptr); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + const auto version3 = map.publishNextVersion(); + + auto new_handle = map.find("a"); + ASSERT_NE(new_handle, nullptr); + EXPECT_NE(new_handle, old_handle); + ASSERT_NE(new_handle->get(version3), nullptr); + EXPECT_EQ(new_handle->get(version3)->id(), 2); + EXPECT_EQ(old_handle->get(version3), nullptr); + map.gc(version3); +} + +TEST_F(VersionedTest, VersionedMapValidatorsAcceptStablePublishedSnapshots) { + TrackedMap map; + std::array generations{}; + + auto next_value = [&](int key_id) { return new TrackedValue(key_id, ++generations[key_id]); }; + auto expect_snapshot_valid = [&](const std::string& phase) { + auto snapshot = makePublishedHandles(map); + EXPECT_EQ(validatePublishedSnapshot(*snapshot), "") << phase; + }; + + map.prepareNextVersion(); + map.insert("key-10", next_value(10)); + map.insert("key-11", next_value(11)); + const auto version1 = map.publishNextVersion(); + ASSERT_EQ(version1, 1); + expect_snapshot_valid("after initial publish"); + + // Exercise same-version churn on one handle: update, clear, and re-add in one transaction. + map.prepareNextVersion(); + map.insert("key-10", next_value(10)); // generation 2 + map.clear("key-10"); + map.insert("key-10", next_value(10)); // generation 3 + const auto version2 = map.publishNextVersion(); + ASSERT_EQ(version2, 2); + expect_snapshot_valid("after same-version clear and re-add publish"); + + // Exercise both the post-unlink/pre-deletion state and the stable post-deletion state. + { + auto deferred = map.gc(version2); + EXPECT_FALSE(deferred.empty()); + expect_snapshot_valid("after first-phase gc of same-version churn publish"); + } + expect_snapshot_valid("after deferred deletion of same-version churn publish"); + + map.prepareNextVersion(); + map.clear("key-11"); + map.insert("key-12", next_value(12)); + const auto version3 = map.publishNextVersion(); + ASSERT_EQ(version3, 3); + expect_snapshot_valid("after mixed clear and insert publish"); + + { + auto deferred = map.gc(version3); + expect_snapshot_valid("after first-phase final gc"); + } + expect_snapshot_valid("after final deferred deletion"); +} + +TEST_F(VersionedTest, VersionedMapChaosConcurrentReadersAndWriter) { + TrackedMap map; + + constexpr size_t k_reader_count = 4; + constexpr int k_key_count = 32; + constexpr int k_initial_key_count = 16; + constexpr int k_max_ops_per_transaction = 8; + constexpr int k_transaction_count = 2000; + constexpr uint32_t k_seed = 1337; + + std::array key_names; + std::array generations{}; + for (int i = 0; i < k_key_count; ++i) { + key_names[i] = "key-" + std::to_string(i); + } + + auto next_value = [&](int key_id) { return new TrackedValue(key_id, ++generations[key_id]); }; + + map.prepareNextVersion(); + for (int i = 0; i < k_initial_key_count; ++i) { + map.insert(key_names[i], next_value(i)); + } + auto published_version = map.publishNextVersion(); + ASSERT_GT(published_version, 0); + + uint64_t next_snapshot_id = 1; + std::shared_ptr published_snapshot_owner = + makePublishedHandles(map, next_snapshot_id++); + std::shared_ptr previous_snapshot_owner; + std::atomic published_snapshot{published_snapshot_owner.get()}; + std::array, k_reader_count> reader_quiesced_versions; + std::array, k_reader_count> reader_snapshot_ids; + for (auto& reader_version : reader_quiesced_versions) { + reader_version.store(versionMin, std::memory_order_relaxed); + } + for (auto& reader_snapshot_id : reader_snapshot_ids) { + reader_snapshot_id.store(0, std::memory_order_relaxed); + } + + std::atomic stop{false}; + std::atomic failed{false}; + Envoy::Thread::MutexBasicLockable failure_mutex; + std::string failure_message; + Envoy::Thread::MutexBasicLockable history_mutex; + std::vector recent_history; + + auto record_failure = [&](const std::string& message) { + bool expected = false; + if (failed.compare_exchange_strong(expected, true, std::memory_order_acq_rel)) { + Envoy::Thread::LockGuard lock(failure_mutex); + failure_message = message; + } + }; + + auto record_history = [&](std::string message) { + Envoy::Thread::LockGuard lock(history_mutex); + recent_history.push_back(std::move(message)); + constexpr size_t k_history_limit = 128; + if (recent_history.size() > k_history_limit) { + recent_history.erase(recent_history.begin(), + recent_history.begin() + (recent_history.size() - k_history_limit)); + } + }; + + std::vector readers; + readers.reserve(k_reader_count); + for (size_t i = 0; i < k_reader_count; ++i) { + readers.emplace_back([&, i]() { + while (!stop.load(std::memory_order_acquire)) { + const auto* snapshot = published_snapshot.load(std::memory_order_acquire); + if (snapshot == nullptr) { + continue; + } + if (const auto error = validatePublishedSnapshotAtVersion( + *snapshot, snapshot->version, ValidationMode::ConcurrentSafeInvisibleTail); + !error.empty()) { + record_failure("reader " + std::to_string(i) + + " snapshot_id=" + std::to_string(snapshot->snapshot_id) + ": " + error); + break; + } + reader_quiesced_versions[i].store(snapshot->version, std::memory_order_release); + reader_snapshot_ids[i].store(snapshot->snapshot_id, std::memory_order_release); + } + }); + } + + auto publish_snapshot = [&]() { + previous_snapshot_owner = std::move(published_snapshot_owner); + published_snapshot_owner = makePublishedHandles(map, next_snapshot_id++); + published_snapshot.store(published_snapshot_owner.get(), std::memory_order_release); + return published_snapshot_owner; + }; + + auto release_previous_snapshot = [&]() { previous_snapshot_owner.reset(); }; + + auto wait_for_readers_to_observe_snapshot = + [&](const std::shared_ptr& snapshot) { + while (!failed.load(std::memory_order_acquire)) { + bool all_quiesced = true; + for (size_t i = 0; i < k_reader_count; ++i) { + if (reader_quiesced_versions[i].load(std::memory_order_acquire) < snapshot->version || + reader_snapshot_ids[i].load(std::memory_order_acquire) < snapshot->snapshot_id) { + all_quiesced = false; + break; + } + } + if (all_quiesced) { + return true; + } + std::this_thread::yield(); + } + return false; + }; + + auto run_deferred_gc = [&](uint64_t version) { + record_history("gc phase1 " + std::to_string(version)); + auto deferred = map.gc(version); + if (deferred.empty()) { + return true; + } + + const auto post_gc_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(post_gc_snapshot)) { + return false; + } + release_previous_snapshot(); + record_history("gc phase2 " + std::to_string(version)); + return true; + }; + + auto build_active_set = [&]() { + std::array active{}; + for (int i = 0; i < k_key_count; ++i) { + active[i] = map.find(key_names[i]) != nullptr; + } + return active; + }; + + auto random_matching_key = [&](std::minstd_rand& rng, const std::array& active, + bool want_active) { + std::vector candidates; + candidates.reserve(k_key_count); + for (int i = 0; i < k_key_count; ++i) { + if (active[i] == want_active) { + candidates.push_back(i); + } + } + if (candidates.empty()) { + return -1; + } + return candidates[rng() % candidates.size()]; + }; + + std::minstd_rand rng(k_seed); + uint64_t transaction_index = 0; + while (transaction_index < k_transaction_count && !failed.load(std::memory_order_acquire)) { + auto active = build_active_set(); + const auto next_version = map.prepareNextVersion(); + ++transaction_index; + record_history("tx " + std::to_string(transaction_index) + + " prepare next=" + std::to_string(next_version)); + + const int op_count = 1 + (rng() % k_max_ops_per_transaction); + for (int op_index = 0; op_index < op_count; ++op_index) { + const bool has_active = std::ranges::any_of(active, [](bool is_active) { return is_active; }); + const bool has_absent = + std::ranges::any_of(active, [](bool is_active) { return !is_active; }); + + std::vector available_ops; + if (has_active) { + available_ops.push_back(0); // update existing + available_ops.push_back(1); // clear existing + available_ops.push_back(3); // clear and re-add + } + if (has_absent) { + available_ops.push_back(2); // insert absent + } + ASSERT_FALSE(available_ops.empty()); + + switch (available_ops[rng() % available_ops.size()]) { + case 0: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": update " + key_names[key_id] + " -> gen " + + std::to_string(generations[key_id] + 1)); + map.insert(key_names[key_id], next_value(key_id)); + break; + } + case 1: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": clear " + key_names[key_id]); + map.clear(key_names[key_id]); + active[key_id] = false; + break; + } + case 2: { + const int key_id = random_matching_key(rng, active, false); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": insert " + key_names[key_id] + " -> gen " + + std::to_string(generations[key_id] + 1)); + map.insert(key_names[key_id], next_value(key_id)); + active[key_id] = true; + break; + } + case 3: { + const int key_id = random_matching_key(rng, active, true); + ASSERT_GE(key_id, 0); + record_history("tx " + std::to_string(transaction_index) + " op " + + std::to_string(op_index) + ": clear+insert " + key_names[key_id] + + " -> gen " + std::to_string(generations[key_id] + 1)); + map.clear(key_names[key_id]); + map.insert(key_names[key_id], next_value(key_id)); + active[key_id] = true; + break; + } + default: + FAIL() << "unexpected operation"; + } + } + + if ((rng() % 5) == 0) { + record_history("tx " + std::to_string(transaction_index) + " revert"); + auto deferred = map.revert(); + if (!deferred.empty()) { + const auto reverted_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(reverted_snapshot)) { + break; + } + record_history("revert deferred delete"); + } + continue; + } + + published_version = map.publishNextVersion(); + record_history("tx " + std::to_string(transaction_index) + " publish -> " + + std::to_string(published_version)); + if (published_version == 0) { + continue; + } + + const auto next_snapshot = publish_snapshot(); + if (!wait_for_readers_to_observe_snapshot(next_snapshot)) { + break; + } + release_previous_snapshot(); + if (!run_deferred_gc(published_version)) { + break; + } + } + + if (!failed.load(std::memory_order_acquire)) { + const auto final_version = map.getVersion(); + const auto final_snapshot = publish_snapshot(); + if (final_version > versionMin) { + EXPECT_TRUE(wait_for_readers_to_observe_snapshot(final_snapshot)); + release_previous_snapshot(); + if (!failed.load(std::memory_order_acquire)) { + EXPECT_TRUE(run_deferred_gc(final_version)); + } + } + } + + stop.store(true, std::memory_order_release); + for (auto& reader : readers) { + reader.join(); + } + published_snapshot.store(nullptr, std::memory_order_release); + previous_snapshot_owner.reset(); + published_snapshot_owner.reset(); + + if (failed.load(std::memory_order_acquire)) { + Envoy::Thread::LockGuard lock(failure_mutex); + std::ostringstream history; + { + Envoy::Thread::LockGuard history_lock(history_mutex); + for (const auto& entry : recent_history) { + history << "\n " << entry; + } + } + FAIL() << "seed=" << k_seed << " " << failure_message << "\nrecent history:" << history.str(); + } +} + +TEST_F(VersionedTest, VersionedMapGcAndDestructorDeleteValuesExactlyOnce) { + { + TrackedMap map; + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(1)); + auto published_version = map.publishNextVersion(); + map.gc(published_version); + + map.prepareNextVersion(); + map.insert("a", new TrackedValue(2)); + published_version = map.publishNextVersion(); + + EXPECT_EQ(TrackedValue::liveCount(), 2); + + { + auto deferred = map.gc(published_version); + EXPECT_FALSE(deferred.empty()); + EXPECT_EQ(TrackedValue::liveCount(), 2); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 0); + } + + EXPECT_EQ(TrackedValue::liveCount(), 0); + EXPECT_EQ(TrackedValue::destroyedCountFor(1), 1); + EXPECT_EQ(TrackedValue::destroyedCountFor(2), 1); +} + +} // namespace