From b6e28d860f977ea95582772565ffbc212b88ac4e Mon Sep 17 00:00:00 2001 From: Maxim Muzafarov <3415046+Mmuzaf@users.noreply.github.com> Date: Mon, 10 Apr 2023 16:26:59 +0300 Subject: [PATCH 01/22] IGNITE-19266 Use maven_opts for sonar analysis (#10636) --- .github/workflows/sonar-branch.yml | 6 +++--- .github/workflows/sonar-pr-from-fork.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/sonar-branch.yml b/.github/workflows/sonar-branch.yml index 6a8628c536049..f7a4c98914f68 100644 --- a/.github/workflows/sonar-branch.yml +++ b/.github/workflows/sonar-branch.yml @@ -23,9 +23,6 @@ on: - master - 'ignite-[0-9].[0-9]+.[0-9]+*' -env: - MAVEN_OPTS: -Xms2g - jobs: sonarcloud: name: Sonar Analysis @@ -73,5 +70,8 @@ jobs: -Dsonar.login=${SONARCLOUD_TOKEN} -B -V env: + MAVEN_OPTS: "-XX:+UseG1GC -XX:InitialHeapSize=2g -XX:MaxHeapSize=6g -XX:+UseStringDeduplication" + SONAR_OPTS: "-XX:+UseG1GC -XX:InitialHeapSize=2g -XX:MaxHeapSize=6g -XX:+UseStringDeduplication" + JAVA_OPTS: "-XX:+UseG1GC -XX:InitialHeapSize=2g -XX:MaxHeapSize=6g -XX:+UseStringDeduplication" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONARCLOUD_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} diff --git a/.github/workflows/sonar-pr-from-fork.yml b/.github/workflows/sonar-pr-from-fork.yml index 8c1e1f78e08ca..f96b97987fabd 100644 --- a/.github/workflows/sonar-pr-from-fork.yml +++ b/.github/workflows/sonar-pr-from-fork.yml @@ -22,9 +22,6 @@ on: [pull_request_target] permissions: contents: read -env: - MAVEN_OPTS: -Xms2g - concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -83,5 +80,8 @@ jobs: -Dsonar.login=${SONARCLOUD_TOKEN} -B -V env: + MAVEN_OPTS: "-XX:+UseG1GC -XX:InitialHeapSize=2g -XX:MaxHeapSize=6g -XX:+UseStringDeduplication" + SONAR_OPTS: "-XX:+UseG1GC -XX:InitialHeapSize=2g -XX:MaxHeapSize=6g -XX:+UseStringDeduplication" + JAVA_OPTS: "-XX:+UseG1GC -XX:InitialHeapSize=2g -XX:MaxHeapSize=6g -XX:+UseStringDeduplication" GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} SONARCLOUD_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} From a3abaec130e5f17d298531d985183832d5152f5b Mon Sep 17 00:00:00 2001 From: liyujue <18624049226@163.com> Date: Tue, 11 Apr 2023 11:38:58 +0300 Subject: [PATCH 02/22] IGNITE-18260 Documentation: Fix thin client features - Fixes #10401. Signed-off-by: Aleksey Plekhanov (cherry picked from commit b6a1b9bc4bc76f3060f917c64866360ea9125978) --- docs/_docs/thin-client-comparison.csv | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/_docs/thin-client-comparison.csv b/docs/_docs/thin-client-comparison.csv index 8044052d1d07d..1c9bd884543f7 100644 --- a/docs/_docs/thin-client-comparison.csv +++ b/docs/_docs/thin-client-comparison.csv @@ -4,16 +4,16 @@ Scan Query with a filter,{yes},{yes},No,No,No,No SqlFieldsQuery,{yes},{yes},No,{yes},{yes},{yes} Binary Object API,{yes},{yes},No,No,{yes},{yes} Failover,{yes},{yes},{yes},{yes},{yes},{yes} -Async Operations,No,{yes},No,{yes},{yes},{yes} +Async Operations,{yes},{yes},No,{yes},{yes},{yes} SSL/TLS,{yes},{yes},{yes},{yes},{yes},{yes} Authentication,{yes},{yes},{yes},{yes},{yes},{yes} Partition Awareness,{yes},{yes},{yes},{yes},{yes},No Transactions,{yes},{yes},{yes},No,No,No Cluster API,{yes},{yes},No,No,No,No Compute API,{yes},{yes},{yes},No,No,No -Continuous queries,{yes},{yes},No,No,No,No +Continuous queries,{yes},{yes},{yes},No,No,No Service invocation,{yes},{yes},No,No,No,No -Server Discovery,No,{yes},No,No,No,No +Server Discovery,{yes},{yes},No,No,No,No Server Discovery in Kubernetes,{yes},No,No,No,No,No Data Streamer,No,{Yes},No,No,No,No Retry Policy,{yes},{yes},No,No,No,No \ No newline at end of file From 5fad42f001da67c51eca02dcf1cdf7fbe4f3810e Mon Sep 17 00:00:00 2001 From: liyujue Date: Tue, 11 Apr 2023 11:43:05 +0300 Subject: [PATCH 03/22] IGNITE-17892 Documentation: Fix class for cdcEnabled parameter - Fixes #10310. Signed-off-by: Aleksey Plekhanov (cherry picked from commit 60ba8b14a06a7d7aab87c2d3c9c49f0eb6d612d0) --- docs/_docs/persistence/change-data-capture.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/_docs/persistence/change-data-capture.adoc b/docs/_docs/persistence/change-data-capture.adoc index 846e389cb5a97..c491969b265bb 100644 --- a/docs/_docs/persistence/change-data-capture.adoc +++ b/docs/_docs/persistence/change-data-capture.adoc @@ -49,7 +49,7 @@ On startup event processing will be continued from the last saved state. [cols="20%,45%,35%",opts="header"] |=== |Name |Description | Default value -| `DataStorageConfiguration#cdcEnabled` | Flag to enable CDC on the server node. | `false` +| `DataRegionConfiguration#cdcEnabled` | Flag to enable CDC on the server node. | `false` | `DataStorageConfiguration#cdcWalPath` | Path to the CDC directory | `"db/wal/cdc"` | `DataStorageConfiguration#walForceArchiveTimeout` | Timeout to forcefully archive the WAL segment even it is not complete. | `-1` (disabled) |=== From eab71555b3a1eefd096b42af52618487452e85f9 Mon Sep 17 00:00:00 2001 From: Pavel Tupitsyn Date: Wed, 12 Apr 2023 07:18:08 +0300 Subject: [PATCH 04/22] IGNITE-19222 .NET: Add IgniteClientConfiguration.EnableClusterDiscovery (#10642) * Add `IgniteClientConfiguration.EnableClusterDiscovery`. * Allow any combinations of `EnableClusterDiscovery` and `EnablePartitionAwareness`: * Only `EnablePartitionAwareness`: use configured endpoints to route requests. * Only `EnableClusterDiscovery`: use discovered endpoints for failover, but maintain only one connection. * Both enabled: discover all, connect to all. * Both disabled: maintain one connection, use only configured endpoints for failover. (cherry picked from commit 3ecb062d91b21fb23e14a7cbe0805468aef3c398) --- .../Client/Cache/PartitionAwarenessTest.cs | 48 +---------- ...titionAwarenessWithClusterDiscoveryTest.cs | 79 +++++++++++++++++++ .../Cluster/ClientClusterDiscoveryTests.cs | 4 +- .../ClientClusterDiscoveryTestsBase.cs | 32 ++++++++ .../Client/IgniteClientConfiguration.cs | 22 ++++++ .../IgniteClientConfigurationSection.xsd | 5 ++ .../Impl/Client/ClientFailoverSocket.cs | 44 +++++++---- 7 files changed, 172 insertions(+), 62 deletions(-) create mode 100644 modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/PartitionAwarenessWithClusterDiscoveryTest.cs diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/PartitionAwarenessTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/PartitionAwarenessTest.cs index d6adb4929ad02..2ae51699bae5e 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/PartitionAwarenessTest.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/PartitionAwarenessTest.cs @@ -40,7 +40,7 @@ public class PartitionAwarenessTest : ClientTestBase private const int ServerCount = 3; /** */ - private ICacheClient _cache; + protected ICacheClient _cache; /// /// Initializes a new instance of the class. @@ -151,44 +151,6 @@ public void CachePut_UserDefinedTypeWithAffinityKey_ThrowsIgniteException() Assert.AreEqual(expected, ex.Message); } -#if NETCOREAPP // TODO: IGNITE-15710 - [Test] - public void CacheGet_NewNodeEnteredTopology_RequestIsRoutedToNewNode() - { - // Warm-up. - Assert.AreEqual(1, _cache.Get(1)); - - // Before topology change. - Assert.AreEqual(12, _cache.Get(12)); - Assert.AreEqual(1, GetClientRequestGridIndex()); - - Assert.AreEqual(14, _cache.Get(14)); - Assert.AreEqual(2, GetClientRequestGridIndex()); - - // After topology change. - var cfg = GetIgniteConfiguration(); - cfg.AutoGenerateIgniteInstanceName = true; - - using (Ignition.Start(cfg)) - { - TestUtils.WaitForTrueCondition(() => - { - // Keys 12 and 14 belong to a new node now (-1). - Assert.AreEqual(12, _cache.Get(12)); - if (GetClientRequestGridIndex() != -1) - { - return false; - } - - Assert.AreEqual(14, _cache.Get(14)); - Assert.AreEqual(-1, GetClientRequestGridIndex()); - - return true; - }, 6000); - } - } -#endif - [Test] [TestCase(1, 1)] [TestCase(2, 0)] @@ -496,8 +458,6 @@ public void AtomicLong_RequestIsRoutedToPrimaryNode( [TestCase("default-grp-partitioned-set", null, CacheMode.Partitioned, 4, 1)] [TestCase("custom-grp-partitioned-set", "testIgniteSet1", CacheMode.Partitioned, 1, 1)] [TestCase("custom-grp-partitioned-set", "testIgniteSet1", CacheMode.Partitioned, 3, 0)] - [TestCase("custom-grp-replicated-set", "testIgniteSet2", CacheMode.Replicated, 1, 1)] - [TestCase("custom-grp-replicated-set", "testIgniteSet2", CacheMode.Replicated, 3, 1)] public void IgniteSet_RequestIsRoutedToPrimaryNode( string name, string groupName, CacheMode cacheMode, int item, int gridIdx) { @@ -528,9 +488,6 @@ public void IgniteSet_RequestIsRoutedToPrimaryNode( [TestCase("custom-grp-partitioned-set-2", "testIgniteSetColocated1", CacheMode.Partitioned, 1, 1)] [TestCase("custom-grp-partitioned-set-2", "testIgniteSetColocated1", CacheMode.Partitioned, 2, 1)] [TestCase("custom-grp-partitioned-set-2", "testIgniteSetColocated1", CacheMode.Partitioned, 3, 1)] - [TestCase("custom-grp-replicated-set-2", "testIgniteSetColocated2", CacheMode.Replicated, 1, 1)] - [TestCase("custom-grp-replicated-set-2", "testIgniteSetColocated2", CacheMode.Replicated, 2, 1)] - [TestCase("custom-grp-replicated-set-2", "testIgniteSetColocated2", CacheMode.Replicated, 3, 1)] public void IgniteSetColocated_RequestIsRoutedToPrimaryNode( string name, string groupName, CacheMode cacheMode, int item, int gridIdx) { @@ -559,13 +516,14 @@ protected override IgniteClientConfiguration GetClientConfiguration() var cfg = base.GetClientConfiguration(); cfg.EnablePartitionAwareness = true; + cfg.EnableClusterDiscovery = false; cfg.Endpoints.Add(string.Format("{0}:{1}", IPAddress.Loopback, IgniteClientConfiguration.DefaultPort + 1)); cfg.Endpoints.Add(string.Format("{0}:{1}", IPAddress.Loopback, IgniteClientConfiguration.DefaultPort + 2)); return cfg; } - private int GetClientRequestGridIndex(string message = null, string prefix = null) + protected int GetClientRequestGridIndex(string message = null, string prefix = null) { message = message ?? "Get"; diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/PartitionAwarenessWithClusterDiscoveryTest.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/PartitionAwarenessWithClusterDiscoveryTest.cs new file mode 100644 index 0000000000000..7b9b4a29eb38b --- /dev/null +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cache/PartitionAwarenessWithClusterDiscoveryTest.cs @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +namespace Apache.Ignite.Core.Tests.Client.Cache +{ + using System.Linq; + using Apache.Ignite.Core.Client; + using NUnit.Framework; + + /// + /// Tests Partition Awareness functionality combined with Cluster Discovery. + /// + public class PartitionAwarenessWithClusterDiscoveryTest : PartitionAwarenessTest + { +#if NETCOREAPP // TODO: IGNITE-15710 + [Test] + public void CacheGet_NewNodeEnteredTopology_RequestIsRoutedToNewNode() + { + // Warm-up. + Assert.AreEqual(1, _cache.Get(1)); + + // Before topology change. + Assert.AreEqual(12, _cache.Get(12)); + Assert.AreEqual(1, GetClientRequestGridIndex()); + + Assert.AreEqual(14, _cache.Get(14)); + Assert.AreEqual(2, GetClientRequestGridIndex()); + + // After topology change. + var cfg = GetIgniteConfiguration(); + cfg.AutoGenerateIgniteInstanceName = true; + + using (Ignition.Start(cfg)) + { + TestUtils.WaitForTrueCondition(() => + { + // Keys 12 and 14 belong to a new node now (-1). + Assert.AreEqual(12, _cache.Get(12)); + if (GetClientRequestGridIndex() != -1) + { + return false; + } + + Assert.AreEqual(14, _cache.Get(14)); + Assert.AreEqual(-1, GetClientRequestGridIndex()); + + return true; + }, 6000); + } + } +#endif + + + protected override IgniteClientConfiguration GetClientConfiguration() + { + var cfg = base.GetClientConfiguration(); + + // Enable discovery and keep only one endpoint, let the client discover others. + cfg.EnableClusterDiscovery = true; + cfg.Endpoints = cfg.Endpoints.Take(1).ToList(); + + return cfg; + } + } +} diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cluster/ClientClusterDiscoveryTests.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cluster/ClientClusterDiscoveryTests.cs index dec7946b4c99d..bbab57ff46bb6 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cluster/ClientClusterDiscoveryTests.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cluster/ClientClusterDiscoveryTests.cs @@ -82,12 +82,14 @@ public void TestClientDiscoveryWithRandomTopologyChanges() /// Tests that originally known node can leave and client maintains connections to other cluster nodes. /// [Test] - public void TestClientMaintainsConnectionWhenOriginalNodeLeaves() + public void TestClientMaintainsConnectionWhenOriginalNodeLeaves( + [Values(true, false)] bool enablePartitionAwareness) { // Client knows about single server node initially. var ignite = Ignition.Start(GetIgniteConfiguration()); var cfg = GetClientConfiguration(); cfg.Endpoints = new[] {IPAddress.Loopback + ":10803"}; + cfg.EnablePartitionAwareness = enablePartitionAwareness; // Client starts and discovers other server nodes. var client = Ignition.StartClient(cfg); diff --git a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cluster/ClientClusterDiscoveryTestsBase.cs b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cluster/ClientClusterDiscoveryTestsBase.cs index 31617f0d526fd..349c518f0f580 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cluster/ClientClusterDiscoveryTestsBase.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core.Tests/Client/Cluster/ClientClusterDiscoveryTestsBase.cs @@ -60,6 +60,38 @@ public void TestClientWithOneEndpointDiscoversAllServers() AssertClientConnectionCount(client, 3); } } + + /// + /// Tests that client with one initial endpoint discovers all servers. + /// + [Test] + public void TestDisabledDiscovery() + { + var cfg = new IgniteClientConfiguration(GetClientConfiguration()) + { + EnableClusterDiscovery = false + }; + + using var client = Ignition.StartClient(cfg); + + AssertClientConnectionCount(client, 1); + } + + /// + /// Tests that client with one initial endpoint discovers all servers. + /// + [Test] + public void TestDisabledPartitionAwareness() + { + var cfg = new IgniteClientConfiguration(GetClientConfiguration()) + { + EnablePartitionAwareness = false + }; + + using var client = Ignition.StartClient(cfg); + + AssertClientConnectionCount(client, 1); + } #endif /// diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs index 87717ab40b613..c6754e750ee8b 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Client/IgniteClientConfiguration.cs @@ -60,6 +60,11 @@ public class IgniteClientConfiguration /// public const bool DefaultEnablePartitionAwareness = true; + /// + /// Default value of property. + /// + public const bool DefaultEnableClusterDiscovery = true; + /// /// Default socket timeout. /// @@ -84,6 +89,7 @@ public IgniteClientConfiguration() SocketTimeout = DefaultSocketTimeout; Logger = new ConsoleLogger(); EnablePartitionAwareness = DefaultEnablePartitionAwareness; + EnableClusterDiscovery = DefaultEnableClusterDiscovery; } /// @@ -130,6 +136,7 @@ public IgniteClientConfiguration(IgniteClientConfiguration cfg) : this() Endpoints = cfg.Endpoints == null ? null : cfg.Endpoints.ToList(); ReconnectDisabled = cfg.ReconnectDisabled; EnablePartitionAwareness = cfg.EnablePartitionAwareness; + EnableClusterDiscovery = cfg.EnableClusterDiscovery; Logger = cfg.Logger; ProtocolVersion = cfg.ProtocolVersion; @@ -239,10 +246,25 @@ public IgniteClientConfiguration(IgniteClientConfiguration cfg) : this() /// To do so, connection is established to every known server node at all times. /// /// When false: only one connection is established at a given moment to a random server node. + /// + /// See also . /// [DefaultValue(DefaultEnablePartitionAwareness)] public bool EnablePartitionAwareness { get; set; } + /// + /// Gets or sets a value indicating whether Cluster Discovery should be enabled. + /// + /// Default is true: Ignite will maintain an actual list of all server nodes in the cluster and connect to + /// them when necessary. + /// When is false, this list of nodes will be used for failover. + /// When is true, connections will be established to all known nodes. + /// + /// When false: Ignite client will connect only to nodes in the list. + /// + [DefaultValue(DefaultEnableClusterDiscovery)] + public bool EnableClusterDiscovery { get; set; } + /// /// Gets or sets the logger. /// Default is . Set to null to disable logging. diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd index 9b4a74016e164..bd9068da9ec42 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd +++ b/modules/platforms/dotnet/Apache.Ignite.Core/IgniteClientConfigurationSection.xsd @@ -335,6 +335,11 @@ Enables affinity-aware connection: client will establish connection to every known server and route requests to primary nodes for cache operations. + + + Enables cluster discovery: client will maintain an actual list of all server nodes and connect to all of them. + + Username to be used to connect to secured cluster. diff --git a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientFailoverSocket.cs b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientFailoverSocket.cs index 263f4f6ecc60f..21721468e7a2d 100644 --- a/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientFailoverSocket.cs +++ b/modules/platforms/dotnet/Apache.Ignite.Core/Impl/Client/ClientFailoverSocket.cs @@ -509,7 +509,12 @@ private void OnNewDefaultConnection() ); } - if (!_socket.Features.HasFeature(ClientBitmaskFeature.ClusterGroupGetNodesEndpoints)) + if (!_config.EnableClusterDiscovery) + { + _enableDiscovery = false; + } + + if (_enableDiscovery && !_socket.Features.HasFeature(ClientBitmaskFeature.ClusterGroupGetNodesEndpoints)) { _enableDiscovery = false; @@ -587,27 +592,34 @@ private void OnAffinityTopologyVersionChange(AffinityTopologyVersion affinityTop { _affinityTopologyVersion = affinityTopologyVersion; - if (_discoveryTopologyVersion < affinityTopologyVersion.Version &&_config.EnablePartitionAwareness) + if (!_config.EnablePartitionAwareness && !_enableDiscovery) + { + return; + } + + if (_discoveryTopologyVersion >= affinityTopologyVersion.Version) + { + return; + } + + ThreadPool.QueueUserWorkItem(_ => { - ThreadPool.QueueUserWorkItem(_ => + try { - try + lock (_topologyUpdateLock) { - lock (_topologyUpdateLock) + if (!_disposed) { - if (!_disposed) - { - DiscoverEndpoints(); - InitSocketMap(); - } + DiscoverEndpoints(); + InitSocketMap(); } } - catch (Exception e) - { - _logger.Log(LogLevel.Error, e, "Failed to update topology information"); - } - }); - } + } + catch (Exception e) + { + _logger.Log(LogLevel.Error, e, "Failed to update topology information"); + } + }); } /// From 472765858bb590b7f23fed1427fc739a6fa10093 Mon Sep 17 00:00:00 2001 From: Denis Kuznetsov Date: Fri, 14 Apr 2023 10:48:06 +0300 Subject: [PATCH 05/22] IGNITE-18766 Fix incorrect id check in ClusterGroupAdapter.forNodeId - Fixes #10640. Signed-off-by: Aleksey Plekhanov (cherry picked from commit 91e66669ffc11950fe4a4358c04893a71aa98953) --- .../internal/cluster/ClusterGroupAdapter.java | 2 +- .../internal/GridProjectionForCachesSelfTest.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupAdapter.java index b6d07ece7047b..880a2b0460f79 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/ClusterGroupAdapter.java @@ -458,7 +458,7 @@ else if (ids.size() == 1) { nodeIds = U.newHashSet(ids.length + 1); for (UUID id0 : ids) { - if (contains(id)) + if (contains(id0)) nodeIds.add(id0); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/GridProjectionForCachesSelfTest.java b/modules/core/src/test/java/org/apache/ignite/internal/GridProjectionForCachesSelfTest.java index e68b9a07472e4..ebec6bd4ddf0a 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/GridProjectionForCachesSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/GridProjectionForCachesSelfTest.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.UUID; import org.apache.ignite.Ignite; @@ -281,6 +282,19 @@ public void testProjections() throws Exception { } } + /** + * @throws Exception If failed. + */ + @Test + public void testProjectionWithBadId() { + ClusterNode locNode = ignite.cluster().localNode(); + + ClusterGroup prj = ignite.cluster().forNodeId(UUID.randomUUID(), locNode.id()); + Collection nodes = prj.nodes(); + + assertEquals(1, nodes.size()); + } + /** * */ From d9f6bfe283a528b889aaddc4701b25e31da78354 Mon Sep 17 00:00:00 2001 From: Slava Koptilin Date: Fri, 14 Apr 2023 11:29:55 +0300 Subject: [PATCH 06/22] IGNITE-19115 Fixed handling cache messages for recreated cache (#10618) (cherry picked from commit 910b7d5883711eb31224d03860f92e18470880b9) --- .../processors/cache/GridCacheAdapter.java | 2 +- .../processors/cache/GridCacheIoManager.java | 125 +++++- .../processors/cache/GridCacheMessage.java | 5 +- .../GridCachePartitionExchangeManager.java | 6 +- .../processors/cache/GridCacheProcessor.java | 6 + .../cache/distributed/dht/GridDhtCache.java | 2 - .../distributed/dht/GridDhtCacheAdapter.java | 8 +- .../dht/GridDhtTransactionalCacheAdapter.java | 117 ++--- .../dht/atomic/GridDhtAtomicCache.java | 74 ++- .../dht/colocated/GridDhtColocatedCache.java | 44 +- .../distributed/near/GridNearAtomicCache.java | 16 +- .../near/GridNearTransactionalCache.java | 30 +- .../GridCacheDistributedQueryManager.java | 34 +- .../cache/query/GridCacheQueryRequest.java | 6 +- .../CacheContinuousQueryManager.java | 17 +- .../cache/transactions/IgniteTxEntry.java | 33 +- .../cache/transactions/IgniteTxHandler.java | 48 +- .../cluster/GridClusterStateProcessor.java | 2 +- .../dht/IgniteCacheRecreateTest.java | 421 ++++++++++++++++++ .../GridCacheMessageSelfTest.java | 6 +- .../IgniteCacheRestartTestSuite2.java | 3 + 21 files changed, 742 insertions(+), 263 deletions(-) create mode 100755 modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheRecreateTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java index 9a5e378ced96b..55cd09b4e9410 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheAdapter.java @@ -646,7 +646,7 @@ protected final String stopInfo() { * * @throws IgniteCheckedException If callback failed. */ - protected void onKernalStart() throws IgniteCheckedException { + public void onKernalStart() throws IgniteCheckedException { // No-op. } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java index cb7bef39cf830..c9457994b1617 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheIoManager.java @@ -22,6 +22,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.UUID; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; @@ -102,6 +103,7 @@ import org.jetbrains.annotations.Nullable; import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE; +import static org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion.NONE; import static org.apache.ignite.internal.util.IgniteUtils.nl; /** @@ -117,6 +119,9 @@ public class GridCacheIoManager extends GridCacheSharedManagerAdapter { /** */ private static final int MAX_STORED_PENDING_MESSAGES = 100; + /** Common message handler identifier that does not correspond to any particular cache. */ + public static final int COMMON_MESSAGE_HANDLER_ID = 0; + /** Delay in milliseconds between retries. */ private long retryDelay; @@ -334,19 +339,27 @@ private void handleMessage(UUID nodeId, GridCacheMessage cacheMsg, MessageHandle try { int msgIdx = cacheMsg.lookupIndex(); + AffinityTopologyVersion msgTopVer = cacheMsg.topologyVersion(); + IgniteBiInClosure c = null; if (msgIdx >= 0) { - Map idxClsHandlers0 = msgHandlers.idxClsHandlers; + Map idxClsHandlers0 = msgHandlers.idxClsHandlers; - IgniteBiInClosure[] cacheClsHandlers = idxClsHandlers0.get(cacheMsg.handlerId()); + IndexedClassHandler cacheClsHandlers = idxClsHandlers0.get(cacheMsg.handlerId()); - if (cacheClsHandlers != null) - c = cacheClsHandlers[msgIdx]; + if (cacheClsHandlers != null && + (NONE.equals(msgTopVer) || !msgTopVer.before(cacheClsHandlers.startTopVer))) + c = cacheClsHandlers.hndls[msgIdx]; } - if (c == null) - c = msgHandlers.clsHandlers.get(new ListenerKey(cacheMsg.handlerId(), cacheMsg.getClass())); + if (c == null) { + RegularClassHandler rHnd = msgHandlers.clsHandlers.get( + new ListenerKey(cacheMsg.handlerId(), cacheMsg.getClass())); + + if (rHnd != null && (NONE.equals(msgTopVer) || !msgTopVer.before(rHnd.startTopVer))) + c = rHnd.hnd; + } if (c == null) { if (processMissedHandler(nodeId, cacheMsg)) @@ -365,10 +378,10 @@ private void handleMessage(UUID nodeId, GridCacheMessage cacheMsg, MessageHandle msg0.append(nl()).append("Registered listeners:"); - Map idxClsHandlers0 = msgHandlers.idxClsHandlers; + Map idxClsHandlers0 = msgHandlers.idxClsHandlers; - for (Map.Entry e : idxClsHandlers0.entrySet()) - msg0.append(nl()).append(e.getKey()).append("=").append(Arrays.toString(e.getValue())); + for (Map.Entry e : idxClsHandlers0.entrySet()) + msg0.append(nl()).append(e.getKey()).append("=").append(Arrays.toString(e.getValue().hndls)); if (cctx.kernalContext().isStopping()) { if (log.isDebugEnabled()) @@ -1402,12 +1415,30 @@ void sendNoRetry(ClusterNode node, */ public void addCacheHandler( int hndId, + AffinityTopologyVersion startTopVer, Class type, IgniteBiInClosure c ) { assert !type.isAssignableFrom(GridCacheGroupIdMessage.class) : type; - addHandler(hndId, type, c, cacheHandlers); + addHandler(hndId, startTopVer, type, c, cacheHandlers); + } + + /** + * Registers a new message handler for the given message {@code type}. + * This method is equivalent to {@link #addCacheHandler(int, AffinityTopologyVersion, Class, IgniteBiInClosure)} where is + * the handler id equals to {@link #COMMON_MESSAGE_HANDLER_ID} and deployment id is {@code null}. + * + * @param type Type of message. + * @param c Handler. + */ + public void addCacheHandler( + Class type, + IgniteBiInClosure c + ) { + assert !type.isAssignableFrom(GridCacheGroupIdMessage.class) : type; + + addHandler(COMMON_MESSAGE_HANDLER_ID, NONE, type, c, cacheHandlers); } /** @@ -1421,7 +1452,9 @@ public void addCacheHandler( int hndId, Class msgCls ) { - return cacheHandlers.clsHandlers.get(new ListenerKey(hndId, msgCls)); + RegularClassHandler clsHnd = cacheHandlers.clsHandlers.get(new ListenerKey(hndId, msgCls)); + + return (clsHnd != null) ? clsHnd.hnd : null; } /** @@ -1436,7 +1469,7 @@ public void addCacheGroupHandler( ) { assert !type.isAssignableFrom(GridCacheIdMessage.class) : type; - addHandler(hndId, type, c, grpHandlers); + addHandler(hndId, NONE, type, c, grpHandlers); } /** @@ -1447,6 +1480,7 @@ public void addCacheGroupHandler( */ private void addHandler( int hndId, + AffinityTopologyVersion startTopVer, Class type, IgniteBiInClosure c, MessageHandlers msgHandlers @@ -1454,16 +1488,16 @@ private void addHandler( int msgIdx = messageIndex(type); if (msgIdx != -1) { - Map idxClsHandlers0 = msgHandlers.idxClsHandlers; + Map idxClsHandlers0 = msgHandlers.idxClsHandlers; - IgniteBiInClosure[] cacheClsHandlers = idxClsHandlers0.compute(hndId, (key, clsHandlers) -> { + IndexedClassHandler cacheClsHandlers = idxClsHandlers0.compute(hndId, (key, clsHandlers) -> { if (clsHandlers == null) - clsHandlers = new IgniteBiInClosure[GridCacheMessage.MAX_CACHE_MSG_LOOKUP_INDEX]; + clsHandlers = new IndexedClassHandler(startTopVer); - if (clsHandlers[msgIdx] != null) + if (clsHandlers.hndls[msgIdx] != null || !Objects.equals(clsHandlers.startTopVer, startTopVer)) return null; - clsHandlers[msgIdx] = c; + clsHandlers.hndls[msgIdx] = c; return clsHandlers; }); @@ -1476,11 +1510,12 @@ private void addHandler( } else { ListenerKey key = new ListenerKey(hndId, type); + RegularClassHandler regHnd = new RegularClassHandler(startTopVer, (IgniteBiInClosure)c); - if (msgHandlers.clsHandlers.putIfAbsent(key, - (IgniteBiInClosure)c) != null) + if (msgHandlers.clsHandlers.putIfAbsent(key, regHnd) != null) { assert false : "Handler for class already registered [hndId=" + hndId + ", cls=" + type + ", old=" + msgHandlers.clsHandlers.get(key) + ", new=" + c + ']'; + } } IgniteLogger log0 = log; @@ -1694,16 +1729,62 @@ else if (msg instanceof GridCacheGroupIdMessage) X.println(">>> cacheGrpOrderedHandlersSize: " + grpHandlers.orderedHandlers.size()); } + /** */ + abstract static class MessageHandler { + /** Start topology version. */ + final AffinityTopologyVersion startTopVer; + + /** + * Creates a new message handler descriptor. + * + * @param startTopVer Start affinity topology version. + */ + MessageHandler(AffinityTopologyVersion startTopVer) { + this.startTopVer = startTopVer; + } + } + + /** */ + static class IndexedClassHandler extends MessageHandler { + /** Actual handlers. */ + final IgniteBiInClosure[] hndls = new IgniteBiInClosure[GridCacheMessage.MAX_CACHE_MSG_LOOKUP_INDEX]; + + /** + * Creates a new message handler descriptor. + * + * @param startTopVer Start affinity topology version. + */ + IndexedClassHandler(AffinityTopologyVersion startTopVer) { + super(startTopVer); + } + } + + /** */ + static class RegularClassHandler extends MessageHandler { + /** Actual handler. */ + IgniteBiInClosure hnd; + + /** + * Creates a new message handler descriptor. + * + * @param startTopVer Start affinity topology version. + */ + RegularClassHandler(AffinityTopologyVersion startTopVer, IgniteBiInClosure hnd) { + super(startTopVer); + + this.hnd = hnd; + } + } + /** * */ static class MessageHandlers { /** Indexed class handlers. */ - volatile Map idxClsHandlers = new ConcurrentHashMap<>(); + volatile Map idxClsHandlers = new ConcurrentHashMap<>(); /** Handler registry. */ - ConcurrentMap> - clsHandlers = new ConcurrentHashMap<>(); + ConcurrentMap clsHandlers = new ConcurrentHashMap<>(); /** Ordered handler registry. */ ConcurrentMap> orderedHandlers = diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java index a9685a1eb8d42..c655dc487ef9b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheMessage.java @@ -442,8 +442,11 @@ protected final void unmarshalTx(Iterable txEntries, assert ctx != null; if (txEntries != null) { - for (IgniteTxEntry e : txEntries) + for (IgniteTxEntry e : txEntries) { + e.prepareUnmarshal(ctx, topologyVersion(), near); + e.unmarshal(ctx, near, ldr); + } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java index 4115e56fa87fa..3957151db29be 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCachePartitionExchangeManager.java @@ -413,7 +413,7 @@ private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) { cctx.gridEvents().addDiscoveryEventListener(discoLsnr, EVT_NODE_JOINED, EVT_NODE_LEFT, EVT_NODE_FAILED, EVT_DISCOVERY_CUSTOM_EVT); - cctx.io().addCacheHandler(0, GridDhtPartitionsSingleMessage.class, + cctx.io().addCacheHandler(GridDhtPartitionsSingleMessage.class, new MessageHandler() { @Override public void onMessage(final ClusterNode node, final GridDhtPartitionsSingleMessage msg) { GridDhtPartitionExchangeId exchangeId = msg.exchangeId(); @@ -450,7 +450,7 @@ private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) { } }); - cctx.io().addCacheHandler(0, GridDhtPartitionsFullMessage.class, + cctx.io().addCacheHandler(GridDhtPartitionsFullMessage.class, new MessageHandler() { @Override public void onMessage(ClusterNode node, GridDhtPartitionsFullMessage msg) { if (msg.exchangeId() == null) { @@ -470,7 +470,7 @@ private void processEventInactive(DiscoveryEvent evt, DiscoCache cache) { } }); - cctx.io().addCacheHandler(0, GridDhtPartitionsSingleRequest.class, + cctx.io().addCacheHandler(GridDhtPartitionsSingleRequest.class, new MessageHandler() { @Override public void onMessage(ClusterNode node, GridDhtPartitionsSingleRequest msg) { processSinglePartitionRequest(node, msg); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index d8bbc50a7f70f..3ad724da2c155 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -2074,6 +2074,12 @@ private void finishRecovery( cacheContext.finishRecovery(cacheStartVer, updatedDescriptor); + if (isNearEnabled(cacheContext)) { + GridDhtCacheAdapter dht = cacheContext.near().dht(); + + dht.context().finishRecovery(cacheStartVer, updatedDescriptor); + } + if (cacheContext.config().getAtomicityMode() == TRANSACTIONAL_SNAPSHOT && groupContext.affinityNode()) sharedCtx.coordinators().ensureStarted(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCache.java index 7889163f1772f..455224bdd514a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCache.java @@ -76,8 +76,6 @@ public GridDhtCache(GridCacheContext ctx, GridCacheConcurrentMap map) { metrics.delegate(ctx.dht().near().metrics0()); ctx.dr().resetMetrics(); - - super.start(); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java index 557153c04956d..a785bb6bc80a0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java @@ -387,8 +387,12 @@ protected GridDhtCacheAdapter(GridCacheContext ctx, GridCacheConcurrentMap } /** {@inheritDoc} */ - @Override public void start() throws IgniteCheckedException { - ctx.io().addCacheHandler(ctx.cacheId(), GridCacheTtlUpdateRequest.class, + @Override public void onKernalStart() throws IgniteCheckedException { + super.onKernalStart(); + + assert !ctx.isRecoveryMode() : "Registering message handlers in recovery mode [cacheName=" + name() + ']'; + + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridCacheTtlUpdateRequest.class, (CI2)this::processTtlUpdateRequest); ctx.gridEvents().addLocalEventListener(discoLsnr, EVT_NODE_LEFT, EVT_NODE_FAILED); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java index fcd7ab1d3be60..5c2e241e247eb 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtTransactionalCacheAdapter.java @@ -141,125 +141,80 @@ protected GridDhtTransactionalCacheAdapter(GridCacheContext ctx, GridCache } /** {@inheritDoc} */ - @Override public void start() throws IgniteCheckedException { - super.start(); + @Override public void onKernalStart() throws IgniteCheckedException { + super.onKernalStart(); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearGetRequest.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearGetRequest req) { - processNearGetRequest(nodeId, req); - } - }); + assert !ctx.isRecoveryMode() : "Registering message handlers in recovery mode [cacheName=" + name() + ']'; - ctx.io().addCacheHandler(ctx.cacheId(), GridNearSingleGetRequest.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearSingleGetRequest req) { - processNearSingleGetRequest(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearGetRequest.class, + (CI2)this::processNearGetRequest); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearLockRequest.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearLockRequest req) { - processNearLockRequest(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearSingleGetRequest.class, + (CI2)this::processNearSingleGetRequest); - ctx.io().addCacheHandler(ctx.cacheId(), GridDhtLockRequest.class, new CI2() { - @Override public void apply(UUID nodeId, GridDhtLockRequest req) { - processDhtLockRequest(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearLockRequest.class, + (CI2)this::processNearLockRequest); - ctx.io().addCacheHandler(ctx.cacheId(), GridDhtLockResponse.class, new CI2() { - @Override public void apply(UUID nodeId, GridDhtLockResponse req) { - processDhtLockResponse(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridDhtLockRequest.class, + (CI2)this::processDhtLockRequest); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearUnlockRequest.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearUnlockRequest req) { - processNearUnlockRequest(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridDhtLockResponse.class, + (CI2)this::processDhtLockResponse); - ctx.io().addCacheHandler(ctx.cacheId(), GridDhtUnlockRequest.class, new CI2() { - @Override public void apply(UUID nodeId, GridDhtUnlockRequest req) { - processDhtUnlockRequest(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearUnlockRequest.class, + (CI2)this::processNearUnlockRequest); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxQueryEnlistRequest.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearTxQueryEnlistRequest req) { - processNearTxQueryEnlistRequest(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridDhtUnlockRequest.class, + (CI2)this::processDhtUnlockRequest); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxQueryEnlistResponse.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearTxQueryEnlistResponse req) { - processNearTxQueryEnlistResponse(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearTxQueryEnlistRequest.class, + (CI2)this::processNearTxQueryEnlistRequest); + + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearTxQueryEnlistResponse.class, + (CI2)this::processNearTxQueryEnlistResponse); - ctx.io().addCacheHandler(ctx.cacheId(), GridDhtForceKeysRequest.class, + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridDhtForceKeysRequest.class, new MessageHandler() { @Override public void onMessage(ClusterNode node, GridDhtForceKeysRequest msg) { processForceKeysRequest(node, msg); } }); - ctx.io().addCacheHandler(ctx.cacheId(), GridDhtForceKeysResponse.class, + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridDhtForceKeysResponse.class, new MessageHandler() { @Override public void onMessage(ClusterNode node, GridDhtForceKeysResponse msg) { processForceKeyResponse(node, msg); } }); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxQueryResultsEnlistRequest.class, - new CI2() { - @Override public void apply(UUID nodeId, GridNearTxQueryResultsEnlistRequest req) { - processNearTxQueryResultsEnlistRequest(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearTxQueryResultsEnlistRequest.class, + (CI2)this::processNearTxQueryResultsEnlistRequest); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxQueryResultsEnlistResponse.class, - new CI2() { - @Override public void apply(UUID nodeId, GridNearTxQueryResultsEnlistResponse req) { - processNearTxQueryResultsEnlistResponse(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearTxQueryResultsEnlistResponse.class, + (CI2)this::processNearTxQueryResultsEnlistResponse); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxEnlistRequest.class, - new CI2() { - @Override public void apply(UUID nodeId, GridNearTxEnlistRequest req) { - processNearTxEnlistRequest(nodeId, req); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearTxEnlistRequest.class, + (CI2)this::processNearTxEnlistRequest); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearTxEnlistResponse.class, - new CI2() { - @Override public void apply(UUID nodeId, GridNearTxEnlistResponse msg) { - processNearTxEnlistResponse(nodeId, msg); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridNearTxEnlistResponse.class, + (CI2)this::processNearTxEnlistResponse); - ctx.io().addCacheHandler(ctx.cacheId(), GridDhtTxQueryEnlistRequest.class, + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridDhtTxQueryEnlistRequest.class, new CI2() { @Override public void apply(UUID nodeId, GridDhtTxQueryEnlistRequest msg) { processDhtTxQueryEnlistRequest(nodeId, msg, false); } }); - ctx.io().addCacheHandler(ctx.cacheId(), GridDhtTxQueryFirstEnlistRequest.class, + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridDhtTxQueryFirstEnlistRequest.class, new CI2() { @Override public void apply(UUID nodeId, GridDhtTxQueryEnlistRequest msg) { processDhtTxQueryEnlistRequest(nodeId, msg, true); } }); - ctx.io().addCacheHandler(ctx.cacheId(), GridDhtTxQueryEnlistResponse.class, - new CI2() { - @Override public void apply(UUID nodeId, GridDhtTxQueryEnlistResponse msg) { - processDhtTxQueryEnlistResponse(nodeId, msg); - } - }); + ctx.io().addCacheHandler(ctx.cacheId(), ctx.startTopologyVersion(), GridDhtTxQueryEnlistResponse.class, + (CI2)this::processDhtTxQueryEnlistResponse); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java index 6b00da0ecfa45..c3b2da0a9b0e5 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/atomic/GridDhtAtomicCache.java @@ -241,44 +241,26 @@ else if (res.error() != null) { } /** {@inheritDoc} */ - @Override public void start() throws IgniteCheckedException { - super.start(); - - assert metrics != null : "Cache metrics instance isn't initialized."; + @Override public void onKernalStart() throws IgniteCheckedException { + super.onKernalStart(); - if (ctx.dht().near() != null) - metrics.delegate(ctx.dht().near().metrics0()); + assert !ctx.isRecoveryMode() : "Registering message handlers in recovery mode [cacheName=" + name() + ']'; ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridNearGetRequest.class, - new CI2() { - @Override public void apply( - UUID nodeId, - GridNearGetRequest req - ) { - processNearGetRequest( - nodeId, - req); - } - }); + (CI2)this::processNearGetRequest); ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridNearSingleGetRequest.class, - new CI2() { - @Override public void apply( - UUID nodeId, - GridNearSingleGetRequest req - ) { - processNearSingleGetRequest( - nodeId, - req); - } - }); + (CI2)this::processNearSingleGetRequest); ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridNearAtomicAbstractUpdateRequest.class, new CI2() { @Override public void apply( @@ -298,6 +280,7 @@ else if (res.error() != null) { ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridNearAtomicUpdateResponse.class, new CI2() { @Override public void apply( @@ -317,6 +300,7 @@ else if (res.error() != null) { ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridDhtAtomicAbstractUpdateRequest.class, new CI2() { @Override public void apply( @@ -336,6 +320,7 @@ else if (res.error() != null) { ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridDhtAtomicUpdateResponse.class, new CI2() { @Override public void apply( @@ -355,6 +340,7 @@ else if (res.error() != null) { ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridDhtAtomicDeferredUpdateResponse.class, new CI2() { @Override public void apply( @@ -374,6 +360,7 @@ else if (res.error() != null) { ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridDhtAtomicNearResponse.class, new CI2() { @Override public void apply(UUID uuid, GridDhtAtomicNearResponse msg) { @@ -388,6 +375,7 @@ else if (res.error() != null) { ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridNearAtomicCheckUpdateRequest.class, new CI2() { @Override public void apply(UUID uuid, GridNearAtomicCheckUpdateRequest msg) { @@ -402,6 +390,7 @@ else if (res.error() != null) { ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridDhtForceKeysRequest.class, new MessageHandler() { @Override public void onMessage(ClusterNode node, GridDhtForceKeysRequest msg) { @@ -411,6 +400,7 @@ else if (res.error() != null) { ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridDhtForceKeysResponse.class, new MessageHandler() { @Override public void onMessage(ClusterNode node, GridDhtForceKeysResponse msg) { @@ -421,34 +411,26 @@ else if (res.error() != null) { if (near == null) { ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridNearGetResponse.class, - new CI2() { - @Override public void apply( - UUID nodeId, - GridNearGetResponse res - ) { - processNearGetResponse( - nodeId, - res); - } - }); + (CI2)this::processNearGetResponse); ctx.io().addCacheHandler( ctx.cacheId(), + ctx.startTopologyVersion(), GridNearSingleGetResponse.class, - new CI2() { - @Override public void apply( - UUID nodeId, - GridNearSingleGetResponse res - ) { - processNearSingleGetResponse( - nodeId, - res); - } - }); + (CI2)this::processNearSingleGetResponse); } } + /** {@inheritDoc} */ + @Override public void start() throws IgniteCheckedException { + assert metrics != null : "Cache metrics instance isn't initialized."; + + if (ctx.dht().near() != null) + metrics.delegate(ctx.dht().near().metrics0()); + } + /** * @param near Near cache. */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java index f6601a79e8e17..4da88bef33ab7 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/colocated/GridDhtColocatedCache.java @@ -120,26 +120,32 @@ public GridDhtColocatedCache(GridCacheContext ctx, GridCacheConcurrentMap } /** {@inheritDoc} */ - @Override public void start() throws IgniteCheckedException { - super.start(); - - ctx.io().addCacheHandler(ctx.cacheId(), GridNearGetResponse.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearGetResponse res) { - processNearGetResponse(nodeId, res); - } - }); - - ctx.io().addCacheHandler(ctx.cacheId(), GridNearSingleGetResponse.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearSingleGetResponse res) { - processNearSingleGetResponse(nodeId, res); - } - }); + @Override public void onKernalStart() throws IgniteCheckedException { + super.onKernalStart(); + + assert !ctx.isRecoveryMode() : "Registering message handlers in recovery mode [cacheName=" + name() + ']'; + + ctx.io().addCacheHandler( + ctx.cacheId(), + ctx.startTopologyVersion(), + GridNearGetResponse.class, + (CI2)this::processNearGetResponse); + + ctx.io().addCacheHandler( + ctx.cacheId(), + ctx.startTopologyVersion(), + GridNearSingleGetResponse.class, + (CI2)this::processNearSingleGetResponse); + + ctx.io().addCacheHandler( + ctx.cacheId(), + ctx.startTopologyVersion(), + GridNearLockResponse.class, + (CI2)this::processNearLockResponse); + } - ctx.io().addCacheHandler(ctx.cacheId(), GridNearLockResponse.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearLockResponse res) { - processNearLockResponse(nodeId, res); - } - }); + /** {@inheritDoc} */ + @Override public void start() throws IgniteCheckedException { } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java index 6068f0fe6e068..2a3cf58b139a0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearAtomicCache.java @@ -104,14 +104,16 @@ public GridNearAtomicCache(GridCacheContext ctx) { } /** {@inheritDoc} */ - @Override public void start() throws IgniteCheckedException { - super.start(); + @Override public void onKernalStart() throws IgniteCheckedException { + super.onKernalStart(); - ctx.io().addCacheHandler(ctx.cacheId(), GridNearGetResponse.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearGetResponse res) { - processGetResponse(nodeId, res); - } - }); + assert !ctx.isRecoveryMode() : "Registering message handlers in recovery mode [cacheName=" + name() + ']'; + + ctx.io().addCacheHandler( + ctx.cacheId(), + ctx.startTopologyVersion(), + GridNearGetResponse.class, + (CI2)this::processGetResponse); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java index 5887b8d95d8c8..26789218f923e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/near/GridNearTransactionalCache.java @@ -86,20 +86,22 @@ public GridNearTransactionalCache(GridCacheContext ctx) { } /** {@inheritDoc} */ - @Override public void start() throws IgniteCheckedException { - super.start(); - - ctx.io().addCacheHandler(ctx.cacheId(), GridNearGetResponse.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearGetResponse res) { - processGetResponse(nodeId, res); - } - }); - - ctx.io().addCacheHandler(ctx.cacheId(), GridNearLockResponse.class, new CI2() { - @Override public void apply(UUID nodeId, GridNearLockResponse res) { - processLockResponse(nodeId, res); - } - }); + @Override public void onKernalStart() throws IgniteCheckedException { + super.onKernalStart(); + + assert !ctx.isRecoveryMode() : "Registering message handlers in recovery mode [cacheName=" + name() + ']'; + + ctx.io().addCacheHandler( + ctx.cacheId(), + ctx.startTopologyVersion(), + GridNearGetResponse.class, + (CI2)this::processGetResponse); + + ctx.io().addCacheHandler( + ctx.cacheId(), + ctx.startTopologyVersion(), + GridNearLockResponse.class, + (CI2)this::processLockResponse); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java index 0c6dbe3e1b973..f7b3f7a8b8fa3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheDistributedQueryManager.java @@ -54,6 +54,7 @@ import org.apache.ignite.lang.IgniteFuture; import org.apache.ignite.lang.IgniteReducer; import org.jetbrains.annotations.Nullable; + import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_LEFT; import static org.apache.ignite.internal.GridTopic.TOPIC_CACHE; @@ -98,26 +99,27 @@ public class GridCacheDistributedQueryManager extends GridCacheQueryManage }; /** Event listener. */ - private GridLocalEventListener lsnr; + private GridLocalEventListener lsnr = new GridLocalEventListener() { + /** {@inheritDoc} */ + @Override public void onEvent(Event evt) { + DiscoveryEvent discoEvt = (DiscoveryEvent)evt; - /** {@inheritDoc} */ - @Override public void start0() throws IgniteCheckedException { - super.start0(); + for (GridCacheDistributedQueryFuture fut : futs.values()) + fut.onNodeLeft(discoEvt.eventNode().id()); + } + }; - cctx.io().addCacheHandler(cctx.cacheId(), GridCacheQueryRequest.class, new CI2() { - @Override public void apply(UUID nodeId, GridCacheQueryRequest req) { - processQueryRequest(nodeId, req); - } - }); + /** {@inheritDoc} */ + @Override public void onKernalStart0() throws IgniteCheckedException { + super.onKernalStart0(); - lsnr = new GridLocalEventListener() { - @Override public void onEvent(Event evt) { - DiscoveryEvent discoEvt = (DiscoveryEvent)evt; + assert !cctx.isRecoveryMode() : "Registering message handlers in recovery mode [cacheName=" + cctx.name() + ']'; - for (GridCacheDistributedQueryFuture fut : futs.values()) - fut.onNodeLeft(discoEvt.eventNode().id()); - } - }; + cctx.io().addCacheHandler( + cctx.cacheId(), + cctx.startTopologyVersion(), + GridCacheQueryRequest.class, + (CI2)this::processQueryRequest); cctx.events().addListener(lsnr, EVT_NODE_LEFT, EVT_NODE_FAILED); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java index 0912db3c8765f..55e86fe9d246d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryRequest.java @@ -195,7 +195,7 @@ public static GridCacheQueryRequest startQueryRequest(GridCacheContext cct qry.includeMetadata(), qry.keepBinary(), qry.taskHash(), - cctx.startTopologyVersion(), + cctx.affinity().affinityTopologyVersion(), qry.mvccSnapshot(), // Force deployment anyway if scan query is used. cctx.deploymentEnabled() || deployFilterOrTransformer, @@ -220,7 +220,7 @@ public static GridCacheQueryRequest pageRequest(GridCacheContext cctx, lon false, qry.keepBinary(), qry.taskHash(), - cctx.startTopologyVersion(), + cctx.affinity().affinityTopologyVersion(), // Force deployment anyway if scan query is used. cctx.deploymentEnabled() || (qry.scanFilter() != null && cctx.gridDeploy().enabled()), qry.isDataPageScanEnabled()); @@ -236,7 +236,7 @@ public static GridCacheQueryRequest cancelRequest(GridCacheContext cctx, l return new GridCacheQueryRequest(cctx.cacheId(), reqId, fieldsQry, - cctx.startTopologyVersion(), + cctx.affinity().affinityTopologyVersion(), cctx.deploymentEnabled()); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java index 6d5253c2e9078..8fbd4c979ce16 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/continuous/CacheContinuousQueryManager.java @@ -158,11 +158,22 @@ protected GridTimeoutProcessor.CancelableTask getCancelableTask() { /** {@inheritDoc} */ @Override protected void start0() throws IgniteCheckedException { + super.start0(); + // Append cache name to the topic. topicPrefix = "CONTINUOUS_QUERY" + (cctx.name() == null ? "" : "_" + cctx.name()); + } + + /** {@inheritDoc} */ + @SuppressWarnings("unchecked") + @Override protected void onKernalStart0() throws IgniteCheckedException { + assert !cctx.isRecoveryMode() : "Registering message handlers in recovery mode [cacheName=" + cctx.name() + ']'; if (cctx.affinityNode()) { - cctx.io().addCacheHandler(cctx.cacheId(), CacheContinuousQueryBatchAck.class, + cctx.io().addCacheHandler( + cctx.cacheId(), + cctx.startTopologyVersion(), + CacheContinuousQueryBatchAck.class, new CI2() { @Override public void apply(UUID uuid, CacheContinuousQueryBatchAck msg) { CacheContinuousQueryListener lsnr = lsnrs.get(msg.routineId()); @@ -174,11 +185,7 @@ protected GridTimeoutProcessor.CancelableTask getCancelableTask() { cancelableTask = cctx.time().schedule(new BackupCleaner(lsnrs, cctx.kernalContext()), BACKUP_ACK_FREQ, BACKUP_ACK_FREQ); } - } - /** {@inheritDoc} */ - @SuppressWarnings("unchecked") - @Override protected void onKernalStart0() throws IgniteCheckedException { Iterable cfgs = cctx.config().getCacheEntryListenerConfigurations(); if (cfgs != null) { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java index 3780d842fc0a8..14647675e3d3b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxEntry.java @@ -29,6 +29,7 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.GridDirectTransient; import org.apache.ignite.internal.IgniteCodeGeneratingFail; +import org.apache.ignite.internal.processors.affinity.AffinityTopologyVersion; import org.apache.ignite.internal.processors.cache.CacheEntryPredicate; import org.apache.ignite.internal.processors.cache.CacheInvalidStateException; import org.apache.ignite.internal.processors.cache.CacheInvokeEntry; @@ -951,23 +952,23 @@ public void marshal(GridCacheSharedContext ctx, boolean transferExpiry) th } /** - * Unmarshalls entry. + * Prepares this entry to unmarshall. In particular, this method initialize a cache context. * * @param ctx Cache context. + * @param topVer Topology version that is used to validate a cache context. + * If this parameter is {@code null} then validation will be skipped. * @param near Near flag. - * @param clsLdr Class loader. * @throws IgniteCheckedException If un-marshalling failed. */ - public void unmarshal( + public void prepareUnmarshal( GridCacheSharedContext ctx, - boolean near, - ClassLoader clsLdr + AffinityTopologyVersion topVer, + boolean near ) throws IgniteCheckedException { - if (this.ctx == null) { GridCacheContext cacheCtx = ctx.cacheContext(cacheId); - if (cacheCtx == null) + if (cacheCtx == null || (topVer != null && topVer.before(cacheCtx.startTopologyVersion()))) throw new CacheInvalidStateException( "Failed to perform cache operation (cache is stopped), cacheId=" + cacheId); @@ -978,6 +979,24 @@ else if (!cacheCtx.isNear() && near) this.ctx = cacheCtx; } + } + + /** + * Unmarshalls entry. + * + * @param ctx Cache context. + * @param near Near flag. + * @param clsLdr Class loader. + * @throws IgniteCheckedException If un-marshalling failed. + */ + public void unmarshal( + GridCacheSharedContext ctx, + boolean near, + ClassLoader clsLdr + ) throws IgniteCheckedException { + + if (this.ctx == null) + prepareUnmarshal(ctx, null, near); CacheObjectValueContext coctx = this.ctx.cacheObjectContext(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java index 506b589b8e415..4f8ef80316e7b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/transactions/IgniteTxHandler.java @@ -108,6 +108,7 @@ import static org.apache.ignite.cache.CacheWriteSynchronizationMode.FULL_SYNC; import static org.apache.ignite.internal.managers.communication.GridIoPolicy.SYSTEM_POOL; import static org.apache.ignite.internal.managers.communication.GridIoPolicy.UTILITY_CACHE_POOL; +import static org.apache.ignite.internal.processors.cache.GridCacheIoManager.COMMON_MESSAGE_HANDLER_ID; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.NOOP; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.TRANSFORM; import static org.apache.ignite.internal.processors.cache.GridCacheUtils.isNearEnabled; @@ -134,9 +135,6 @@ * Isolated logic to process cache messages. */ public class IgniteTxHandler { - /** */ - private static final int TX_MSG_HND_ID = 0; - /** Logger. */ private IgniteLogger log; @@ -222,80 +220,68 @@ public IgniteTxHandler(GridCacheSharedContext ctx) { txPrepareMsgLog = ctx.logger(CU.TX_MSG_PREPARE_LOG_CATEGORY); txFinishMsgLog = ctx.logger(CU.TX_MSG_FINISH_LOG_CATEGORY); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridNearTxPrepareRequest.class, new CI2() { + ctx.io().addCacheHandler(GridNearTxPrepareRequest.class, new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { processNearTxPrepareRequest(nodeId, (GridNearTxPrepareRequest)msg); } }); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridNearTxPrepareResponse.class, new CI2() { + ctx.io().addCacheHandler(GridNearTxPrepareResponse.class, new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { processNearTxPrepareResponse(nodeId, (GridNearTxPrepareResponse)msg); } }); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridNearTxFinishRequest.class, new CI2() { + ctx.io().addCacheHandler(GridNearTxFinishRequest.class, new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { processNearTxFinishRequest(nodeId, (GridNearTxFinishRequest)msg); } }); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridNearTxFinishResponse.class, new CI2() { + ctx.io().addCacheHandler(GridNearTxFinishResponse.class, new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { processNearTxFinishResponse(nodeId, (GridNearTxFinishResponse)msg); } }); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxPrepareRequest.class, new CI2() { + ctx.io().addCacheHandler(GridDhtTxPrepareRequest.class, new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { processDhtTxPrepareRequest(nodeId, (GridDhtTxPrepareRequest)msg); } }); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxPrepareResponse.class, new CI2() { + ctx.io().addCacheHandler(GridDhtTxPrepareResponse.class, new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { processDhtTxPrepareResponse(nodeId, (GridDhtTxPrepareResponse)msg); } }); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxFinishRequest.class, new CI2() { + ctx.io().addCacheHandler(GridDhtTxFinishRequest.class, new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { processDhtTxFinishRequest(nodeId, (GridDhtTxFinishRequest)msg); } }); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxOnePhaseCommitAckRequest.class, new CI2() { + ctx.io().addCacheHandler(GridDhtTxOnePhaseCommitAckRequest.class, new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { processDhtTxOnePhaseCommitAckRequest(nodeId, (GridDhtTxOnePhaseCommitAckRequest)msg); } }); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridDhtTxFinishResponse.class, new CI2() { + ctx.io().addCacheHandler(GridDhtTxFinishResponse.class, new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { processDhtTxFinishResponse(nodeId, (GridDhtTxFinishResponse)msg); } }); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridCacheTxRecoveryRequest.class, - new CI2() { - @Override public void apply(UUID nodeId, GridCacheTxRecoveryRequest req) { - processCheckPreparedTxRequest(nodeId, req); - } - }); + ctx.io().addCacheHandler(GridCacheTxRecoveryRequest.class, + (CI2)this::processCheckPreparedTxRequest); - ctx.io().addCacheHandler(TX_MSG_HND_ID, GridCacheTxRecoveryResponse.class, - new CI2() { - @Override public void apply(UUID nodeId, GridCacheTxRecoveryResponse res) { - processCheckPreparedTxResponse(nodeId, res); - } - }); + ctx.io().addCacheHandler(GridCacheTxRecoveryResponse.class, + (CI2)this::processCheckPreparedTxResponse); - ctx.io().addCacheHandler(TX_MSG_HND_ID, IncrementalSnapshotAwareMessage.class, - new CI2() { - @Override public void apply(UUID nodeId, IncrementalSnapshotAwareMessage msg) { - processIncrementalSnapshotAwareMessage(nodeId, msg); - } - }); + ctx.io().addCacheHandler(IncrementalSnapshotAwareMessage.class, + (CI2)this::processIncrementalSnapshotAwareMessage); } /** */ @@ -310,7 +296,7 @@ private void processIncrementalSnapshotAwareMessage(UUID nodeId, IncrementalSnap GridCacheMessage cacheMsg = msg.payload(); ctx.io() - .cacheHandler(TX_MSG_HND_ID, cacheMsg.getClass()) + .cacheHandler(COMMON_MESSAGE_HANDLER_ID, cacheMsg.getClass()) .apply(nodeId, cacheMsg); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java index 14ac86ef782a2..da485b02294d1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cluster/GridClusterStateProcessor.java @@ -892,7 +892,7 @@ protected IgniteCheckedException concurrentStateChangeError(ClusterState state, sharedCtx = cacheProc.context(); sharedCtx.io().addCacheHandler( - 0, GridChangeGlobalStateMessageResponse.class, + GridChangeGlobalStateMessageResponse.class, this::processChangeGlobalStateResponse ); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheRecreateTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheRecreateTest.java new file mode 100755 index 0000000000000..aca4448f420c7 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/distributed/dht/IgniteCacheRecreateTest.java @@ -0,0 +1,421 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.cache.distributed.dht; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.TreeSet; +import java.util.concurrent.TimeUnit; +import javax.cache.CacheException; +import javax.cache.processor.EntryProcessorException; +import javax.cache.processor.MutableEntry; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.CacheAtomicityMode; +import org.apache.ignite.cache.CacheEntryProcessor; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.cache.query.QueryCursor; +import org.apache.ignite.cache.query.ScanQuery; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.IgniteInternalFuture; +import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.GridCacheIdMessage; +import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicAbstractUpdateRequest; +import org.apache.ignite.internal.processors.cache.distributed.dht.atomic.GridNearAtomicFullUpdateRequest; +import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsFullMessage; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearGetRequest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearLockRequest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearSingleGetRequest; +import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest; +import org.apache.ignite.internal.processors.cache.query.GridCacheQueryRequest; +import org.apache.ignite.internal.util.typedef.X; +import org.apache.ignite.internal.util.typedef.internal.CU; +import org.apache.ignite.lang.IgniteBiInClosure; +import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.apache.ignite.transactions.Transaction; +import org.junit.Test; + +import static org.apache.ignite.cache.CacheAtomicityMode.ATOMIC; +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.testframework.GridTestUtils.runAsync; +import static org.apache.ignite.transactions.TransactionConcurrency.PESSIMISTIC; +import static org.apache.ignite.transactions.TransactionIsolation.SERIALIZABLE; + +/** + * Tests handling of ppending cache messages/operations when the required cache was re created. + * */ +public class IgniteCacheRecreateTest extends GridCommonAbstractTest { + /** Cache name to be used in tests. */ + private static final String CACHE_NAME = "test-recreate-cache"; + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setCommunicationSpi(new TestRecordingCommunicationSpi()); + + return cfg; + } + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrids(1); + + startClientGrid(1); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + grid(0).destroyCache(CACHE_NAME); + + awaitPartitionMapExchange(); + } + + /** {@inheritDoc} */ + @Override protected void afterTestsStopped() throws Exception { + super.afterTestsStopped(); + + stopAllGrids(); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testAtomicPutAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + ATOMIC, + GridNearAtomicAbstractUpdateRequest.class, + (cache, keys) -> cache.put(keys.get(0), 42)); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testAtomicGetAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + ATOMIC, + GridNearSingleGetRequest.class, + (cache, keys) -> cache.get(keys.get(0))); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testAtomicPutAllAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + ATOMIC, + GridNearAtomicFullUpdateRequest.class, + (cache, keys) -> { + Map vals = new TreeMap<>(); + vals.put(keys.get(0), 24); + vals.put(keys.get(1), 42); + + cache.putAll(vals); + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testAtomicGetAllAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + ATOMIC, + GridNearGetRequest.class, + (cache, keys) -> { + Set vals = new TreeSet<>(); + vals.add(keys.get(0)); + vals.add(keys.get(1)); + + cache.getAll(vals); + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testImplicitInvokeAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + ATOMIC, + GridNearAtomicAbstractUpdateRequest.class, + (cache, keys) -> { + cache.invoke(keys.get(0), new CacheEntryProcessor() { + @Override public Integer process( + MutableEntry entry, + Object... arguments + ) throws EntryProcessorException { + if (entry.exists()) + return entry.getValue(); + + entry.setValue(123); + + return entry.getValue(); + } + }); + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testImplicitOptimisticTxPutAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + TRANSACTIONAL, + GridNearTxPrepareRequest.class, + (cache, keys) -> cache.put(keys.get(0), 42)); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testImplicitOptimisticTxGetAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + TRANSACTIONAL, + GridNearSingleGetRequest.class, + (cache, keys) -> cache.get(keys.get(0))); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testImplicitOptimisticTxPutAllAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + TRANSACTIONAL, + GridNearTxPrepareRequest.class, + (cache, keys) -> { + Map vals = new TreeMap<>(); + vals.put(keys.get(0), 24); + vals.put(keys.get(1), 42); + + cache.putAll(vals); + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testImplicitTxInvokeAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + TRANSACTIONAL, + GridNearTxPrepareRequest.class, + (cache, keys) -> { + cache.invoke(keys.get(0), new CacheEntryProcessor() { + @Override public Integer process( + MutableEntry entry, + Object... arguments + ) throws EntryProcessorException { + if (entry.exists()) + return entry.getValue(); + + entry.setValue(123); + + return entry.getValue(); + } + }); + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testPessimisticTxPutAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + TRANSACTIONAL, + GridNearLockRequest.class, + (cache, keys) -> { + try (Transaction tx = grid(1).transactions().txStart(PESSIMISTIC, SERIALIZABLE)) { + cache.put(keys.get(0), 42); + + tx.commit(); + } + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testPessimisticTxPutAllAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + TRANSACTIONAL, + GridNearLockRequest.class, + (cache, keys) -> { + try (Transaction tx = grid(1).transactions().txStart(PESSIMISTIC, SERIALIZABLE)) { + Map vals = new TreeMap<>(); + vals.put(keys.get(0), 24); + vals.put(keys.get(1), 42); + + cache.putAll(vals); + + tx.commit(); + } + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testPessimisticTxGetAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + TRANSACTIONAL, + GridNearLockRequest.class, + (cache, keys) -> { + try (Transaction tx = grid(1).transactions().txStart(PESSIMISTIC, SERIALIZABLE)) { + cache.get(keys.get(0)); + + tx.commit(); + } + }); + } + + /** + * @throws Exception If failed. + */ + @Test + public void testAtomicScanAndCacheRecreate() throws Exception { + testCacheOperationAndCacheRecreate( + ATOMIC, + GridCacheQueryRequest.class, + (cache, keys) -> { + ScanQuery scanQuery = new ScanQuery<>(); + scanQuery.setPageSize(1); + + try (QueryCursor qry = cache.query(scanQuery)) { + for (Object o : qry.getAll()) { + IgniteBiTuple tuple = (IgniteBiTuple)o; + + throw new RuntimeException("Succesfully read unexpected value [k=" + tuple.getKey() + ", v=" + tuple.getValue()); + } + } + }); + } + + /** + * + * @param mode Cache atomicity mode. + * @param clazz Cache message type to be blocked before re-creating a cache. + * @param cacheOp Cache operation. + * @throws Exception If failed. + */ + private void testCacheOperationAndCacheRecreate( + CacheAtomicityMode mode, + Class clazz, + IgniteBiInClosure, List> cacheOp + ) throws Exception { + IgniteEx g0 = grid(0); + IgniteEx client = grid(1); + + // Initial loading. + IgniteCache clientCache = createCache(client, mode); + for (int i = 0; i < 100; i++) + clientCache.put(i, i); + + TestRecordingCommunicationSpi clientSpi = TestRecordingCommunicationSpi.spi(client); + TestRecordingCommunicationSpi crdSpi = TestRecordingCommunicationSpi.spi(g0); + + // Block cache operation. + clientSpi.blockMessages((node, msg) -> { + if (clazz.isAssignableFrom(msg.getClass())) { + GridCacheIdMessage msg0 = (GridCacheIdMessage)msg; + + if (msg0.cacheId() == 0 || msg0.cacheId() == CU.cacheId(CACHE_NAME)) + return true; + } + + return false; + }); + + // Block notifying the client node about upcoming changes. + crdSpi.blockMessages((node, msg) -> { + if (msg instanceof GridDhtPartitionsFullMessage) + return true; + + return false; + }); + + List primaryKeys = primaryKeys(g0.cache(CACHE_NAME), 2, 1); + + // Initiate cache operation. + IgniteInternalFuture updFut = runAsync(() -> cacheOp.apply(clientCache, primaryKeys)); + + // Wait for operation is initiated on the client node. + clientSpi.waitForBlocked(); + + // Destoy the existing cache and re-create it once again in order to deliver the blocked cache message to the server node + // when the reqired cache is destroyed and new cache handlers are registered. + g0.destroyCache(clientCache.getName()); + + // Create a new cache with the same name. + IgniteCache newCache = createCache(g0, mode); + + // Upload new values. + for (int i = 0; i < 100; i++) + newCache.put(i, i + 1_000); + + // Unblock cache operation. + clientSpi.stopBlock(); + + try { + updFut.get(10, TimeUnit.SECONDS); + + fail("Exception was not thrown."); + } + catch (Exception e) { + assertTrue("Unexpected exception [err=" + e + ']', X.hasCause(e, CacheException.class)); + } + finally { + crdSpi.stopBlock(); + } + } + + /** + * Creates a cache using the given node as initiator node and the given atomicity mode. + * + * @param ignite Node to be used to initiate creating a new cache. + * @param mode Cache atomicity mode. + * @return Ignite cache. + */ + private IgniteCache createCache(IgniteEx ignite, CacheAtomicityMode mode) { + CacheConfiguration cfg = new CacheConfiguration<>(CACHE_NAME); + + cfg.setBackups(1) + .setReadFromBackup(false) + .setAtomicityMode(mode) + .setAffinity(new RendezvousAffinityFunction(false, 32)); + + return ignite.getOrCreateCache(cfg); + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java index b6c14912874bb..09dc5e841b54a 100644 --- a/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java +++ b/modules/core/src/test/java/org/apache/ignite/spi/communication/GridCacheMessageSelfTest.java @@ -121,14 +121,16 @@ public void testSendBadMessage() throws Exception { IgniteEx ignite1 = grid(1); ignite0.context().cache().context().io().addCacheHandler( - 0, TestBadMessage.class, new CI2() { + TestBadMessage.class, + new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { throw new RuntimeException("Test bad message exception"); } }); ignite1.context().cache().context().io().addCacheHandler( - 0, TestBadMessage.class, new CI2() { + TestBadMessage.class, + new CI2() { @Override public void apply(UUID nodeId, GridCacheMessage msg) { throw new RuntimeException("Test bad message exception"); } diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java index 354c26c048dcc..2a9c2c6208f4a 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheRestartTestSuite2.java @@ -23,6 +23,7 @@ import org.apache.ignite.internal.processors.cache.distributed.IgniteBinaryMetadataUpdateNodeRestartTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheAtomicNodeRestartTest; import org.apache.ignite.internal.processors.cache.distributed.IgniteCacheGetRestartTest; +import org.apache.ignite.internal.processors.cache.distributed.dht.IgniteCacheRecreateTest; import org.apache.ignite.internal.processors.cache.distributed.replicated.IgniteCacheAtomicReplicatedNodeRestartSelfTest; import org.junit.runner.RunWith; import org.junit.runners.Suite; @@ -41,6 +42,8 @@ IgniteBinaryMetadataUpdateNodeRestartTest.class, + IgniteCacheRecreateTest.class, + IgniteCacheGetRestartTest.class }) public class IgniteCacheRestartTestSuite2 { From d321c5a1eb65acc3d2f999bfee833a742c706a43 Mon Sep 17 00:00:00 2001 From: Maksim Timonin Date: Thu, 13 Apr 2023 16:25:51 +0300 Subject: [PATCH 07/22] IGNITE-19279 Fix AssertionError on inactive cluster for thin client cache request (#10645) (cherry picked from commit 2b1406e00fdd9c226b60697fde7e3b3fc176b121) --- .../cache/ClientCachePartitionsRequest.java | 3 + .../thin/InactiveClusterCacheRequestTest.java | 88 +++++++++++++++++++ .../apache/ignite/client/ClientTestSuite.java | 2 + 3 files changed, 93 insertions(+) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/client/thin/InactiveClusterCacheRequestTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePartitionsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePartitionsRequest.java index 062d0ebf528d9..35740da13c610 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePartitionsRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePartitionsRequest.java @@ -82,6 +82,9 @@ public ClientCachePartitionsRequest(BinaryRawReader reader, ClientProtocolContex /** {@inheritDoc} */ @Override public ClientResponse process(ClientConnectionContext ctx) { + if (!ctx.kernalContext().state().publicApiActiveState(true)) + return new ClientResponse(requestId(), "Cannot perform cache operation because the cluster is inactive."); + Map grps = new HashMap<>(cacheIds.length); ClientAffinityTopologyVersion affinityVer = ctx.checkAffinityTopologyVersion(); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/client/thin/InactiveClusterCacheRequestTest.java b/modules/core/src/test/java/org/apache/ignite/internal/client/thin/InactiveClusterCacheRequestTest.java new file mode 100644 index 0000000000000..b1c87d18b9106 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/client/thin/InactiveClusterCacheRequestTest.java @@ -0,0 +1,88 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.client.thin; + +import java.util.List; +import org.apache.ignite.Ignition; +import org.apache.ignite.client.ClientException; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.cluster.ClusterState; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.testframework.GridTestUtils; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** */ +@RunWith(Parameterized.class) +public class InactiveClusterCacheRequestTest extends AbstractThinClientTest { + /** */ + @Parameterized.Parameter + public boolean partitionAwarenessEnabled; + + /** */ + @Parameterized.Parameters(name = "partitionAwareness={0}") + public static List params() { + return F.asList(false, true); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() { + stopAllGrids(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + cfg.setDataStorageConfiguration(new DataStorageConfiguration() + .setDefaultDataRegionConfiguration(new DataRegionConfiguration().setPersistenceEnabled(true))); + + cfg.setCacheConfiguration(new CacheConfiguration<>(DEFAULT_CACHE_NAME)); + + return cfg; + } + + /** */ + @Test + public void testCacheOperationReturnErrorOnInactiveCluster() throws Exception { + startGrids(2); + + ClientConfiguration ccfg = getClientConfiguration(grid(0), grid(1)) + .setPartitionAwarenessEnabled(partitionAwarenessEnabled); + + stopAllGrids(); + + IgniteEx ign = startGrid(0); + + assertTrue(ign.cluster().state() == ClusterState.INACTIVE); + + try (IgniteClient cln = Ignition.startClient(ccfg)) { + GridTestUtils.assertThrows(log, + () -> cln.cache(DEFAULT_CACHE_NAME).get(0), + ClientException.class, + "cluster is inactive"); + } + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java index 5c4c610f88243..cbb128e44a764 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java @@ -26,6 +26,7 @@ import org.apache.ignite.internal.client.thin.DataReplicationOperationsTest; import org.apache.ignite.internal.client.thin.FunctionalTest; import org.apache.ignite.internal.client.thin.IgniteSetTest; +import org.apache.ignite.internal.client.thin.InactiveClusterCacheRequestTest; import org.apache.ignite.internal.client.thin.MetadataRegistrationTest; import org.apache.ignite.internal.client.thin.OptimizedMarshallerClassesCachedTest; import org.apache.ignite.internal.client.thin.ReliableChannelTest; @@ -84,6 +85,7 @@ IgniteClientConnectionEventListenerTest.class, IgniteClientRequestEventListenerTest.class, ThinClientEnpointsDiscoveryTest.class, + InactiveClusterCacheRequestTest.class, }) public class ClientTestSuite { // No-op. From 6748078008588903d9c337a07eb5921cb1a4aad0 Mon Sep 17 00:00:00 2001 From: Nikita Amelchev Date: Mon, 17 Apr 2023 19:41:59 +0300 Subject: [PATCH 08/22] IGNITE-19286 Fixed NPE in case of simultaneous cache destroy and transaction rollback (#10647) (cherry picked from commit 378f0643117eca00f6081a08a783b9683f3adc19) --- .../processors/cache/GridCacheProcessor.java | 4 + .../StartImplicitlyTxOnStopCacheTest.java | 92 ++++++++++++++++++- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java index 3ad724da2c155..85b4bcef3d7f2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheProcessor.java @@ -2777,10 +2777,14 @@ private void processCacheStopRequestOnExchangeDone(AffinityTopologyVersion topVe } for (ExchangeActions.CacheActionData action : cachesToStopByGrp.getValue()) { + // Rollback tx started before gateway blocked to avoid deadlock with gateway stop. context().tm().rollbackTransactionsForStoppingCache(action.descriptor().cacheId()); stopGateway(action.request()); + // Rollback tx started after gateway blocked but not stopped. + context().tm().rollbackTransactionsForStoppingCache(action.descriptor().cacheId()); + String cacheName = action.request().cacheName(); GridCacheAdapter cache = caches.get(cacheName); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/StartImplicitlyTxOnStopCacheTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/StartImplicitlyTxOnStopCacheTest.java index 3305d84286bca..ea19f4214e11c 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/StartImplicitlyTxOnStopCacheTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/transactions/StartImplicitlyTxOnStopCacheTest.java @@ -18,32 +18,60 @@ package org.apache.ignite.internal.processors.cache.transactions; import java.util.concurrent.BrokenBarrierException; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.CyclicBarrier; +import java.util.concurrent.atomic.AtomicBoolean; +import javax.cache.CacheException; +import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.configuration.CacheConfiguration; import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.failure.AbstractFailureHandler; +import org.apache.ignite.failure.FailureContext; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.IgniteInternalFuture; import org.apache.ignite.internal.TestRecordingCommunicationSpi; +import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.cache.GridCacheGateway; +import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.GridDhtPartitionsExchangeFuture; import org.apache.ignite.internal.processors.cache.distributed.dht.preloader.PartitionsExchangeAware; import org.apache.ignite.internal.processors.cache.distributed.near.GridNearTxPrepareRequest; +import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.junit.Test; +import org.mockito.Mockito; + +import static org.apache.ignite.testframework.GridTestUtils.runAsync; +import static org.apache.ignite.testframework.GridTestUtils.setFieldValue; /** * The test starts an implicit transaction during the cache is stopping. * The transaction has to be completed, and the cache is stopped. */ public class StartImplicitlyTxOnStopCacheTest extends GridCommonAbstractTest { + /** */ + private static final String GROUP = "test-group"; + + /** Node failure occurs. */ + private final AtomicBoolean failure = new AtomicBoolean(); + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName) .setConsistentId(igniteInstanceName) .setCommunicationSpi(new TestRecordingCommunicationSpi()) + .setFailureHandler(new AbstractFailureHandler() { + @Override protected boolean handle(Ignite ignite, FailureContext failureCtx) { + failure.set(true); + + return true; + } + }) .setCacheConfiguration(new CacheConfiguration(DEFAULT_CACHE_NAME) + .setGroupName(GROUP) .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); return cfg; @@ -87,9 +115,9 @@ public void test() throws Exception { } }); - IgniteInternalFuture runTxFut = GridTestUtils.runAsync(() -> cache.put(100, 100)); + IgniteInternalFuture runTxFut = runAsync(() -> cache.put(100, 100)); - IgniteInternalFuture destroyCacheFut = GridTestUtils.runAsync(() -> + IgniteInternalFuture destroyCacheFut = runAsync(() -> client.destroyCache(DEFAULT_CACHE_NAME)); exchnageStartedBarrier.await(); @@ -100,4 +128,64 @@ public void test() throws Exception { assertNull(client.cache(DEFAULT_CACHE_NAME)); } + + /** @throws Exception If failed. */ + @Test + public void testTxStartAfterGatewayBlockedOnCacheDestroy() throws Exception { + IgniteEx crd = (IgniteEx)startGridsMultiThreaded(2); + + // Cache group with multiple caches are important here, partition topology will not be stopped on cache destroy. + crd.createCache(new CacheConfiguration<>(DEFAULT_CACHE_NAME + "_1") + .setGroupName(GROUP) + .setAtomicityMode(CacheAtomicityMode.TRANSACTIONAL)); + + GridCacheSharedContext sharedCtx = crd.context().cache().context(); + GridCacheContext cacheCtx = sharedCtx.cacheContext(CU.cacheId(DEFAULT_CACHE_NAME)); + + CountDownLatch txStarted = new CountDownLatch(1); + CountDownLatch gatewayStopped = new CountDownLatch(1); + + IgniteTxManager tm = Mockito.spy(sharedCtx.tm()); + sharedCtx.setTxManager(tm); + + Mockito.doAnswer(m -> { + // Create tx after gateway was stopped (but not blocked). + txStarted.countDown(); + gatewayStopped.await(); + + return m.callRealMethod(); + }).when(tm).onCreated(Mockito.any(), Mockito.any()); + + GridCacheGateway gate = Mockito.spy(cacheCtx.gate()); + setFieldValue(cacheCtx, "gate", gate); + + Mockito.doAnswer(m -> { + // Gateway is ready to block. + gatewayStopped.countDown(); + + return m.callRealMethod(); + }).when(gate).onStopped(); + + IgniteInternalFuture fut = runAsync(() -> { + txStarted.await(); + + grid(1).destroyCache(DEFAULT_CACHE_NAME); + }); + + try { + Integer remoteKey = primaryKey(grid(1).cache(DEFAULT_CACHE_NAME)); + + crd.cache(DEFAULT_CACHE_NAME).putAsync(remoteKey, "val").get(); + } + catch (CacheException e) { + // No-op. + } + + fut.get(); + + // Make sure exchange worker processed previous task to get potential failure. + crd.getOrCreateCache("new-cache"); + + assertFalse(failure.get()); + } } From d06ae350f51e7579bc304eaebb958f241700330d Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Tue, 18 Apr 2023 10:01:51 +0300 Subject: [PATCH 09/22] IGNITE-17345 Thin client: Add affinity hits/misses metric - Fixes #10649. Signed-off-by: Aleksey Plekhanov (cherry picked from commit 4633ca4f2cc88949579ce2de213252fa0a0f8471) --- .../_docs/monitoring-metrics/new-metrics.adoc | 4 + .../RunningQueryInfoCheckInitiatorTest.java | 2 +- .../thin/JdbcThinWalModeChangeSelfTest.java | 2 +- .../ignite/internal/GridKernalContext.java | 6 +- .../internal/GridKernalContextImpl.java | 8 +- .../meta/tasks/MetadataRemoveTask.java | 2 +- .../reader/StandaloneGridKernalContext.java | 2 +- .../odbc/ClientListenerMetrics.java | 66 +++++- .../odbc/ClientListenerNioListener.java | 11 +- .../odbc/ClientListenerProcessor.java | 14 +- .../platform/client/ClientRequestHandler.java | 2 +- .../cache/ClientCacheClearKeyRequest.java | 4 +- .../cache/ClientCacheContainsKeyRequest.java | 2 +- .../ClientCacheGetAndPutIfAbsentRequest.java | 2 +- .../cache/ClientCacheGetAndPutRequest.java | 2 +- .../cache/ClientCacheGetAndRemoveRequest.java | 2 +- .../ClientCacheGetAndReplaceRequest.java | 2 +- .../client/cache/ClientCacheGetRequest.java | 2 +- .../cache/ClientCacheIndexQueryRequest.java | 5 +- .../client/cache/ClientCacheKeyRequest.java | 38 ++++ .../cache/ClientCacheKeyValueRequest.java | 2 +- .../cache/ClientCacheLocalPeekRequest.java | 2 +- .../cache/ClientCachePutIfAbsentRequest.java | 2 +- .../client/cache/ClientCachePutRequest.java | 4 +- .../client/cache/ClientCacheQueryRequest.java | 57 +++++ .../ClientCacheRemoveIfEqualsRequest.java | 2 +- .../cache/ClientCacheRemoveKeyRequest.java | 2 +- .../ClientCacheReplaceIfEqualsRequest.java | 2 +- .../cache/ClientCacheReplaceRequest.java | 2 +- .../cache/ClientCacheScanQueryRequest.java | 5 +- .../ClientCacheSqlFieldsQueryRequest.java | 2 +- .../cache/ClientCacheSqlQueryRequest.java | 2 +- .../client/compute/ClientComputeTask.java | 2 +- .../internal/sql/SqlCommandProcessor.java | 4 +- .../client/VisorClientConnectionDropTask.java | 2 +- .../client/thin/AffinityMetricsTest.java | 213 ++++++++++++++++++ .../apache/ignite/client/ClientTestSuite.java | 2 + 37 files changed, 438 insertions(+), 47 deletions(-) create mode 100644 modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheQueryRequest.java create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/client/thin/AffinityMetricsTest.java diff --git a/docs/_docs/monitoring-metrics/new-metrics.adoc b/docs/_docs/monitoring-metrics/new-metrics.adoc index 42aedf50a6550..7eae8cd24da48 100644 --- a/docs/_docs/monitoring-metrics/new-metrics.adoc +++ b/docs/_docs/monitoring-metrics/new-metrics.adoc @@ -315,6 +315,10 @@ Register name: `client.connector` |SentBytesCount| long| Sent bytes count. |SslEnabled| boolean| Indicates whether SSL is enabled. |SslHandshakeDurationHistogram| histogram| Histogram of SSL handshake duration in milliseconds (metric is exported only if SSL is enabled). +|AffinityKeyRequestsHits| long| The number of affinity-aware cache key requests that were sent to the primary node. +|AffinityKeyRequestsMisses| long| The number of affinity-aware cache key requests that were sent not to the primary node. +|AffinityQueryRequestsHits| long| The number of affinity-aware query requests that were sent to the primary node. +|AffinityQueryRequestsMisses| long| The number of affinity-aware query requests that were sent not to the primary node. |=== diff --git a/modules/clients/src/test/java/org/apache/ignite/common/RunningQueryInfoCheckInitiatorTest.java b/modules/clients/src/test/java/org/apache/ignite/common/RunningQueryInfoCheckInitiatorTest.java index 38776d727c915..bc721296bd5eb 100644 --- a/modules/clients/src/test/java/org/apache/ignite/common/RunningQueryInfoCheckInitiatorTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/common/RunningQueryInfoCheckInitiatorTest.java @@ -364,7 +364,7 @@ private void checkRunningQueriesCount(IgniteEx node, int expectedQryCount, int t /** */ private static int clientPort(IgniteEx ign) { - return ign.context().sqlListener().port(); + return ign.context().clientListener().port(); } /** diff --git a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java index c94ead9134a44..718ead5502675 100644 --- a/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java +++ b/modules/clients/src/test/java/org/apache/ignite/jdbc/thin/JdbcThinWalModeChangeSelfTest.java @@ -135,7 +135,7 @@ private static void execute(Ignite node, String cmd) { private static Connection connect(Ignite node) throws Exception { IgniteKernal node0 = (IgniteKernal)node; - int port = node0.context().sqlListener().port(); + int port = node0.context().clientListener().port(); return DriverManager.getConnection("jdbc:ignite:thin://127.0.0.1:" + port); } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java index 7bf057e5fb19a..544b6b5d2eb9e 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContext.java @@ -341,11 +341,11 @@ public interface GridKernalContext extends Iterable { public GridQueryProcessor query(); /** - * Gets SQL listener processor. + * Gets client listener processor. * - * @return SQL listener processor. + * @return Client listener processor. */ - public ClientListenerProcessor sqlListener(); + public ClientListenerProcessor clientListener(); /** * @return Plugin processor. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java index 8f3b685dc14a4..18e70f0326ae6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/GridKernalContextImpl.java @@ -194,7 +194,7 @@ public class GridKernalContextImpl implements GridKernalContext, Externalizable /** */ @GridToStringInclude - private ClientListenerProcessor sqlListenerProc; + private ClientListenerProcessor clientListenerProc; /** */ @GridToStringInclude @@ -558,7 +558,7 @@ else if (comp instanceof IgnitePluginProcessor) else if (comp instanceof GridQueryProcessor) qryProc = (GridQueryProcessor)comp; else if (comp instanceof ClientListenerProcessor) - sqlListenerProc = (ClientListenerProcessor)comp; + clientListenerProc = (ClientListenerProcessor)comp; else if (comp instanceof DataStructuresProcessor) dataStructuresProc = (DataStructuresProcessor)comp; else if (comp instanceof ClusterProcessor) @@ -853,8 +853,8 @@ public void addHelper(Object helper) { } /** {@inheritDoc} */ - @Override public ClientListenerProcessor sqlListener() { - return sqlListenerProc; + @Override public ClientListenerProcessor clientListener() { + return clientListenerProc; } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/commandline/meta/tasks/MetadataRemoveTask.java b/modules/core/src/main/java/org/apache/ignite/internal/commandline/meta/tasks/MetadataRemoveTask.java index 653f9bd35bd35..6b6a585272d3d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/commandline/meta/tasks/MetadataRemoveTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/commandline/meta/tasks/MetadataRemoveTask.java @@ -155,7 +155,7 @@ private static class DropAllThinSessionsJob implements IgniteRunnable { /** {@inheritDoc} */ @Override public void run() throws IgniteException { - ignite.context().sqlListener().closeAllSessions(); + ignite.context().clientListener().closeAllSessions(); } } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java index 890c1198c38a5..8c1f52a4c15a1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java @@ -457,7 +457,7 @@ private void setField(IgniteEx kernal, String name, Object val) throws NoSuchFie } /** {@inheritDoc} */ - @Override public ClientListenerProcessor sqlListener() { + @Override public ClientListenerProcessor clientListener() { return null; } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerMetrics.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerMetrics.java index d9c614cff5805..0e6f19b534387 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerMetrics.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerMetrics.java @@ -16,12 +16,12 @@ */ package org.apache.ignite.internal.processors.odbc; -import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.processors.metric.MetricRegistry; +import org.apache.ignite.internal.processors.metric.impl.AtomicLongMetric; import org.apache.ignite.internal.processors.metric.impl.IntMetricImpl; +import org.apache.ignite.internal.processors.metric.impl.LongAdderMetric; import org.apache.ignite.internal.processors.metric.impl.MetricUtils; -import static org.apache.ignite.internal.processors.metric.GridMetricManager.CLIENT_CONNECTOR_METRICS; import static org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.CLI_TYPES; import static org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.JDBC_CLIENT; import static org.apache.ignite.internal.processors.odbc.ClientListenerNioListener.ODBC_CLIENT; @@ -43,6 +43,18 @@ public class ClientListenerMetrics { /** Number of successfully established sessions. */ public static final String METRIC_ACEPTED = "AcceptedSessions"; + /** */ + public static final String AFF_KEY_HITS = "AffinityKeyRequestsHits"; + + /** */ + public static final String AFF_KEY_MISSES = "AffinityKeyRequestsMisses"; + + /** */ + public static final String AFF_QRY_HITS = "AffinityQueryRequestsHits"; + + /** */ + public static final String AFF_QRY_MISSES = "AffinityQueryRequestsMisses"; + /** Rejected by timeout. */ private final IntMetricImpl rejectedTimeout; @@ -55,12 +67,22 @@ public class ClientListenerMetrics { /** Connections accepted. */ private final IntMetricImpl[] accepted; + /** */ + private final LongAdderMetric affKeyHits; + + /** */ + private final LongAdderMetric affKeyMisses; + + /** */ + private final AtomicLongMetric affQryHits; + + /** */ + private final AtomicLongMetric affQryMisses; + /** - * @param ctx Kernal context. + * @param mreg Metrics registry. */ - public ClientListenerMetrics(GridKernalContext ctx) { - MetricRegistry mreg = ctx.metric().registry(CLIENT_CONNECTOR_METRICS); - + public ClientListenerMetrics(MetricRegistry mreg) { rejectedTimeout = mreg.intMetric(METRIC_REJECTED_TIMEOUT, "TCP sessions count that were rejected due to handshake timeout."); @@ -71,6 +93,18 @@ public ClientListenerMetrics(GridKernalContext ctx) { accepted = new IntMetricImpl[CLI_TYPES.length]; + affKeyHits = mreg.longAdderMetric(AFF_KEY_HITS, + "The number of affinity-aware cache key requests that were sent to the primary node"); + + affKeyMisses = mreg.longAdderMetric(AFF_KEY_MISSES, + "The number of affinity-aware cache key requests that were sent not to the primary node"); + + affQryHits = mreg.longMetric(AFF_QRY_HITS, + "The number of affinity-aware query requests that were sent to the primary node"); + + affQryMisses = mreg.longMetric(AFF_QRY_MISSES, + "The number of affinity-aware query requests that were sent not to the primary node"); + for (byte clientType : CLI_TYPES) { String clientLabel = clientTypeLabel(clientType); @@ -112,6 +146,26 @@ public void onHandshakeAccept(byte clientType) { accepted[clientType].increment(); } + /** */ + public void onAffinityKeyHit() { + affKeyHits.increment(); + } + + /** */ + public void onAffinityKeyMiss() { + affKeyMisses.increment(); + } + + /** */ + public void onAffinityQryHit() { + affQryHits.increment(); + } + + /** */ + public void onAffinityQryMiss() { + affQryMisses.increment(); + } + /** * Get label for a client. * @param clientType Client type. diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java index 7ed6296dd078d..eaaca39233932 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/odbc/ClientListenerNioListener.java @@ -101,9 +101,14 @@ public class ClientListenerNioListener extends GridNioServerListenerAdapter srv; + /** Metrics. */ + private ClientListenerMetrics metrics; + /** Executor service. */ private ExecutorService execSvc; @@ -151,12 +154,14 @@ public ClientListenerProcessor(GridKernalContext ctx) { MetricRegistry mreg = ctx.metric().registry(CLIENT_CONNECTOR_METRICS); + metrics = new ClientListenerMetrics(mreg); + for (int port = cliConnCfg.getPort(); port <= portTo && port <= 65535; port++) { try { srv = GridNioServer.builder() .address(hostAddr) .port(port) - .listener(new ClientListenerNioListener(ctx, busyLock, cliConnCfg)) + .listener(new ClientListenerNioListener(ctx, busyLock, cliConnCfg, metrics)) .logger(log) .selectorCount(selectorCnt) .igniteInstanceName(ctx.igniteInstanceName()) @@ -466,6 +471,13 @@ public int port() { return srv.port(); } + /** + * @return Client listener metrics. + */ + public ClientListenerMetrics metrics() { + return metrics; + } + /** * Prepare connector configuration. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java index 076bbaab59cd6..ffbe717479586 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/ClientRequestHandler.java @@ -125,7 +125,7 @@ public class ClientRequestHandler implements ClientListenerRequestHandler { msg = sqlState + ": " + msg; } - if (ctx.kernalContext().sqlListener().sendServerExceptionStackTraceToClient()) + if (ctx.kernalContext().clientListener().sendServerExceptionStackTraceToClient()) msg = msg + U.nl() + X.getFullStackTrace(e); return new ClientResponse(req.requestId(), status, msg); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java index 6bcbbe89b2636..d0aa19e778b11 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheClearKeyRequest.java @@ -36,9 +36,9 @@ public ClientCacheClearKeyRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { cache(ctx).clear(key()); - return super.process(ctx); + return new ClientResponse(requestId()); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java index 8470828e424a1..6cea754199e4c 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheContainsKeyRequest.java @@ -37,7 +37,7 @@ public ClientCacheContainsKeyRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { boolean val = cache(ctx).containsKey(key()); return new ClientBooleanResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java index 836021313c5fc..7562808a02d77 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutIfAbsentRequest.java @@ -37,7 +37,7 @@ public ClientCacheGetAndPutIfAbsentRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { Object res = cache(ctx).getAndPutIfAbsent(key(), val()); return new ClientObjectResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java index 7a540e8473ac9..15794c2d389e6 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndPutRequest.java @@ -37,7 +37,7 @@ public ClientCacheGetAndPutRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { Object res = cache(ctx).getAndPut(key(), val()); return new ClientObjectResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java index e4fd735b186ac..ebbe1a854b838 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndRemoveRequest.java @@ -37,7 +37,7 @@ public ClientCacheGetAndRemoveRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { Object val = cache(ctx).getAndRemove(key()); return new ClientObjectResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java index dba8639e4c07a..d3af07d19ad31 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetAndReplaceRequest.java @@ -37,7 +37,7 @@ public ClientCacheGetAndReplaceRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { Object res = cache(ctx).getAndReplace(key(), val()); return new ClientObjectResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java index 41558c2863d03..6fc525613cf75 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheGetRequest.java @@ -37,7 +37,7 @@ public ClientCacheGetRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { Object val = cache(ctx).get(key()); return new ClientObjectResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheIndexQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheIndexQueryRequest.java index 69fc597307400..51d72478fbf7b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheIndexQueryRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheIndexQueryRequest.java @@ -38,7 +38,7 @@ * IndexQuery request. */ @SuppressWarnings({"rawtypes", "unchecked"}) -public class ClientCacheIndexQueryRequest extends ClientCacheRequest { +public class ClientCacheIndexQueryRequest extends ClientCacheQueryRequest { /** IndexQuery. */ private final IndexQuery qry; @@ -144,6 +144,9 @@ private IndexQueryCriterion readInCriterion(BinaryRawReaderEx reader) { @Override public ClientResponse process(ClientConnectionContext ctx) { IgniteCache cache = !isKeepBinary() ? rawCache(ctx) : cache(ctx); + if (qry.getPartition() != null) + updateAffinityMetrics(ctx, qry.getPartition()); + ctx.incrementCursors(); try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheKeyRequest.java index f6517393cbd9b..56013524b5854 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheKeyRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheKeyRequest.java @@ -17,8 +17,15 @@ package org.apache.ignite.internal.processors.platform.client.cache; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.GridKernalContext; import org.apache.ignite.internal.binary.BinaryRawReaderEx; +import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; +import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; +import org.apache.ignite.internal.processors.platform.client.ClientResponse; import org.apache.ignite.internal.processors.platform.client.tx.ClientTxAwareRequest; +import org.apache.ignite.internal.util.typedef.F; /** * Cache request involving key. @@ -38,6 +45,37 @@ public abstract class ClientCacheKeyRequest extends ClientCacheDataRequest imple key = reader.readObjectDetached(); } + /** {@inheritDoc} */ + @Override public final ClientResponse process(ClientConnectionContext ctx) { + if (!isTransactional()) { + // Calculate affinity metrics. + DynamicCacheDescriptor desc = cacheDescriptor(ctx); + CacheConfiguration cfg = desc.cacheConfiguration(); + + if (cfg.getCacheMode() == CacheMode.PARTITIONED && cfg.isStatisticsEnabled()) { + String cacheName = desc.cacheName(); + + try { + GridKernalContext kctx = ctx.kernalContext(); + + if (F.first(kctx.affinity().mapKeyToPrimaryAndBackups(cacheName, key, null)).isLocal()) + kctx.clientListener().metrics().onAffinityKeyHit(); + else + kctx.clientListener().metrics().onAffinityKeyMiss(); + } + catch (Exception ignored) { + // No-op. + } + } + } + + // Process request in overriden method. + return process0(ctx); + } + + /** */ + protected abstract ClientResponse process0(ClientConnectionContext ctx); + /** * Gets the key. * diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheKeyValueRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheKeyValueRequest.java index 03b85d6e87f84..060d3d891f7ea 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheKeyValueRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheKeyValueRequest.java @@ -22,7 +22,7 @@ /** * Cache request involving key and value. */ -public class ClientCacheKeyValueRequest extends ClientCacheKeyRequest { +public abstract class ClientCacheKeyValueRequest extends ClientCacheKeyRequest { /** Value. */ private final Object val; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheLocalPeekRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheLocalPeekRequest.java index 2002664d00714..40e01085710cd 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheLocalPeekRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheLocalPeekRequest.java @@ -39,7 +39,7 @@ public ClientCacheLocalPeekRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { Object val = cache(ctx).localPeek(key(), CachePeekMode.ALL); return new ClientObjectResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java index 4dd2cde58ce06..b0bd0e6317f22 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutIfAbsentRequest.java @@ -37,7 +37,7 @@ public ClientCachePutIfAbsentRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { boolean res = cache(ctx).putIfAbsent(key(), val()); return new ClientBooleanResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java index 2c396b7ede87a..bc249fc07f3d3 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCachePutRequest.java @@ -36,10 +36,10 @@ public ClientCachePutRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { cache(ctx).put(key(), val()); - return super.process(ctx); + return new ClientResponse(requestId()); } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheQueryRequest.java new file mode 100644 index 0000000000000..ff2337cec43e8 --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheQueryRequest.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.platform.client.cache; + +import org.apache.ignite.binary.BinaryRawReader; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.internal.GridKernalContext; +import org.apache.ignite.internal.processors.cache.DynamicCacheDescriptor; +import org.apache.ignite.internal.processors.platform.client.ClientConnectionContext; + +/** + * Abstract query request. + */ +public abstract class ClientCacheQueryRequest extends ClientCacheDataRequest { + /** */ + ClientCacheQueryRequest(BinaryRawReader reader) { + super(reader); + } + + /** */ + protected void updateAffinityMetrics(ClientConnectionContext ctx, int part) { + DynamicCacheDescriptor desc = cacheDescriptor(ctx); + CacheConfiguration cfg = desc.cacheConfiguration(); + + if (cfg.getCacheMode() == CacheMode.PARTITIONED && cfg.isStatisticsEnabled()) { + String cacheName = desc.cacheName(); + + try { + GridKernalContext kctx = ctx.kernalContext(); + + if (kctx.affinity().mapPartitionToNode(cacheName, part, null).isLocal()) + kctx.clientListener().metrics().onAffinityQryHit(); + else + kctx.clientListener().metrics().onAffinityQryMiss(); + } + catch (Exception ignored) { + // No-op. + } + } + } +} diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java index b86f2f8895d64..14948f0e591fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveIfEqualsRequest.java @@ -37,7 +37,7 @@ public ClientCacheRemoveIfEqualsRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { boolean res = cache(ctx).remove(key(), val()); return new ClientBooleanResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java index a68c32730f4fe..f1c50451188fe 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheRemoveKeyRequest.java @@ -37,7 +37,7 @@ public ClientCacheRemoveKeyRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { boolean val = cache(ctx).remove(key()); return new ClientBooleanResponse(requestId(), val); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java index 8645fbb817322..83d40b66e57e2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceIfEqualsRequest.java @@ -42,7 +42,7 @@ public ClientCacheReplaceIfEqualsRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { boolean res = cache(ctx).replace(key(), val(), newVal); return new ClientBooleanResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java index bd7a642bb39e0..cbfa0fcc21942 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheReplaceRequest.java @@ -37,7 +37,7 @@ public ClientCacheReplaceRequest(BinaryRawReaderEx reader) { /** {@inheritDoc} */ @SuppressWarnings("unchecked") - @Override public ClientResponse process(ClientConnectionContext ctx) { + @Override public ClientResponse process0(ClientConnectionContext ctx) { boolean res = cache(ctx).replace(key(), val()); return new ClientBooleanResponse(requestId(), res); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java index 1bdc4f14b49cb..9321db3c3ad68 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheScanQueryRequest.java @@ -36,7 +36,7 @@ * Scan query request. */ @SuppressWarnings({"unchecked", "rawtypes"}) -public class ClientCacheScanQueryRequest extends ClientCacheDataRequest implements ClientTxAwareRequest { +public class ClientCacheScanQueryRequest extends ClientCacheQueryRequest implements ClientTxAwareRequest { /** Local flag. */ private final boolean loc; @@ -82,6 +82,9 @@ public ClientCacheScanQueryRequest(BinaryRawReaderEx reader) { .setPartition(part) .setFilter(createFilter(ctx.kernalContext(), filterObj, filterPlatform)); + if (part != null) + updateAffinityMetrics(ctx, part); + ctx.incrementCursors(); try { diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java index 43a6b4609895b..a1e5376d82e66 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlFieldsQueryRequest.java @@ -41,7 +41,7 @@ * Sql query request. */ @SuppressWarnings("unchecked") -public class ClientCacheSqlFieldsQueryRequest extends ClientCacheDataRequest implements ClientTxAwareRequest { +public class ClientCacheSqlFieldsQueryRequest extends ClientCacheQueryRequest implements ClientTxAwareRequest { /** Query. */ private final SqlFieldsQuery qry; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java index 29b0de8d1e1ec..00c75c217d86a 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/cache/ClientCacheSqlQueryRequest.java @@ -31,7 +31,7 @@ * Sql query request. */ @SuppressWarnings("unchecked") -public class ClientCacheSqlQueryRequest extends ClientCacheDataRequest implements ClientTxAwareRequest { +public class ClientCacheSqlQueryRequest extends ClientCacheQueryRequest implements ClientTxAwareRequest { /** Query. */ private final SqlQuery qry; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/compute/ClientComputeTask.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/compute/ClientComputeTask.java index ed84382ec7cf0..f1525fa3f4a2d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/compute/ClientComputeTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/platform/client/compute/ClientComputeTask.java @@ -128,7 +128,7 @@ void onResponseSent() { ClientNotification notification; if (f.error() != null) { - String msg = ctx.kernalContext().sqlListener().sendServerExceptionStackTraceToClient() + String msg = ctx.kernalContext().clientListener().sendServerExceptionStackTraceToClient() ? f.error().getMessage() + U.nl() + X.getFullStackTrace(f.error()) : f.error().getMessage(); diff --git a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlCommandProcessor.java b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlCommandProcessor.java index 3a1ff15b40af6..698d2a6261329 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlCommandProcessor.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/sql/SqlCommandProcessor.java @@ -189,9 +189,9 @@ private void processKillScanQueryCommand(SqlKillScanQueryCommand cmd) { */ private void processKillClientCommand(SqlKillClientCommand cmd) { if (cmd.connectionId() == null) - ctx.sqlListener().mxBean().dropAllConnections(); + ctx.clientListener().mxBean().dropAllConnections(); else - ctx.sqlListener().mxBean().dropConnection(cmd.connectionId()); + ctx.clientListener().mxBean().dropConnection(cmd.connectionId()); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/visor/client/VisorClientConnectionDropTask.java b/modules/core/src/main/java/org/apache/ignite/internal/visor/client/VisorClientConnectionDropTask.java index b5a0d949e4ef4..123325d76374d 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/visor/client/VisorClientConnectionDropTask.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/visor/client/VisorClientConnectionDropTask.java @@ -77,7 +77,7 @@ protected VisorClientConnectionDropJob(@Nullable Long arg, boolean debug) { /** {@inheritDoc} */ @Override protected Boolean run(@Nullable Long arg) throws IgniteException { - ClientProcessorMXBean bean = ignite.context().sqlListener().mxBean(); + ClientProcessorMXBean bean = ignite.context().clientListener().mxBean(); if (arg != null) return bean.dropConnection(arg); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/client/thin/AffinityMetricsTest.java b/modules/core/src/test/java/org/apache/ignite/internal/client/thin/AffinityMetricsTest.java new file mode 100644 index 0000000000000..5877a211075be --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/client/thin/AffinityMetricsTest.java @@ -0,0 +1,213 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.client.thin; + +import java.util.HashMap; +import java.util.Map; +import org.apache.ignite.Ignite; +import org.apache.ignite.cache.affinity.Affinity; +import org.apache.ignite.cache.query.ScanQuery; +import org.apache.ignite.client.ClientCache; +import org.apache.ignite.client.ClientTransaction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.metric.GridMetricManager; +import org.apache.ignite.internal.processors.metric.MetricRegistry; +import org.apache.ignite.internal.util.typedef.G; +import org.apache.ignite.spi.metric.LongMetric; +import org.junit.Test; + +import static org.apache.ignite.cache.CacheAtomicityMode.TRANSACTIONAL; +import static org.apache.ignite.cache.CacheMode.PARTITIONED; +import static org.apache.ignite.cache.CacheMode.REPLICATED; +import static org.apache.ignite.internal.processors.odbc.ClientListenerMetrics.AFF_KEY_HITS; +import static org.apache.ignite.internal.processors.odbc.ClientListenerMetrics.AFF_KEY_MISSES; +import static org.apache.ignite.internal.processors.odbc.ClientListenerMetrics.AFF_QRY_HITS; +import static org.apache.ignite.internal.processors.odbc.ClientListenerMetrics.AFF_QRY_MISSES; + +/** + * Test thin client affinity hits/misses metrics. + */ +public class AffinityMetricsTest extends ThinClientAbstractPartitionAwarenessTest { + /** Grids count. */ + private static final int GRIDS_CNT = 3; + + /** */ + private static final String PART_CACHE = "partCache"; + + /** */ + private static final String REPL_CACHE = "replCache"; + + /** */ + private static final String[] ALL_AFF_METRICS = + new String[] {AFF_KEY_HITS, AFF_KEY_MISSES, AFF_QRY_HITS, AFF_QRY_MISSES}; + + /** */ + private final Map lastMetricValues = new HashMap<>(); + + /** {@inheritDoc} */ + @Override protected void beforeTestsStarted() throws Exception { + super.beforeTestsStarted(); + + startGrids(GRIDS_CNT); + + awaitPartitionMapExchange(); + } + + /** {@inheritDoc} */ + @Override protected void beforeTest() throws Exception { + initClient(getClientConfiguration(0, 1).setClusterDiscoveryEnabled(false), 0, 1); + + super.beforeTest(); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + return super.getConfiguration(igniteInstanceName).setCacheConfiguration( + new CacheConfiguration<>(PART_CACHE).setCacheMode(PARTITIONED).setAtomicityMode(TRANSACTIONAL) + .setStatisticsEnabled(true), + new CacheConfiguration<>(REPL_CACHE).setCacheMode(REPLICATED).setAtomicityMode(TRANSACTIONAL) + .setStatisticsEnabled(true) + ); + } + + /** */ + @Test + public void testCacheKeyAffinityMetricsPartitioned() { + resetMetricValues(); + + Integer affKey0 = primaryKey(ignite(0).cache(PART_CACHE)); + Integer affKey1 = primaryKey(ignite(1).cache(PART_CACHE)); + Integer affKey2 = primaryKey(ignite(2).cache(PART_CACHE)); + ClientCache cache = client.cache(PART_CACHE); + + cache.put(affKey0, affKey0); + + assertEquals(1, calcMetricIncrement(ignite(0), AFF_KEY_HITS)); + + cache.put(affKey1, affKey1); + + assertEquals(1, calcMetricIncrement(ignite(1), AFF_KEY_HITS)); + + cache.put(affKey2, affKey2); + + assertEquals(1, calcMetricIncrement(ignite(0), AFF_KEY_MISSES) + + calcMetricIncrement(ignite(1), AFF_KEY_MISSES)); + } + + /** */ + @Test + public void testCacheKeyAffinityMetricsReplicated() { + resetMetricValues(); + + ClientCache cache = client.cache(REPL_CACHE); + + for (int i = 0; i < 100; i++) + cache.put(i, i); + + for (Ignite ignite : G.allGrids()) { + for (String metricName : ALL_AFF_METRICS) + assertEquals(0, calcMetricIncrement(ignite, metricName)); + } + } + + /** */ + @Test + public void testCacheKeyAffinityMetricsTx() { + resetMetricValues(); + + for (int i = 0; i < 100; i++) { + try (ClientTransaction tx = client.transactions().txStart()) { + client.cache(PART_CACHE).put(i, i); + tx.commit(); + } + } + + for (Ignite ignite : G.allGrids()) { + for (String metricName : ALL_AFF_METRICS) + assertEquals(0, calcMetricIncrement(ignite, metricName)); + } + } + + /** */ + @Test + public void testQueryAffinityMetricsPartitioned() { + resetMetricValues(); + + Integer affKey0 = primaryKey(ignite(0).cache(PART_CACHE)); + Integer affKey1 = primaryKey(ignite(1).cache(PART_CACHE)); + Integer affKey2 = primaryKey(ignite(2).cache(PART_CACHE)); + + Affinity aff = affinity(ignite(0).cache(PART_CACHE)); + int part0 = aff.partition(affKey0); + int part1 = aff.partition(affKey1); + int part2 = aff.partition(affKey2); + + ClientCache cache = client.cache(PART_CACHE); + + cache.query(new ScanQuery<>().setPartition(part0)).getAll(); + + assertEquals(1, calcMetricIncrement(ignite(0), AFF_QRY_HITS)); + + cache.query(new ScanQuery<>().setPartition(part1)).getAll(); + + assertEquals(1, calcMetricIncrement(ignite(1), AFF_QRY_HITS)); + + cache.query(new ScanQuery<>().setPartition(part2)).getAll(); + + assertEquals(1, calcMetricIncrement(ignite(0), AFF_QRY_MISSES) + + calcMetricIncrement(ignite(1), AFF_QRY_MISSES)); + } + + /** */ + @Test + public void testQueryAffinityMetricsReplicated() { + resetMetricValues(); + + ClientCache cache = client.cache(REPL_CACHE); + + for (int i = 0; i < 10; i++) + cache.query(new ScanQuery<>().setPartition(i)).getAll(); + + for (Ignite ignite : G.allGrids()) { + for (String metricName : ALL_AFF_METRICS) + assertEquals(0, calcMetricIncrement(ignite, metricName)); + } + } + + /** */ + private void resetMetricValues() { + lastMetricValues.clear(); + + for (Ignite ignite : G.allGrids()) { + for (String metricName : ALL_AFF_METRICS) + calcMetricIncrement(ignite, metricName); + } + } + + /** */ + private long calcMetricIncrement(Ignite ignite, String metricName) { + MetricRegistry mreg = ((IgniteEx)ignite).context().metric().registry(GridMetricManager.CLIENT_CONNECTOR_METRICS); + LongMetric metric = mreg.findMetric(metricName); + long newVal = metric.value(); + Long oldVal = lastMetricValues.put(ignite.name() + '.' + metricName, newVal); + + return newVal - (oldVal == null ? 0 : oldVal); + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java index cbb128e44a764..04d9e2b76d16e 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/client/ClientTestSuite.java @@ -17,6 +17,7 @@ package org.apache.ignite.client; +import org.apache.ignite.internal.client.thin.AffinityMetricsTest; import org.apache.ignite.internal.client.thin.AtomicLongTest; import org.apache.ignite.internal.client.thin.CacheAsyncTest; import org.apache.ignite.internal.client.thin.CacheEntryListenersTest; @@ -86,6 +87,7 @@ IgniteClientRequestEventListenerTest.class, ThinClientEnpointsDiscoveryTest.class, InactiveClusterCacheRequestTest.class, + AffinityMetricsTest.class, }) public class ClientTestSuite { // No-op. From 6ee15a9911727f08807764e484f2c614538a049b Mon Sep 17 00:00:00 2001 From: Mikhail Petrov <32207922+petrov-mg@users.noreply.github.com> Date: Tue, 18 Apr 2023 18:27:04 +0300 Subject: [PATCH 10/22] IGNITE-19277 Added authorization of Ignite Cluster Node start/stop operations. (#10644) (cherry picked from commit f5e69ba922c80ce059658db7483fb3dee584e978) --- .../internal/cluster/IgniteClusterImpl.java | 14 + .../plugin/security/SecurityPermission.java | 6 + .../ClusterNodeOperationPermissionTest.java | 301 ++++++++++++++++++ .../security/impl/TestSecurityContext.java | 2 + .../ignite/testsuites/SecurityTestSuite.java | 4 +- 5 files changed, 326 insertions(+), 1 deletion(-) create mode 100644 modules/core/src/test/java/org/apache/ignite/internal/processors/security/cluster/ClusterNodeOperationPermissionTest.java diff --git a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java index e02317249409b..b094d8df42bc1 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/cluster/IgniteClusterImpl.java @@ -84,6 +84,8 @@ import static org.apache.ignite.internal.processors.task.TaskExecutionOptions.options; import static org.apache.ignite.internal.util.nodestart.IgniteNodeStartUtils.parseFile; import static org.apache.ignite.internal.util.nodestart.IgniteNodeStartUtils.specifications; +import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_CLUSTER_NODE_START; +import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_CLUSTER_NODE_STOP; /** * @@ -276,6 +278,8 @@ public void start() { guard(); try { + ctx.security().authorize(ADMIN_CLUSTER_NODE_STOP); + ctx.task().execute(IgniteKillTask.class, false, options().withProjection(nodes())).get(); } catch (IgniteCheckedException e) { @@ -291,6 +295,8 @@ public void start() { guard(); try { + ctx.security().authorize(ADMIN_CLUSTER_NODE_STOP); + ctx.task().execute(IgniteKillTask.class, false, options().withProjection(forNodeIds(ids).nodes())).get(); } catch (IgniteCheckedException e) { @@ -306,6 +312,8 @@ public void start() { guard(); try { + ctx.security().authorize(ADMIN_CLUSTER_NODE_STOP); + ctx.task().execute(IgniteKillTask.class, true, options().withProjection(nodes())).get(); } catch (IgniteCheckedException e) { @@ -321,6 +329,8 @@ public void start() { guard(); try { + ctx.security().authorize(ADMIN_CLUSTER_NODE_STOP); + ctx.task().execute(IgniteKillTask.class, true, options().withProjection(forNodeIds(ids).nodes())).get(); } catch (IgniteCheckedException e) { @@ -882,6 +892,8 @@ IgniteInternalFuture> startNodesAsync0( guard(); try { + ctx.security().authorize(ADMIN_CLUSTER_NODE_START); + IgniteSshHelper sshHelper = IgniteComponentType.SSH.create(false); Map> specsMap = specifications(hosts, dflts); @@ -920,6 +932,8 @@ IgniteInternalFuture> startNodesAsync0( if (neighbors != null) { if (restart && !neighbors.isEmpty()) { + ctx.security().authorize(ADMIN_CLUSTER_NODE_STOP); + try { ctx.task().execute(IgniteKillTask.class, false, options(forNodes(neighbors).nodes())).get(); } diff --git a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java index ccd07a87dd176..5deb365d0bbca 100644 --- a/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java +++ b/modules/core/src/main/java/org/apache/ignite/plugin/security/SecurityPermission.java @@ -101,6 +101,12 @@ public enum SecurityPermission { */ ADMIN_CLUSTER_STATE, + /** Start new cluster node permission. */ + ADMIN_CLUSTER_NODE_START, + + /** Stop/restart cluster node permission. */ + ADMIN_CLUSTER_NODE_STOP, + /** Permission to execute REFRESH STATISTICS command. */ REFRESH_STATISTICS, diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/cluster/ClusterNodeOperationPermissionTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/cluster/ClusterNodeOperationPermissionTest.java new file mode 100644 index 0000000000000..6b4111bcb15a6 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/cluster/ClusterNodeOperationPermissionTest.java @@ -0,0 +1,301 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.processors.security.cluster; + +import java.io.File; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteException; +import org.apache.ignite.Ignition; +import org.apache.ignite.client.ClientConnectionException; +import org.apache.ignite.client.ClientException; +import org.apache.ignite.client.IgniteClient; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.compute.ComputeJob; +import org.apache.ignite.compute.ComputeJobAdapter; +import org.apache.ignite.compute.ComputeJobResult; +import org.apache.ignite.compute.ComputeTaskAdapter; +import org.apache.ignite.configuration.ClientConfiguration; +import org.apache.ignite.configuration.ClientConnectorConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.ThinClientConfiguration; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.processors.security.AbstractSecurityTest; +import org.apache.ignite.internal.processors.security.impl.TestSecurityData; +import org.apache.ignite.internal.processors.security.impl.TestSecurityPluginProvider; +import org.apache.ignite.internal.util.lang.RunnableX; +import org.apache.ignite.plugin.security.SecurityException; +import org.apache.ignite.plugin.security.SecurityPermissionSetBuilder; +import org.apache.ignite.resources.IgniteInstanceResource; +import org.apache.ignite.testframework.junits.multijvm.IgniteProcessProxy; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; +import org.junit.Test; + +import static java.util.Collections.singleton; +import static java.util.Collections.singletonList; +import static org.apache.ignite.IgniteSystemProperties.IGNITE_SUCCESS_FILE; +import static org.apache.ignite.internal.processors.security.cluster.ClusterNodeOperationPermissionTest.Operation.RESTART; +import static org.apache.ignite.internal.processors.security.cluster.ClusterNodeOperationPermissionTest.Operation.RESTART_ALL; +import static org.apache.ignite.plugin.security.SecurityPermission.ADMIN_CLUSTER_NODE_STOP; +import static org.apache.ignite.plugin.security.SecurityPermission.JOIN_AS_SERVER; +import static org.apache.ignite.plugin.security.SecurityPermission.TASK_EXECUTE; +import static org.apache.ignite.plugin.security.SecurityPermissionSetBuilder.systemPermissions; +import static org.apache.ignite.testframework.GridTestUtils.assertThrowsAnyCause; +import static org.apache.ignite.testframework.GridTestUtils.waitForCondition; + +/** */ +public class ClusterNodeOperationPermissionTest extends AbstractSecurityTest { + /** */ + private IgniteClient cli; + + /** */ + private IgniteProcessProxy ignite; + + /** */ + public static final String RESTART_FILE_PATH = "src/test/resources/ignite-restart.tmp"; + + /** */ + private IgniteConfiguration configuration(int idx) throws Exception { + String login = getTestIgniteInstanceName(idx); + + return getConfiguration( + login, + new TestSecurityPluginProvider( + login, + "", + systemPermissions(JOIN_AS_SERVER), + null, + false, + new TestSecurityData("stop-allowed", clientPermissionsBuilder() + .appendSystemPermissions(ADMIN_CLUSTER_NODE_STOP) + .build()), + new TestSecurityData("stop-forbidden", clientPermissionsBuilder().build()) + )) + .setClientConnectorConfiguration(new ClientConnectorConfiguration() + .setThinClientConfiguration(new ThinClientConfiguration() + .setMaxActiveComputeTasksPerConnection(1))); + } + + /** {@inheritDoc} */ + @Override protected void afterTest() throws Exception { + super.afterTest(); + + stopAllGrids(); + } + + /** */ + @Test + public void testStartNodeAuthorization() throws Exception { + IgniteEx ignite = startGrid(configuration(0)); + + assertAuthorizationFailed( + () -> ignite.cluster().startNodes(new File("src/test/config/start-nodes.ini"), true, 1, 1), + SecurityException.class + ); + + assertAuthorizationFailed(() -> + ignite.cluster().startNodesAsync(new File("src/test/config/start-nodes.ini"), true, 1, 1).get(), + SecurityException.class + ); + + assertAuthorizationFailed(() -> + ignite.cluster().startNodes(Collections.emptyList(), Collections.emptyMap(), true, 1, 1), + SecurityException.class + ); + + assertAuthorizationFailed(() -> + ignite.cluster().startNodes(Collections.emptyList(), Collections.emptyMap(), true, 1, 1), + SecurityException.class + ); + } + + /** */ + @Test + public void testStopNodeAuthorization() throws Exception { + for (Operation op : Operation.values()) { + doClusterNodeTest("stop-allowed", op, true); + doClusterNodeTest("stop-forbidden", op, false); + } + } + + /** + * We are forced to initiate the test operation through the thin client, because otherwise the node stop/restart + * operations will halt the local test JVM. + */ + private void doClusterNodeTest(String login, Operation op, boolean isSuccessExpected) throws Exception { + prepareCluster(login); + + try { + if (isSuccessExpected) { + try { + cli.compute().execute(ClusterNodeOperationExecutor.class.getName(), op); + } + catch (ClientException e) { + // It is possible for the Ignite node to shut down before the task execution response is sent. + assertTrue(e.getMessage().contains("Task cancelled due to stopping of the grid")); + } + + assertTrue(waitForCondition(() -> !ignite.getProcess().getProcess().isAlive(), getTestTimeout())); + + assertTrue((op != RESTART && op != RESTART_ALL) || new File(RESTART_FILE_PATH).exists()); + } + else { + assertAuthorizationFailed( + () -> cli.compute().execute(ClusterNodeOperationExecutor.class.getName(), op), + ClientException.class + ); + + assertTrue(ignite.getProcess().getProcess().isAlive()); + } + } + finally { + stopCluster(); + } + } + + /** */ + private void prepareCluster(String opExecutorLogin) throws Exception { + ignite = new IgniteProcessProxy( + configuration(0), + log, + null, + true, + singletonList("-D" + IGNITE_SUCCESS_FILE + "=" + RESTART_FILE_PATH) + ); + + AtomicReference cli = new AtomicReference<>(); + + assertTrue(waitForCondition(() -> { + try { + cli.set(startClient(opExecutorLogin)); + + return true; + } + catch (ClientConnectionException e) { + return false; + } + }, getTestTimeout())); + + this.cli = cli.get(); + } + + /** */ + private void stopCluster() throws Exception { + cli.close(); + + ignite.kill(); + + waitForCondition(() -> !ignite.getProcess().getProcess().isAlive(), getTestTimeout()); + + File restartFile = new File(RESTART_FILE_PATH); + + if (restartFile.exists()) + restartFile.delete(); + } + + /** */ + private void assertAuthorizationFailed(RunnableX r, Class exCls) { + assertThrowsAnyCause( + log, + () -> { + r.run(); + + return null; + }, + exCls, + "Authorization failed" + ); + } + + /** */ + private IgniteClient startClient(String login) { + return Ignition.startClient(new ClientConfiguration() + .setAddresses("127.0.0.1:10800") + .setUserName(login) + .setUserPassword("")); + } + + /** */ + private SecurityPermissionSetBuilder clientPermissionsBuilder() { + return new SecurityPermissionSetBuilder() + .defaultAllowAll(false) + .appendTaskPermissions(ClusterNodeOperationExecutor.class.getName(), TASK_EXECUTE); + } + + /** */ + public static final class ClusterNodeOperationExecutor extends ComputeTaskAdapter { + /** {@inheritDoc} */ + @Override public @NotNull Map map( + List subgrid, + @Nullable Operation arg + ) throws IgniteException { + return Collections.singletonMap(new Job(arg), subgrid.get(0)); + } + + /** {@inheritDoc} */ + @Override public @Nullable Void reduce(List results) throws IgniteException { + return null; + } + + /** */ + private static final class Job extends ComputeJobAdapter { + /** */ + private final Operation op; + + /** */ + @IgniteInstanceResource + Ignite ignite; + + /** */ + public Job(Operation op) { + this.op = op; + } + + /** {@inheritDoc} */ + @Override public Object execute() throws IgniteException { + switch (op) { + case STOP: ignite.cluster().stopNodes(singleton(ignite.cluster().localNode().id())); break; + case STOP_ALL: ignite.cluster().stopNodes(); break; + case RESTART: ignite.cluster().restartNodes(singleton(ignite.cluster().localNode().id())); break; + case RESTART_ALL: ignite.cluster().restartNodes(); break; + } + + return null; + } + } + } + + /** */ + public enum Operation { + /** */ + STOP, + + /** */ + STOP_ALL, + + /** */ + RESTART, + + /** */ + RESTART_ALL + } +} diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/impl/TestSecurityContext.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/impl/TestSecurityContext.java index 4abfd5d0f693c..c02d4b557fbeb 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/security/impl/TestSecurityContext.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/security/impl/TestSecurityContext.java @@ -73,6 +73,8 @@ public boolean operationAllowed(String opName, SecurityPermission perm) { case JOIN_AS_SERVER: case ADMIN_KILL: case ADMIN_USER_ACCESS: + case ADMIN_CLUSTER_NODE_STOP: + case ADMIN_CLUSTER_NODE_START: return systemOperationAllowed(perm); default: diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java index 32e62d67ce9e8..9bd400f66d6ad 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/SecurityTestSuite.java @@ -38,6 +38,7 @@ import org.apache.ignite.internal.processors.security.client.ThinClientPermissionCheckTest; import org.apache.ignite.internal.processors.security.client.ThinClientSecurityContextOnRemoteNodeTest; import org.apache.ignite.internal.processors.security.client.ThinClientSslPermissionCheckTest; +import org.apache.ignite.internal.processors.security.cluster.ClusterNodeOperationPermissionTest; import org.apache.ignite.internal.processors.security.cluster.ClusterStatePermissionTest; import org.apache.ignite.internal.processors.security.compute.ComputePermissionCheckTest; import org.apache.ignite.internal.processors.security.compute.closure.ComputeTaskCancelRemoteSecurityContextCheckTest; @@ -132,7 +133,8 @@ MultipleSSLContextsTest.class, MaintenanceModeNodeSecurityTest.class, ServiceAuthorizationTest.class, - ServiceStaticConfigTest.class + ServiceStaticConfigTest.class, + ClusterNodeOperationPermissionTest.class }) public class SecurityTestSuite { /** */ From c58578390c9146d5e1651592a94310523f5fef4f Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Tue, 18 Apr 2023 20:20:36 +0300 Subject: [PATCH 11/22] IGNITE-19297 Cleanup assembly files - Fixes #10652. Signed-off-by: Aleksey Plekhanov (cherry picked from commit 28930c2beb5c57b29626116efe395035a65478a1) --- LICENSE | 9 +- assembly/LICENSE_HADOOP | 257 ------------------ assembly/LICENSE_IGNITE | 6 - assembly/NOTICE_HADOOP | 12 - assembly/dependencies-apache-ignite-slim.xml | 1 - assembly/release-apache-ignite-hadoop.xml | 86 ------ bin/include/service.sh | 4 +- .../core/licenses/snaptree-bsd-license.txt | 35 --- modules/core/src/main/java/META-INF/LICENSE | 9 +- modules/hadoop/licenses/apache-2.0.txt | 202 -------------- parent/pom.xml | 2 - 11 files changed, 4 insertions(+), 619 deletions(-) delete mode 100644 assembly/LICENSE_HADOOP delete mode 100644 assembly/NOTICE_HADOOP delete mode 100644 assembly/release-apache-ignite-hadoop.xml delete mode 100644 modules/core/licenses/snaptree-bsd-license.txt delete mode 100644 modules/hadoop/licenses/apache-2.0.txt diff --git a/LICENSE b/LICENSE index 8ab6fe621fcb2..fc17bd92852a5 100644 --- a/LICENSE +++ b/LICENSE @@ -212,13 +212,6 @@ subcomponents is subject to the terms and conditions of the following licenses. -============================================================================== -For SnapTree: -============================================================================== -This product bundles SnapTree, which is available under a -"3-clause BSD" license. For details, see -https://github.com/nbronson/snaptree/blob/master/LICENSE. - ============================================================================== For netlib-java: ============================================================================== @@ -234,7 +227,7 @@ For details, see CC0 1.0 Universal (1.0), Public Domain Dedication, http://creativecommons.org/publicdomain/zero/1.0/ ============================================================================== -For books used for tests in "org.apache.ignite.internal.processors.hadoop.books" +For books used in "examples" module ============================================================================== This code bundles book text files used for testing purposes which contain the following header: diff --git a/assembly/LICENSE_HADOOP b/assembly/LICENSE_HADOOP deleted file mode 100644 index 3db236094f828..0000000000000 --- a/assembly/LICENSE_HADOOP +++ /dev/null @@ -1,257 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - - - -============================================================================== -Apache Ignite Subcomponents: - -The Apache Ignite project contains subcomponents with separate copyright -notices and license terms. Your use of the source code for the these -subcomponents is subject to the terms and conditions of the following -licenses. - - -============================================================================== -For SnapTree: -============================================================================== -This product bundles SnapTree, which is available under the following: -"3-clause BSD" license. For details, see https://github.com/nbronson/snaptree/blob/master/LICENSE. - -============================================================================== -For JSR 166 classes in "org.jsr166" package -============================================================================== -This product bundles JSR-166 classes which are donated to public domain. -For details, see CC0 1.0 Universal (1.0), Public Domain Dedication, -http://creativecommons.org/publicdomain/zero/1.0/ - -============================================================================== -For JSR107 API and SPI (https://github.com/jsr107/jsr107spec) javax.cache:cache-api:jar:1.0.0 -============================================================================== -This product bundles JSR107 API and SPI which is available under the following: -JSR-000107 JCACHE 2.9 Public Review - Updated Specification License. https://raw.github.com/jsr107/jsr107spec/master/LICENSE.txt - -============================================================================== -For JSch (http://www.jcraft.com/jsch/) com.jcraft:jsch:jar:0.1.50 -============================================================================== -This product bundles JSch which is available under the following: -Revised BSD. For details, see http://www.jcraft.com/jsch/LICENSE.txt. - -============================================================================== -For JLine (http://nexus.sonatype.org/oss-repository-hosting.html/jline) jline:jline:jar:2.12.1 -============================================================================== -This product bundles JLine which is available under the following: -The BSD License. For details, see http://www.opensource.org/licenses/bsd-license.php. - -============================================================================== -For Scala Library (http://www.scala-lang.org/) org.scala-lang:scala-library:jar:2.11.2 -============================================================================== -This product bundles Scala Library which is available under the following: -BSD 3-Clause. For details, see http://www.scala-lang.org/license.html. - -============================================================================== -For ASM All (http://asm.objectweb.org/asm-all/) org.ow2.asm:asm-all:jar:5.0.3 -============================================================================== -This product bundles ASM All which is available under the following: -BSD. For details, see http://asm.objectweb.org/license.html. - - diff --git a/assembly/LICENSE_IGNITE b/assembly/LICENSE_IGNITE index ce44bea0ff729..c8780890818d8 100644 --- a/assembly/LICENSE_IGNITE +++ b/assembly/LICENSE_IGNITE @@ -211,12 +211,6 @@ subcomponents is subject to the terms and conditions of the following licenses. -============================================================================== -For SnapTree: -============================================================================== -This product bundles SnapTree, which is available under the following: -"3-clause BSD" license. For details, see https://github.com/nbronson/snaptree/blob/master/LICENSE. - ============================================================================== For JSR 166 classes in "org.jsr166" package ============================================================================== diff --git a/assembly/NOTICE_HADOOP b/assembly/NOTICE_HADOOP deleted file mode 100644 index d232b97e36967..0000000000000 --- a/assembly/NOTICE_HADOOP +++ /dev/null @@ -1,12 +0,0 @@ -Apache Ignite -Copyright 2022 The Apache Software Foundation - -This product includes software developed at -The Apache Software Foundation (http://www.apache.org/). - - -This software includes code from IntelliJ IDEA Community Edition -Copyright (C) JetBrains s.r.o. -https://www.jetbrains.com/idea/ -Licensed under Apache License, Version 2.0. -http://search.maven.org/#artifactdetails%7Corg.jetbrains%7Cannotations%7C13.0%7Cjar diff --git a/assembly/dependencies-apache-ignite-slim.xml b/assembly/dependencies-apache-ignite-slim.xml index 872f205529728..21eb8ac2a194f 100644 --- a/assembly/dependencies-apache-ignite-slim.xml +++ b/assembly/dependencies-apache-ignite-slim.xml @@ -123,7 +123,6 @@ ${project.groupId}:ignite-extdata-pluggable ${project.groupId}:ignite-examples ${project.groupId}:ignite-indexing - ${project.groupId}:ignite-hadoop ${project.groupId}:ignite-codegen ${project.groupId}:ignite-schedule ${project.groupId}:ignite-appserver-test diff --git a/assembly/release-apache-ignite-hadoop.xml b/assembly/release-apache-ignite-hadoop.xml deleted file mode 100644 index 940babbdc0ea8..0000000000000 --- a/assembly/release-apache-ignite-hadoop.xml +++ /dev/null @@ -1,86 +0,0 @@ - - - - - - hadoop - - false - - - dir - - - - release-base.xml - - - - - assembly/LICENSE_HADOOP - LICENSE - / - - - - assembly/NOTICE_HADOOP - NOTICE - / - - - - modules/hadoop/config/core-site.ignite.xml - /config/hadoop - - - modules/hadoop/config/mapred-site.ignite.xml - /config/hadoop - - - modules/hadoop/config/hive-site.ignite.xml - /config/hadoop - - - - modules/hadoop/docs/HADOOP_README.txt - / - - - - - - libs - /libs - - optional/** - - - - - bin - /bin - 0755 - - **/service.sh - - - - diff --git a/bin/include/service.sh b/bin/include/service.sh index 73beb86f3bbf1..7a6217b7183f6 100755 --- a/bin/include/service.sh +++ b/bin/include/service.sh @@ -17,9 +17,9 @@ # #################################################################### -# Ignite Hadoop service start/stop script. +# Ignite service start/stop script. # Supposed to be called from unix `init.d` script. Environment must -# be set via the call of /etc/default/{hadoop,ignite-hadoop} +# be set via the call of /etc/default/{ignite} #################################################################### # Stop script on error. diff --git a/modules/core/licenses/snaptree-bsd-license.txt b/modules/core/licenses/snaptree-bsd-license.txt deleted file mode 100644 index a4bd0d98fb498..0000000000000 --- a/modules/core/licenses/snaptree-bsd-license.txt +++ /dev/null @@ -1,35 +0,0 @@ -This product sources bundles SnapTree classes, which is available under a SNAPTREE LICENSE: - -Copyright (c) 2009 Stanford University, unless otherwise specified. -All rights reserved. - -This software was developed by the Pervasive Parallelism Laboratory of -Stanford University, California, USA. - -Permission to use, copy, modify, and distribute this software in source -or binary form for any purpose with or without fee is hereby granted, -provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of Stanford University nor the names of its - contributors may be used to endorse or promote products derived - from this software without specific prior written permission. - - -THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE -ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY -OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF -SUCH DAMAGE. diff --git a/modules/core/src/main/java/META-INF/LICENSE b/modules/core/src/main/java/META-INF/LICENSE index c971434936359..a92a40d07e4f5 100644 --- a/modules/core/src/main/java/META-INF/LICENSE +++ b/modules/core/src/main/java/META-INF/LICENSE @@ -212,13 +212,6 @@ subcomponents is subject to the terms and conditions of the following licenses. -============================================================================== -For SnapTree: -============================================================================== -This product bundles SnapTree, which is available under a -"3-clause BSD" license. For details, see -https://github.com/nbronson/snaptree/blob/master/LICENSE. - ============================================================================== For JSR 166 classes in "org.jsr166" package ============================================================================== @@ -227,7 +220,7 @@ For details, see CC0 1.0 Universal (1.0), Public Domain Dedication, http://creativecommons.org/publicdomain/zero/1.0/ ============================================================================== -For books used for tests in "org.apache.ignite.internal.processors.hadoop.books" +For books used in "examples" module ============================================================================== This code bundles book text files used for testing purposes which contain the following header: diff --git a/modules/hadoop/licenses/apache-2.0.txt b/modules/hadoop/licenses/apache-2.0.txt deleted file mode 100644 index d645695673349..0000000000000 --- a/modules/hadoop/licenses/apache-2.0.txt +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright [yyyy] [name of copyright owner] - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/parent/pom.xml b/parent/pom.xml index 82b5a0ba4c808..644fc48037064 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -780,9 +780,7 @@ **/NOTICE* **/LICENSE* **/MIGRATION_GUIDE* - src/main/java/org/apache/ignite/internal/util/offheap/unsafe/GridOffHeapSnapTreeMap.java src/main/java/org/apache/ignite/internal/util/nio/SelectedSelectionKeySet.java - src/main/java/org/apache/ignite/internal/util/snaptree/*.java src/main/java/org/apache/ignite/internal/processors/query/calcite/sql/generated/*.java src/main/java/org/jsr166/*.java src/main/java/org/mindrot/*.java From 6ac7e124764a19b6e31e540cb59bd2b8b30b2f37 Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Wed, 19 Apr 2023 18:14:58 +0300 Subject: [PATCH 12/22] IGNITE-19308 Change copyright to 2023 - Fixes #10653. Signed-off-by: Aleksey Plekhanov (cherry picked from commit b5e73970febcce45a9dfbcc6e1bed0cf04313177) --- NOTICE | 2 +- assembly/NOTICE_IGNITE | 2 +- modules/core/src/main/java/META-INF/NOTICE | 2 +- modules/platforms/dotnet/SharedAssemblyInfo.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/NOTICE b/NOTICE index d232b97e36967..2589f6776e94b 100644 --- a/NOTICE +++ b/NOTICE @@ -1,5 +1,5 @@ Apache Ignite -Copyright 2022 The Apache Software Foundation +Copyright 2023 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/assembly/NOTICE_IGNITE b/assembly/NOTICE_IGNITE index 8aa271493491c..b257ca8095da0 100644 --- a/assembly/NOTICE_IGNITE +++ b/assembly/NOTICE_IGNITE @@ -1,5 +1,5 @@ Apache Ignite -Copyright 2022 The Apache Software Foundation +Copyright 2023 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/modules/core/src/main/java/META-INF/NOTICE b/modules/core/src/main/java/META-INF/NOTICE index d232b97e36967..2589f6776e94b 100644 --- a/modules/core/src/main/java/META-INF/NOTICE +++ b/modules/core/src/main/java/META-INF/NOTICE @@ -1,5 +1,5 @@ Apache Ignite -Copyright 2022 The Apache Software Foundation +Copyright 2023 The Apache Software Foundation This product includes software developed at The Apache Software Foundation (http://www.apache.org/). diff --git a/modules/platforms/dotnet/SharedAssemblyInfo.cs b/modules/platforms/dotnet/SharedAssemblyInfo.cs index 9780574487f03..3169d721eb272 100644 --- a/modules/platforms/dotnet/SharedAssemblyInfo.cs +++ b/modules/platforms/dotnet/SharedAssemblyInfo.cs @@ -20,7 +20,7 @@ [assembly: AssemblyConfiguration("")] [assembly: AssemblyCompany("Apache Software Foundation")] [assembly: AssemblyProduct("Apache Ignite.NET")] -[assembly: AssemblyCopyright("Copyright 2022")] +[assembly: AssemblyCopyright("Copyright 2023")] [assembly: AssemblyTrademark("")] [assembly: AssemblyCulture("")] From 52170819598b784f13fd1f02bb3681cbc166138b Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Thu, 20 Apr 2023 11:38:02 +0300 Subject: [PATCH 13/22] IGNITE-19309 Update DEB & RPM package version to 2.15.0 - Fixes #10655. Signed-off-by: Aleksey Plekhanov (cherry picked from commit 0093f855be3bcaca4e25f96953557415b2720de4) --- packaging/deb/changelog | 6 ++++++ packaging/rpm/apache-ignite.spec | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/packaging/deb/changelog b/packaging/deb/changelog index b94a1c6a8510b..06c29aae90c17 100644 --- a/packaging/deb/changelog +++ b/packaging/deb/changelog @@ -1,3 +1,9 @@ +apache-ignite (2.15.0-1) unstable; urgency=low + + * Updated Apache Ignite to version 2.15.0 + + -- Alexey Plekhanov Thu, 20 Apr 2023 11:00:00 +0300 + apache-ignite (2.14.0-1) unstable; urgency=low * Updated Apache Ignite to version 2.14.0 diff --git a/packaging/rpm/apache-ignite.spec b/packaging/rpm/apache-ignite.spec index d3b980625e95e..76482271fdb77 100644 --- a/packaging/rpm/apache-ignite.spec +++ b/packaging/rpm/apache-ignite.spec @@ -12,7 +12,7 @@ # Name: apache-ignite -Version: 2.14.0 +Version: 2.15.0 Release: 1 Summary: Apache Ignite In-Memory Computing, Database and Caching Platform Group: Development/System From 79dbfc7ca32395cbeba04c7e2221863124928274 Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Thu, 20 Apr 2023 15:06:46 +0300 Subject: [PATCH 14/22] IGNITE-17194 Fix update version script - Fixes #9235. Signed-off-by: Aleksey Plekhanov (cherry picked from commit 3762c8b0a296f5768795d831c679b07a692e2588) --- docs/_config.yml | 2 +- modules/core/pom.xml | 27 ------- modules/ducktests/pom.xml | 27 ------- modules/platforms/cpp/pom.xml | 68 ----------------- modules/platforms/dotnet/DEVNOTES.txt | 2 +- modules/platforms/dotnet/pom.xml | 29 ------- pom.xml | 104 +++++++++++++++++++++++++- scripts/update-versions.sh | 2 +- 8 files changed, 105 insertions(+), 156 deletions(-) delete mode 100644 modules/platforms/cpp/pom.xml diff --git a/docs/_config.yml b/docs/_config.yml index 48045baf026fb..e658e18378f56 100644 --- a/docs/_config.yml +++ b/docs/_config.yml @@ -25,7 +25,7 @@ attrs: &asciidoc_attributes csharpCodeDir: code-snippets/dotnet githubUrl: https://github.com/apache/ignite/tree/master githubVersionUrl: https://github.com/apache/ignite/tree/ignite-{version} - docSourceUrl: https://github.com/apache/ignite/tree/IGNITE-7595/docs + docSourceUrl: https://github.com/apache/ignite/tree/master/docs collections: docs: permalink: /docs/:path:output_ext diff --git a/modules/core/pom.xml b/modules/core/pom.xml index 9913700b80712..6981f15671e42 100644 --- a/modules/core/pom.xml +++ b/modules/core/pom.xml @@ -295,33 +295,6 @@ false - - - org.apache.maven.plugins - maven-antrun-plugin - - - update-versions - - run - - initialize - - - - - - - - - - - - - - - - diff --git a/modules/ducktests/pom.xml b/modules/ducktests/pom.xml index 8ee6628000a0c..7855d62bf4ac9 100644 --- a/modules/ducktests/pom.xml +++ b/modules/ducktests/pom.xml @@ -121,33 +121,6 @@ - - org.apache.maven.plugins - maven-antrun-plugin - - - update-versions - - run - - initialize - - - - - - - - - - - - - - - - - maven-dependency-plugin diff --git a/modules/platforms/cpp/pom.xml b/modules/platforms/cpp/pom.xml deleted file mode 100644 index 6edf82ad14dfa..0000000000000 --- a/modules/platforms/cpp/pom.xml +++ /dev/null @@ -1,68 +0,0 @@ - - - - - - - 4.0.0 - - - org.apache.ignite - ignite-parent-internal - ${revision} - ../../../parent-internal/pom.xml - - - ignite-cpp - pom - - - - - org.apache.maven.plugins - maven-antrun-plugin - - - update-versions - - run - - initialize - - - - - - - - - - - - - - - - - - - diff --git a/modules/platforms/dotnet/DEVNOTES.txt b/modules/platforms/dotnet/DEVNOTES.txt index ed060dcc49098..80d5c04d5cd29 100644 --- a/modules/platforms/dotnet/DEVNOTES.txt +++ b/modules/platforms/dotnet/DEVNOTES.txt @@ -24,7 +24,7 @@ Note that .NET Core SDK 3.1 and PowerShell must be installed first. The command below will build the binaries and NuGet packages with Maven: - ./mvnw clean install -am -pl modules/platforms/dotnet -DskipTests -P platforms + ./mvnw clean install -am -pl modules/platforms/dotnet -DskipTests -P build-dotnet Solutions and Projects diff --git a/modules/platforms/dotnet/pom.xml b/modules/platforms/dotnet/pom.xml index 23dc450ec0350..bb7fecb2fc5f1 100644 --- a/modules/platforms/dotnet/pom.xml +++ b/modules/platforms/dotnet/pom.xml @@ -60,35 +60,6 @@ org.apache.maven.plugins maven-antrun-plugin - - update-versions - - run - - initialize - - - - - - - - - - - - - - - - - - - - - - build-dotnet generate-sources diff --git a/pom.xml b/pom.xml index f38183fcc2f7f..9ae0ee861b090 100644 --- a/pom.xml +++ b/pom.xml @@ -116,10 +116,9 @@ - platforms + build-dotnet modules/platforms/dotnet - modules/platforms/cpp @@ -474,6 +473,107 @@ + + update-versions + + + + + org.apache.maven.plugins + maven-antrun-plugin + + + update-versions + + run + + validate + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ${revision} From 2479e25a8da5eaf4af079f99a805b7c8e5e775e9 Mon Sep 17 00:00:00 2001 From: Sergey Korotkov Date: Sat, 22 Apr 2023 22:12:55 +0300 Subject: [PATCH 16/22] IGNITE-19347 Bump the log4j2 version because of a performance regression - Fixes #10661. Signed-off-by: Aleksey Plekhanov (cherry picked from commit b04fe32f36f9eaa0b4be68dfdc28a53b5bf40058) --- parent/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parent/pom.xml b/parent/pom.xml index 16ac012c24cbe..9485875b837a9 100644 --- a/parent/pom.xml +++ b/parent/pom.xml @@ -115,7 +115,7 @@ 2.4 r938 2.0.1 - 2.18.0 + 2.20.0 7.4.0_1 8.11.2 1.8.0 From df9faede59aa0cbb4a4eb90315eaf4e9ff78f6d5 Mon Sep 17 00:00:00 2001 From: liyujue <18624049226@163.com> Date: Sat, 22 Apr 2023 22:21:29 +0300 Subject: [PATCH 17/22] IGNITE-19349 Documentation: Remove docs related to ignitevisorcmd - Fixes #10662. Signed-off-by: Aleksey Plekhanov (cherry picked from commit 1a69152c2f6bec549410d0f391f00a34e5f19edd) --- docs/_data/toc.yaml | 2 - docs/_docs/tools/visor-cmd.adoc | 68 --------------------------------- 2 files changed, 70 deletions(-) delete mode 100644 docs/_docs/tools/visor-cmd.adoc diff --git a/docs/_data/toc.yaml b/docs/_data/toc.yaml index 55feaa956a264..4f561ccdabb01 100644 --- a/docs/_data/toc.yaml +++ b/docs/_data/toc.yaml @@ -458,8 +458,6 @@ items: - title: Control Script url: tools/control-script - - title: Visor CMD - url: tools/visor-cmd - title: GridGain Control Center url: tools/gg-control-center - title: SQLLine diff --git a/docs/_docs/tools/visor-cmd.adoc b/docs/_docs/tools/visor-cmd.adoc deleted file mode 100644 index 590ec16603cbd..0000000000000 --- a/docs/_docs/tools/visor-cmd.adoc +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one or more -// contributor license agreements. See the NOTICE file distributed with -// this work for additional information regarding copyright ownership. -// The ASF licenses this file to You under the Apache License, Version 2.0 -// (the "License"); you may not use this file except in compliance with -// the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -= Visor CMD - -== Overview - -Visor Command Line Interface (CMD) is a command-line tool for Ignite clusters monitoring. It provides basic statistics -about cluster nodes, caches, and compute tasks. It also lets you manage the size of your cluster by starting or stopping nodes. - -[NOTE] -==== -[discrete] -=== Ignite Control Script -The link:tools/control-script[Control Script] is another command-line tool developed by the Ignite community. -It complements and expands capabilities of Visor CMD. -==== - -image::images/tools/visor-cmd.png[Visor CMD] - -== Usage - -Ignite ships the `IGNITE_HOME/bin/ignitevisorcmd.{sh|bat}` script that starts Visor CMD. To connect Visor CMD to a cluster, -use the `open` command. - -The following commands are supported by Visor. To get full information on a command, type `help "cmd"` or `? "cmd"`. - -[cols="15%,15%,70%", opts="header"] -|=== -|Command | Alias | Description - -| `ack`| | Acks arguments on all remote nodes. -| `alert`| | Alerts for user-defined events. -| `cache`| | Prints cache statistics, clears cache, prints list of all entries from cache. -| `close`| | Disconnects Visor CMD console from the cluster. -| `config`| | Prints nodes' configurations. -| `deploy`| | Copies file or folder to remote host. -| `disco`| | Prints topology change log. -| `events`| | Prints events from a node. -| `gc`| | Runs GC on remote nodes. -| `help`| `?`| Visor CMD's help. -| `kill`| | Kills or restarts a node. -| `log`| | Starts or stops the cluster-wide events logging. -| `mclear`| | Clears Visor CMD's memory variables. -| `mget`| | Gets Visor CMD' memory variables -| `mlist`| | Prints Visor CMD's memory variables. -| `node`| | Prints node's statistics. -| `open`| | Connects Visor CMD to the cluster. -| `ping`| | Pings a node. -| `quit`| | Close Visor CMD's connection. -| `start`| | Starts or restarts remote nodes. -| `status`| `!`| Prints detailed Visor CMD's status. -| `tasks`| | Prints tasks' execution statistics. -| `top`| | Prints the current cluster topology. -| `vvm`| | Opens VisualVM for nodes in the cluster. -|=== - From b2dd87eed93d42fec53bb9391f53e7e990c22261 Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Tue, 25 Apr 2023 17:13:37 +0300 Subject: [PATCH 18/22] IGNITE-19312 Apache Ignite 2.15.0 release notes - Fixes #10665. Signed-off-by: Aleksey Plekhanov (cherry picked from commit 2a7cb59571f6dd98bb5e6a4e4c49cb799504a239) --- RELEASE_NOTES.txt | 150 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt index e0baeab3b1b41..b7ef164c98187 100644 --- a/RELEASE_NOTES.txt +++ b/RELEASE_NOTES.txt @@ -1,6 +1,156 @@ Apache Ignite Release Notes =========================== +Apache Ignite In-Memory Distributed Database 2.15.0 +----------------------------------------------------------- + +(!) WARNINGS: +* Removed deprecated daemon mode for Ignite nodes. +* Removed deprecated ignitevisorcmd tool. +* Removed legacy JMX metric beans (ThreadPoolMXBean, CacheGroupMetricsMXBean, CacheMetricsMXBean, PersistenceMetricsMXBean, DataStorageMetricsMXBean, DataRegionMetricsMXBean). +* Removed redundant ignite-spring to ignite-indexing module dependency. If you use SQL in Ignite and get Ignite dependency via maven, check that you have explicit dependency to the one of the SQL engines (ignite-indexing or ignite-calcite). +* Moved ignite-ssh module to the ignite-extensions. +* Deprecated lazy flag of SqlFieldsQuery. Default value for this flag is changed to 'true'. +* Operations on atomic caches within transactions are forbidden by default now. To allow such operations set system property IGNITE_ALLOW_ATOMIC_OPS_IN_TX to 'true'. + +Ignite: +* Added new feature: Incremental snapshots. +* Added cache objects transformation feature (this can be used to encrypt, compress or filter the data). +* Added support of caches with disk page compression for snapshots. +* Added ability to copy only primary partitions to snapshot. +* Added an option to skip snapshot check during restore. +* Added a snapshot exception if a data streamer is loading data with no consistency guarantee. +* Added storing of snapshot operation warnings to a meta. +* Added snapshot delta pages sorting. +* Added partition size and counter check on snapshot create. +* Added system view for Ignite configuration. +* Added "Page free space" column to pages lists system views. +* Added API to kill client (THIN, ODBJ, JDBC) connections. +* Added metrics for page read time and page replace time. +* Added thin client affinity hits/misses metric (for partition awareness requests). +* Added distributed property to disable CDC. +* Added maximum CDC directory size configuration parameter. +* Added expiry policy support for CDC. +* Added ability to provide expireTime during CDC conflict resolution. +* Added LOB and BLOB fields support to CacheJdbcPojoStore. +* Added missing "--add-opens" for Java 15+ runtime. +* Added default constructor to PlatformDotNetSessionLockResult to avoid issues with ReflectionFactory on some application servers. +* Added ability to use configured FailureHandler for segmentation handling. +* Added authorization of .NET and C++ compute tasks. +* Added authorization of Ignite Cluster Node stop/start/restart operations. +* Added authorization of SQL user management operations. +* Added 'ADMIN_CLUSTER_STATE' security permission (now it's the only required permission to change cluster state). +* Granting permissions to system compute tasks by their FQN has been replaced with predefined system permissions. +* Improved rebalance and PME logging (partition ranges disabled). +* Default metrics exporter is now the JMX exporter. +* Optimized index rebuilding (disable writing of WAL records for indexes during rebuilding). +* Optimized deactivation of large memory regions with persistence. +* Optimized zeroing of memory pages. +* Fixed lambdas serialization on Java 17. +* Fixed PDS defragmentation on Windows. +* Fixed JDBC connection leak with cache.invoke() over write-behind enabled cache. +* Fixed an issue that could lead to data corruption of atomic cache when a new updated entry is greater than WAL buffer size. +* Fixed snapshot create operation started from a non baseline node. +* Fixed snapshot inconsistency if it was taken under cache workload. +* Fixed snapshot cancel if one path is configured for several nodes. +* Fixed snapshot restore hanging if the prepare stage fails. +* Fixed ignite-cdc stopping on client node crush. +* Fixed ignite-cdc.sh start failure with non-defailt work directory. +* Fixed CdcMain state committing during index rebuild. +* Fixed CdcConsumer stopping on ignite-cdc.sh stop. +* Fixed the metric for the number of active client sessions. +* Fixed calculation of cache group storage size related metrics (PagesFillFactor, TotalUsedPages, TotalAllocatedPages). +* Fixed assertion on histogram update if currentTimeMillis decreases. +* Fixed REST status code in case of failed request. +* Fixed an issue that could lead to skipping 3rd party cache store while executing the REST API GET command. +* Fixed execution options propagation for compute methods. +* Fixed possible assertion in Communication SPI when the paired connections are set. +* Fixed potential data loss on historical rebalance. +* Fixed WAL timeout rollover. +* Fixed incorrect assertion on CRC calculation when disk page compression is enabled. +* Fixed node start failure if cpMapSnapshot.bin file is corrupted. +* Fixed PDS corruption on checkpoint after deactivation. +* Fixed PDS corruption on deactivation during entries expire. +* Fixed node failure on rebalancing during caches stop. +* Fixed NPE in case of simultaneous cache destroy and transaction rollback. +* Fixed incorrect id check in ClusterGroup.forNodeId. +* Fixed broken affinity after thick client reconnection. +* Fixed near cache creation on non-affinity nodes. +* Fixed possible memory corruption because of insufficient size of compression buffer. +* Fixed skipping verification of index and metastorage partitions in the snapshot verification process. +* Fixed criteria skipping after first IN clause for IndexQuery. +* Fixed WalWritingRate metric for LOG_ONLY and BACKGROUND modes. +* Updated log4j2 dependency to version 2.20.0. +* Updated Jetty dependency to 9.4.49.v20220914. +* Updated Jackson Databind dependency to 2.14.0. +* Updated postgresql dependency to 42.5.0. +* Updated Netty dependency to 4.1.89.Final. +* Updated Ant dependency to 1.10.13. +* Updated Apache Calcite dependency to 1.32.0. + +Control utility: +* Added cache clear command. +* Added cache scan command. +* Added multiple partitions support to consistency repair command. +* Added broken partitions list printing to idle-verify command. +* Added result saving into log file to idle_verify command. +* Improved entries versions checking by idle_verify command. +* Added a command to configure Histogram and Hitrate metrics. +* Added a command to forcefully resend CDC cache data. +* Added a command to delete segment CDC links before last gap. +* Added unified syntax for starting and canceling snapshot operations using control script. +* Added the ability to get a view from multiple nodes. +* Logging mechanism moved to IgniteLogger. + +Java thin-client: +* Added endpoints discovery feature. +* Added ClientConfiguration.clusterDiscoveryEnabled configuration property. +* Added events listening. +* Added connections balancing (use random open connection for requests instead of default). +* Added configurable logging. +* Fixed the pending requests race on close. + +Ignite .NET: +* Added support for default interface methods to Services. +* Added support for Native AOT. +* Added IgniteClientConfiguration.EnableClusterDiscovery thin client configuration property that can be used to enable or disable discovery. +* Fixed marshalling of nested binarizable structures with different schemas. +* Fixed startup on Java 15+. +* Fixed platform cache invalidation on client nodes with near cache. +* Fixed missing DLLs in release package. + +Ignite C++: +* Added ability to use java-based continuous query remote filters to C++ thick clients. +* Fixed SIGSEGV on thin client connection close. + +ODBC: +* Fixed handling of single quotes. + +SQL: +* Fixed redundant query results for commented lines. +* Fixed incorrect type conversion for index scans (for example, when indexed field has string data type and search row has numeric data type). +* Fixed potential deadlock in discovery thread while updating SQL statistics. + +Calcite-based SQL engine: +* Added ability to set up global heap memory quota and per-query hep memory quota. +* Added ability to scan inlined keys of indexes without additional requests to data pages. +* Added native support for SEARCH/SARG operator (this makes possible index scans for IN and some other conditions). +* Added support for LocalDate/LocalTime types. +* Added support of index scans on boolean fields. +* Added EVERY/SOME aggregate functions. +* Added details to Calcite parser exception. +* Added new distribution type (optimization to make some joins colocated). +* Added optimization for COUNT(column) if column is indexed. +* Added optimization: merge index conditions for the same field. +* Added optimization: use index first/last value for min()/max() aggregates. +* Added optimization: push not correlated filter part to the table scan. +* implemented query metadata, required for ODBC and JDBC. +* Optimized index scans with conditions containing NULLs. +* Fixed scans to exclude expired entries. +* Fixed row count estimate by limit with offset. +* Fixed CAST error from numbers to boolean. +* Fixed NPE on DML on non-existed table. + Apache Ignite In-Memory Distributed Database 2.14.0 ----------------------------------------------------------- From f98f7f35de6dc76a9b69299154afaa2139a5ec6d Mon Sep 17 00:00:00 2001 From: Aleksey Plekhanov Date: Tue, 25 Apr 2023 18:16:37 +0300 Subject: [PATCH 19/22] IGNITE-19364 Fix DEB/RPM packages assembly script (remove ignitevisorcmd) - Fixes #10666. Signed-off-by: Aleksey Plekhanov --- packaging/rpm/apache-ignite.spec | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packaging/rpm/apache-ignite.spec b/packaging/rpm/apache-ignite.spec index 76482271fdb77..8e9285c9a129b 100644 --- a/packaging/rpm/apache-ignite.spec +++ b/packaging/rpm/apache-ignite.spec @@ -83,7 +83,7 @@ case $1 in setPermissions # Install alternatives - # Commented out until ignitevisorcmd / ignitesqlline is ready to work from any user + # Commented out until ignitesqlline is ready to work from any user #update-alternatives --install %{_bindir}/ignitesqlline ignitesqlline %{_datadir}/%{name}/bin/sqlline.sh 0 #update-alternatives --auto ignitesqlline #update-alternatives --display ignitesqlline @@ -131,7 +131,7 @@ case $1 in stopIgniteNodes # Remove alternatives - # Commented out until ignitevisorcmd / ignitesqlline is ready to work from any user + # Commented out until ignitesqlline is ready to work from any user #update-alternatives --remove ignitesqlline /usr/share/%{name}/bin/sqlline.sh #update-alternatives --display ignitesqlline || true ;; @@ -198,7 +198,6 @@ mkdir -p %{buildroot}%{_bindir} # Copy nessessary files and remove *.bat files cp -rf benchmarks bin platforms %{buildroot}%{_datadir}/%{name} cp -rf docs/* examples %{buildroot}%{_datadir}/doc/%{name}-%{version} -mv -f %{buildroot}%{_datadir}/%{name}/bin/ignitevisorcmd.sh %{buildroot}%{_datadir}/doc/%{name}-%{version}/bin/ find %{buildroot}%{_datadir}/%{name}/ -name *.bat -exec rm -rf {} \; # Copy libs to /usr/lib and map them to IGNITE_HOME From c93ab9a907e6e87a9062f36b95ae901d7f0df767 Mon Sep 17 00:00:00 2001 From: Maxim Muzafarov Date: Fri, 22 Sep 2023 00:30:09 +0200 Subject: [PATCH 20/22] IGNITE-20466 Investigate and fix the issue with running this workflow on PRs from forks --- .github/workflows/sonar-pr-from-fork.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sonar-pr-from-fork.yml b/.github/workflows/sonar-pr-from-fork.yml index f96b97987fabd..02288bd5b70c0 100644 --- a/.github/workflows/sonar-pr-from-fork.yml +++ b/.github/workflows/sonar-pr-from-fork.yml @@ -17,7 +17,8 @@ name: Sonar Quality Pull Request Analysis -on: [pull_request_target] +# TODO IGNITE-20466 Investigate and fix the issue with running this workflow on PRs from forks. +on: pull_request permissions: contents: read From 30ffbc0d7e88fea076868110e9de0a1deedf038c Mon Sep 17 00:00:00 2001 From: Niall Pemberton Date: Mon, 17 Mar 2025 05:47:27 +0000 Subject: [PATCH 21/22] IGNITE-24264 Replace Google Analytics with Matomo (#11926) --- docs/_layouts/default.html | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/docs/_layouts/default.html b/docs/_layouts/default.html index 9c7a42e4bb820..6ed6fce6ca063 100644 --- a/docs/_layouts/default.html +++ b/docs/_layouts/default.html @@ -20,16 +20,6 @@ - - - - {% if page.content == nil or page.content == "" %} {% endif %} @@ -54,6 +44,23 @@ + + + From 612b9a7a62f164e75d1151552d46c93c2db4e3aa Mon Sep 17 00:00:00 2001 From: Aleksandr Chesnokov Date: Tue, 21 Apr 2026 13:38:38 +0300 Subject: [PATCH 22/22] IGNITE-28581 Fixed performance statistics documentation images (#13051) (cherry picked from commit 890368f8e8a6ac853a557ce1d03791881cb311f4) --- .../performance-statistics.adoc | 12 ++++++------ .../performance-statistics/perf_stat_1.jpg | Bin .../performance-statistics/perf_stat_2.jpg | Bin .../performance-statistics/perf_stat_3.jpg | Bin .../performance-statistics/perf_stat_4.jpg | Bin .../performance-statistics/perf_stat_5.jpg | Bin 6 files changed, 6 insertions(+), 6 deletions(-) rename docs/{assets => _docs}/images/integrations/performance-statistics/perf_stat_1.jpg (100%) rename docs/{assets => _docs}/images/integrations/performance-statistics/perf_stat_2.jpg (100%) rename docs/{assets => _docs}/images/integrations/performance-statistics/perf_stat_3.jpg (100%) rename docs/{assets => _docs}/images/integrations/performance-statistics/perf_stat_4.jpg (100%) rename docs/{assets => _docs}/images/integrations/performance-statistics/perf_stat_5.jpg (100%) diff --git a/docs/_docs/extensions-and-integrations/performance-statistics.adoc b/docs/_docs/extensions-and-integrations/performance-statistics.adoc index e79cb29b5297b..654851960650b 100644 --- a/docs/_docs/extensions-and-integrations/performance-statistics.adoc +++ b/docs/_docs/extensions-and-integrations/performance-statistics.adoc @@ -54,20 +54,20 @@ performance-statistics-tool/build-report.sh --help === Performance Statistics Report Example -:perf_stat_url: ../../assets/images/integrations/performance-statistics +:perf_stat_url: images/integrations/performance-statistics [tabs] -- tab:Cache operations[] -image:{perf_stat_url}/perf_stat_2.jpg[link="{perf_stat_url}/perf_stat_2.jpg",window=_blank] +image:{perf_stat_url}/perf_stat_2.jpg[Cache operations report] tab:Transactions[] -image:{perf_stat_url}/perf_stat_3.jpg[link="{perf_stat_url}/perf_stat_3.jpg",window=_blank] +image:{perf_stat_url}/perf_stat_3.jpg[Transactions report] tab:Queries[] -image:{perf_stat_url}/perf_stat_4.jpg[link="{perf_stat_url}/perf_stat_4.jpg",window=_blank] +image:{perf_stat_url}/perf_stat_4.jpg[Queries report] tab:Compute[] -image:{perf_stat_url}/perf_stat_5.jpg[link="{perf_stat_url}/perf_stat_5.jpg",window=_blank] +image:{perf_stat_url}/perf_stat_5.jpg[Compute report] tab:Cluster info[] -image:{perf_stat_url}/perf_stat_1.jpg[link="{perf_stat_url}/perf_stat_1.jpg",window=_blank] +image:{perf_stat_url}/perf_stat_1.jpg[Cluster info report] -- == Print Statistics diff --git a/docs/assets/images/integrations/performance-statistics/perf_stat_1.jpg b/docs/_docs/images/integrations/performance-statistics/perf_stat_1.jpg similarity index 100% rename from docs/assets/images/integrations/performance-statistics/perf_stat_1.jpg rename to docs/_docs/images/integrations/performance-statistics/perf_stat_1.jpg diff --git a/docs/assets/images/integrations/performance-statistics/perf_stat_2.jpg b/docs/_docs/images/integrations/performance-statistics/perf_stat_2.jpg similarity index 100% rename from docs/assets/images/integrations/performance-statistics/perf_stat_2.jpg rename to docs/_docs/images/integrations/performance-statistics/perf_stat_2.jpg diff --git a/docs/assets/images/integrations/performance-statistics/perf_stat_3.jpg b/docs/_docs/images/integrations/performance-statistics/perf_stat_3.jpg similarity index 100% rename from docs/assets/images/integrations/performance-statistics/perf_stat_3.jpg rename to docs/_docs/images/integrations/performance-statistics/perf_stat_3.jpg diff --git a/docs/assets/images/integrations/performance-statistics/perf_stat_4.jpg b/docs/_docs/images/integrations/performance-statistics/perf_stat_4.jpg similarity index 100% rename from docs/assets/images/integrations/performance-statistics/perf_stat_4.jpg rename to docs/_docs/images/integrations/performance-statistics/perf_stat_4.jpg diff --git a/docs/assets/images/integrations/performance-statistics/perf_stat_5.jpg b/docs/_docs/images/integrations/performance-statistics/perf_stat_5.jpg similarity index 100% rename from docs/assets/images/integrations/performance-statistics/perf_stat_5.jpg rename to docs/_docs/images/integrations/performance-statistics/perf_stat_5.jpg