Extend the APIs for block requests [ECR-3255] (#953)

Add `ExonumClient#findNonEmptyBlocks` to find a certain number
of the most recent non-empty blocks (from the last block and up to the genesis block).

Also change `ExonumClient#getBlocks` and  `ExonumClient#getLastBlocks`
to return as many blocks as requested (regardless of the limitations of 
the underlying transport). 

Both now return blocks from a certain closed range,
with empty blocks filtering no longer extending that range.

Author: dmitry-timofeev

exonum-light-client/CHANGELOG.md

@@ -15,10 +15,23 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 
 ## Unreleased
 
+### Added
+* `ExonumClient#findNonEmptyBlocks` to find a certain number of the most recent non-empty
+  blocks (from the last block and up to the genesis block). (#953)
+
+### Changed
+- `ExonumClient#getBlocks` accepts a closed range of block heights _[from; to]_
+  and returns only the blocks which heights fall into that range. The size of
+  the range _(to - from + 1)_ is no longer limited. If needed, the client
+  performs multiple requests to the node. (#953)
+- `ExonumClient#getLastBlocks` returns only the blocks from the closed range 
+  _[max(0, blockchainHeight - count + 1), blockchainHeight]_. The size of
+  the range _(min(count, blockchainHeight + 1))_ is no longer limited. (#953)
+
 ## 0.2.0 - 2019-05-27
 
 Second release of Exonum Java Light Client which brings
-system API and blockhain explorer API endpoints support.
+system API and blockchain explorer API endpoints support.
 
 ### Versions Support
 - Exonum version, 0.11.0

exonum-light-client/checkstyle-suppressions.xml

@@ -23,4 +23,5 @@
 <suppressions>
     <!-- Allow `aBlock` name -->
     <suppress files="BlockTest\.java" checks="MethodName"/>
+    <suppress files="Blocks\.java" checks="MethodName"/>
 </suppressions>

exonum-light-client/pom.xml

@@ -229,6 +229,18 @@
           <argLine>
             ${jacoco.args}
           </argLine>
+          <!-- Set up JUnit 5 configuration parameters.
+          See also: https://junit.org/junit5/docs/current/user-guide/#running-tests-build-maven-config-params -->
+          <properties>
+            <configurationParameters>
+              # Enable parallel execution, with concurrent mode by default
+              junit.jupiter.execution.parallel.enabled = true
+              junit.jupiter.execution.parallel.mode.default = concurrent
+
+              # Use 1.5 times the number of CPUs
+              junit.jupiter.execution.parallel.config.dynamic.factor = 1.5
+            </configurationParameters>
+          </properties>
         </configuration>
       </plugin>
 
@@ -284,6 +296,19 @@
           </execution>
         </executions>
       </plugin>
+
+      <plugin>
+        <groupId>org.pitest</groupId>
+        <artifactId>pitest-maven</artifactId>
+        <version>1.4.8</version>
+        <dependencies>
+          <dependency>
+            <groupId>org.pitest</groupId>
+            <artifactId>pitest-junit5-plugin</artifactId>
+            <version>0.8</version>
+          </dependency>
+        </dependencies>
+      </plugin>
     </plugins>
   </build>
 
@@ -314,6 +339,7 @@
                 <exclude>com/exonum/client/response/Block.class</exclude>
                 <exclude>com/exonum/client/response/Block*BlockBuilder.class</exclude>
                 <exclude>com/exonum/client/response/BlockResponse.class</exclude>
+                <exclude>com/exonum/client/response/BlocksRange.class</exclude>
                 <exclude>com/exonum/client/response/BlocksResponse.class</exclude>
                 <exclude>com/exonum/client/response/HealthCheckInfo.class</exclude>
                 <exclude>com/exonum/client/response/TransactionResponse.class</exclude>

exonum-light-client/src/main/java/com/exonum/client/ExonumApi.java

@@ -16,11 +16,23 @@
 
 package com.exonum.client;
 
-public final class ExonumApi {
+import com.exonum.binding.common.serialization.json.JsonSerializer;
+import com.google.gson.FieldNamingPolicy;
+import com.google.gson.Gson;
+
+final class ExonumApi {
+  /**
+   * The Gson instance configured to (de)serialize Exonum responses.
+   */
+  static final Gson JSON = JsonSerializer.builder()
+      .setFieldNamingPolicy(FieldNamingPolicy.LOWER_CASE_WITH_UNDERSCORES)
+      .create();
+
   /**
    * The maximum allowed blocks count per the request.
    */
-  public static final int MAX_BLOCKS_PER_REQUEST = 1000;
+  static final int MAX_BLOCKS_PER_REQUEST = 1000;
+
 
   private ExonumApi() {
   }

exonum-light-client/src/main/java/com/exonum/client/ExonumClient.java

@@ -25,11 +25,12 @@
 import com.exonum.client.request.BlockTimeOption;
 import com.exonum.client.response.Block;
 import com.exonum.client.response.BlockResponse;
-import com.exonum.client.response.BlocksResponse;
+import com.exonum.client.response.BlocksRange;
 import com.exonum.client.response.HealthCheckInfo;
 import com.exonum.client.response.TransactionResponse;
 import java.net.MalformedURLException;
 import java.net.URL;
+import java.util.List;
 import java.util.Optional;
 import okhttp3.OkHttpClient;
 
@@ -38,9 +39,10 @@
  * Provides a convenient way for interaction with Exonum framework APIs.
  * All the methods of the interface work in a blocking way
  * i.e. invoke underlying request immediately, and block until the response can be processed
- * or an error occurs.
+ * or an error occurs. In case the thread is interrupted, the blocked methods will complete
+ * exceptionally.
  *
- * <p><i>Implementations of that interface are required to be thread-safe</i>.
+ * <p><em>Implementations of this interface are required to be thread-safe</em>.
  **/
 public interface ExonumClient {
 
@@ -107,45 +109,68 @@ public interface ExonumClient {
   BlockResponse getBlockByHeight(long height);
 
   /**
-   * Returns blockchain blocks information for the requested range. The blocks are returned
-   * in reverse order, starting from the {@code heightMax}.
-   * @param count Number of blocks to return.
-   *        It should be in range [1, {@linkplain ExonumApi#MAX_BLOCKS_PER_REQUEST}]
+   * Returns blockchain blocks in the requested <em>closed</em> range. The blocks are returned
+   * in ascending order by their height.
+   *
+   * @param fromHeight the height of the first block to include. Must be non-negative
+   * @param toHeight the height of the last block to include. Must be greater than
+   *        or equal to {@code fromHeight} and less than or equal to the blockchain height.
+   *        If the {@code toHeight} is greater than actual blockchain height then
+   *        the actual height will be used (such error-prone behaviour will be fixed
+   *        in Exonum 0.12)
    * @param blockFilter controls whether to skip blocks with no transactions
-   * @param heightMax maximum height of the returned blocks.
-   *        If the {@code heightMax} is greater than actual blockchain height then
-   *        the actual height will be used
-   * @param timeOption controls whether to include the block commit time.
-   *        See {@linkplain Block#getCommitTime()}.
-   *        The time value corresponds to the average time of submission of precommits by the
-   *        validators for every returned block
-   * @return blocks information response
+   * @param timeOption controls whether to include
+   *        the {@linkplain Block#getCommitTime() block commit time}
+   * @return blocks in the requested range
    * @throws RuntimeException if the client is unable to complete a request
    *        (e.g., in case of connectivity problems)
-   * @throws IllegalArgumentException if count is out of range
-   *        [1, {@linkplain ExonumApi#MAX_BLOCKS_PER_REQUEST}]
+   * @throws IllegalArgumentException if {@code fromHeight} or {@code toHeight} are not valid
    */
-  BlocksResponse getBlocks(int count, BlockFilteringOption blockFilter, long heightMax,
+  List<Block> getBlocks(long fromHeight, long toHeight, BlockFilteringOption blockFilter,
       BlockTimeOption timeOption);
 
   /**
-   * Returns blockchain blocks information starting from the last block in the blockchain.
-   * @param count Number of blocks to return.
-   *        It should be in range [1, {@linkplain ExonumApi#MAX_BLOCKS_PER_REQUEST}]
-   * @param blockFilter controls whether to skip blocks with no transactions
-   * @param timeOption controls whether to include the block commit time.
-   *        See {@linkplain Block#getCommitTime()}.
-   *        The time value corresponds to the average time of submission of precommits by the
-   *        validators for every returned block
+   * Returns the range of the most recent blockchain blocks in ascending order by their height.
+   * More precisely, returns the blocks in the closed range
+   * {@code [max(0, blockchainHeight - size + 1), blockchainHeight]} of size
+   * {@code max(blockchainHeight + 1, size)}.
+   *
+   * @param size the size of the range. If it exceeds the number of blocks in the blockchain,
+   *        this method will return all blocks ({@code blockchainHeight + 1})
+   * @param blockFilter controls whether to skip blocks with no transactions. If filtering
+   *        is applied, the actual number of blocks may be smaller than {@code size};
+   *        but the range of blocks will not be extended beyond {@code blockchainHeight - size + 1}.
+   *        If a certain <em>number</em> of non-empty blocks is needed (not a certain
+   *        <em>range</em>), use {@link #findNonEmptyBlocks(int, BlockTimeOption)}
+   * @param timeOption controls whether to include
+   *        the {@linkplain Block#getCommitTime() block commit time}
    * @return blocks information response
    * @throws RuntimeException if the client is unable to complete a request
    *        (e.g., in case of connectivity problems)
-   * @throws IllegalArgumentException if count is out of range
-   *        [1, {@linkplain ExonumApi#MAX_BLOCKS_PER_REQUEST}]
+   * @throws IllegalArgumentException if size is non-positive
+   * @see #findNonEmptyBlocks(int, BlockTimeOption)
    */
-  BlocksResponse getLastBlocks(int count, BlockFilteringOption blockFilter,
+  BlocksRange getLastBlocks(int size, BlockFilteringOption blockFilter,
       BlockTimeOption timeOption);
 
+  /**
+   * Returns up to the given number of the most recent non-empty blocks in ascending order
+   * by their height. The search range is not limited, i.e., spans the whole blockchain.
+   *
+   * @param numBlocks the maximum number of blocks to return. Must be positive. If the number
+   *     of non-empty blocks in the blockchain is less than {@code numBlocks}, all such blocks
+   *     will be returned
+   * @param timeOption controls whether to include
+   *        the {@linkplain Block#getCommitTime() block commit time}
+   * @return a list of the most recent non-empty blocks
+   * @throws RuntimeException if the client is unable to complete a request
+   *        (e.g., in case of connectivity problems)
+   * @throws IllegalArgumentException if numBlocks is non-positive
+   * @see #getLastBlocks(int, BlockFilteringOption, BlockTimeOption)
+   * @see BlockFilteringOption#SKIP_EMPTY
+   */
+  List<Block> findNonEmptyBlocks(int numBlocks, BlockTimeOption timeOption);
+
   /**
    * Returns the last block in the blockchain.
    * @throws RuntimeException if the client is unable to complete a request

exonum-light-client/src/main/java/com/exonum/client/ExonumHttpClient.java

@@ -18,6 +18,7 @@
 package com.exonum.client;
 
 import static com.exonum.client.ExonumApi.MAX_BLOCKS_PER_REQUEST;
+import static com.exonum.client.ExonumIterables.indexOf;
 import static com.exonum.client.ExonumUrls.BLOCK;
 import static com.exonum.client.ExonumUrls.BLOCKS;
 import static com.exonum.client.ExonumUrls.HEALTH_CHECK;
@@ -30,6 +31,8 @@
 import static com.exonum.client.request.BlockTimeOption.NO_COMMIT_TIME;
 import static com.google.common.base.Preconditions.checkArgument;
 import static com.google.common.base.Preconditions.checkNotNull;
+import static java.lang.Math.max;
+import static java.lang.Math.min;
 import static java.net.HttpURLConnection.HTTP_NOT_FOUND;
 
 import com.exonum.binding.common.hash.HashCode;
@@ -38,12 +41,17 @@
 import com.exonum.client.request.BlockTimeOption;
 import com.exonum.client.response.Block;
 import com.exonum.client.response.BlockResponse;
+import com.exonum.client.response.BlocksRange;
 import com.exonum.client.response.BlocksResponse;
 import com.exonum.client.response.HealthCheckInfo;
 import com.exonum.client.response.TransactionResponse;
+import com.google.common.collect.ImmutableList;
+import com.google.common.collect.Lists;
 import java.io.IOException;
 import java.net.URL;
+import java.util.ArrayList;
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 import java.util.Optional;
 import java.util.function.Function;
@@ -60,6 +68,7 @@
  */
 class ExonumHttpClient implements ExonumClient {
   private static final MediaType MEDIA_TYPE_JSON = MediaType.get("application/json; charset=utf-8");
+  private static final int GENESIS_BLOCK_HEIGHT = 0;
 
   private final OkHttpClient httpClient;
   private final URL exonumHost;
@@ -141,19 +150,100 @@ public BlockResponse getBlockByHeight(long height) {
   }
 
   @Override
-  public BlocksResponse getBlocks(int count, BlockFilteringOption blockFilter, long heightMax,
+  public List<Block> getBlocks(long fromHeight, long toHeight, BlockFilteringOption blockFilter,
       BlockTimeOption timeOption) {
-    checkArgument(0 < count,
-        "Requested number of blocks should be positive number but was %s", count);
-    return doGetBlocks(count, blockFilter, heightMax, timeOption);
+    checkArgument(0 <= fromHeight, "First block height (%s) must be non-negative", fromHeight);
+    checkArgument(fromHeight <= toHeight,
+        "First block height (%s) should be less than or equal to the last block height (%s)",
+        fromHeight, toHeight);
+
+    // 'maximum' as when skipping empty the actual might be way smaller
+    int maxSize = Math.toIntExact(toHeight - fromHeight + 1);
+    List<Block> blocks = new ArrayList<>(maxSize);
+    for (long rangeLast = toHeight; rangeLast >= fromHeight; ) {
+      int remainingBlocks = Math.toIntExact(rangeLast - fromHeight + 1);
+      int numBlocks = min(remainingBlocks, MAX_BLOCKS_PER_REQUEST);
+      BlocksResponse blocksResponse = doGetBlocks(numBlocks, blockFilter, rangeLast, timeOption);
+
+      blocks.addAll(blocksResponse.getBlocks());
+
+      rangeLast = blocksResponse.getBlocksRangeStart() - 1;
+    }
+
+    return postProcessResponseBlocks(fromHeight, toHeight, blocks)
+        .getBlocks();
   }
 
   @Override
-  public BlocksResponse getLastBlocks(int count, BlockFilteringOption blockFilter,
+  public BlocksRange getLastBlocks(int size, BlockFilteringOption blockFilter,
       BlockTimeOption timeOption) {
-    checkArgument(0 < count,
-        "Requested number of blocks should be positive number but was %s", count);
-    return doGetBlocks(count, blockFilter, null, timeOption);
+    checkArgument(0 < size,
+        "Requested blocks range size should be positive but was %s", size);
+
+    List<Block> blocks = new ArrayList<>(size);
+    // The first request does not specify the maximum height to get the top blocks
+    long blockchainHeight = Long.MIN_VALUE;
+    Long nextHeight = null;
+    int remainingBlocks = size;
+    while (remainingBlocks > 0
+        && (nextHeight == null || nextHeight >= GENESIS_BLOCK_HEIGHT)) {
+      int numBlocks = min(remainingBlocks, MAX_BLOCKS_PER_REQUEST);
+      BlocksResponse blocksResponse = doGetBlocks(numBlocks, blockFilter, nextHeight, timeOption);
+
+      blocks.addAll(blocksResponse.getBlocks());
+
+      nextHeight = blocksResponse.getBlocksRangeStart() - 1;
+      blockchainHeight = max(blockchainHeight, blocksResponse.getBlocksRangeEnd() - 1);
+      remainingBlocks = Math.toIntExact(size - (blockchainHeight - nextHeight));
+    }
+
+    long fromHeight = max(blockchainHeight - size + 1, GENESIS_BLOCK_HEIGHT);
+    long toHeight = blockchainHeight;
+    return postProcessResponseBlocks(fromHeight, toHeight, blocks);
+  }
+
+  /**
+   * Post-processes the blocks, coming from
+   * {@link #doGetBlocks(int, BlockFilteringOption, Long, BlockTimeOption)}:
+   * 1. Turns them in ascending order by height.
+   * 2. Keeps only blocks that fall in range [fromHeight; toHeight].
+   */
+  private static BlocksRange postProcessResponseBlocks(long fromHeight, long toHeight,
+      List<Block> blocks) {
+    // Turn the blocks in ascending order
+    blocks = Lists.reverse(blocks);
+
+    // Filter the possible blocks that are out of the requested range
+    // No Stream#dropWhile in Java 8 :(
+    int firstInRange = indexOf(blocks, b -> b.getHeight() >= fromHeight)
+        .orElse(blocks.size());
+    blocks = blocks.subList(firstInRange, blocks.size());
+
+    // Do not bother trimming — BlocksRange copies the list
+    return new BlocksRange(fromHeight, toHeight, blocks);
+  }
+
+  @Override
+  public List<Block> findNonEmptyBlocks(int numBlocks, BlockTimeOption timeOption) {
+    checkArgument(0 < numBlocks,
+        "Requested number of blocks should be positive but was %s", numBlocks);
+
+    List<Block> blocks = new ArrayList<>(numBlocks);
+    Long nextHeight = null;
+    int remainingBlocks = numBlocks;
+    while (remainingBlocks > 0
+        && (nextHeight == null || nextHeight >= GENESIS_BLOCK_HEIGHT)) {
+      int numRequested = min(remainingBlocks, MAX_BLOCKS_PER_REQUEST);
+      BlocksResponse blocksResponse = doGetBlocks(numRequested, SKIP_EMPTY, nextHeight, timeOption);
+
+      blocks.addAll(blocksResponse.getBlocks());
+
+      nextHeight = blocksResponse.getBlocksRangeStart() - 1;
+      remainingBlocks -= blocksResponse.getBlocks().size();
+    }
+
+    List<Block> ascBlocks = Lists.reverse(blocks);
+    return ImmutableList.copyOf(ascBlocks);
   }
 
   @Override