Merge branch 'master' of https://github.com/apache/nifi into aws-web-api

Author: ottobackwards

nifi-api/src/main/java/org/apache/nifi/controller/status/ProcessGroupStatus.java

@@ -481,7 +481,9 @@ public static void merge(final ProcessGroupStatus target, final ProcessGroupStat
             // and should not differ amongst nodes. however, whether a processor is invalid
             // can be driven by environmental conditions. this check allows any of those to
             // take precedence over the configured run status.
-            if (RunStatus.Invalid.equals(statusToMerge.getRunStatus())) {
+            if (RunStatus.Validating.equals(statusToMerge.getRunStatus())) {
+                merged.setRunStatus(RunStatus.Validating);
+            } else if (RunStatus.Invalid.equals(statusToMerge.getRunStatus())) {
                 merged.setRunStatus(RunStatus.Invalid);
             }
         }

nifi-api/src/main/java/org/apache/nifi/controller/status/RunStatus.java

@@ -23,6 +23,7 @@ public enum RunStatus {
 
     Running,
     Stopped,
+    Validating,
     Invalid,
     Disabled;
 }

nifi-commons/nifi-expression-language/pom.xml

@@ -76,7 +76,7 @@
         <dependency>
             <groupId>com.jayway.jsonpath</groupId>
             <artifactId>json-path</artifactId>
-            <version>2.0.0</version>
+            <version>2.4.0</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

nifi-docs/src/main/asciidoc/administration-guide.adoc

@@ -3474,6 +3474,7 @@ and can be viewed in the Cluster page. This value indicates how many events to k
 the connection a failure. The default value is `5 secs`.
 |nifi.cluster.node.read.timeout|When communicating with another node in the cluster, specifies how long this node should wait to receive information
 from the remote node before considering the communication with the node a failure. The default value is `5 secs`.
+|nifi.cluster.node.max.concurrent.requests|The maximum number of outstanding web requests that can be replicated to nodes in the cluster. If this number of requests is exceeded, the embedded Jetty server will return a "409: Conflict" response.  This property defaults to `100`.
 |nifi.cluster.firewall.file|The location of the node firewall file. This is a file that may be used to list all the nodes that are allowed to connect
 to the cluster. It provides an additional layer of security. This value is blank by default, meaning that no firewall file is to be used.
 |nifi.cluster.flow.election.max.wait.time|Specifies the amount of time to wait before electing a Flow as the "correct" Flow. If the number of Nodes that have voted is equal to the number specified by the `nifi.cluster.flow.election.max.candidates` property, the cluster will not wait this long. The default value is `5 mins`. Note that the time starts as soon as the first vote is cast.

nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/DeleteOperationResponse.java

@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.elasticsearch;
+
+public class DeleteOperationResponse {
+    private long took;
+
+    public DeleteOperationResponse(long took) {
+        this.took = took;
+    }
+
+    public long getTook() {
+        return took;
+    }
+}

nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/ElasticSearchClientService.java

@@ -25,6 +25,8 @@
 import org.apache.nifi.ssl.SSLContextService;
 
 import java.io.IOException;
+import java.util.List;
+import java.util.Map;
 
 @Tags({"elasticsearch", "client"})
 @CapabilityDescription("A controller service for accessing an ElasticSearch client.")
@@ -95,6 +97,66 @@ public interface ElasticSearchClientService extends ControllerService {
             .addValidator(StandardValidators.NON_BLANK_VALIDATOR)
             .build();
 
+    /**
+     * Index a document.
+     *
+     * @param operation A document to index.
+     * @return IndexOperationResponse if successful
+     * @throws IOException thrown when there is an error.
+     */
+    IndexOperationResponse add(IndexOperationRequest operation) throws IOException;
+
+    /**
+     * Index multiple documents.
+     *
+     * @param operations A list of documents to index.
+     * @return IndexOperationResponse if successful.
+     * @throws IOException thrown when there is an error.
+     */
+    IndexOperationResponse add(List<IndexOperationRequest> operations) throws IOException;
+
+    /**
+     * Delete a document by its ID from an index.
+     *
+     * @param index The index to target.
+     * @param type The type to target. Optional.
+     * @param id The document ID to remove from the selected index.
+     * @return A DeleteOperationResponse object if successful.
+     */
+    DeleteOperationResponse deleteById(String index, String type, String id) throws IOException;
+
+
+    /**
+     * Delete multiple documents by ID from an index.
+     * @param index The index to target.
+     * @param type The type to target. Optional.
+     * @param ids A list of document IDs to remove from the selected index.
+     * @return A DeleteOperationResponse object if successful.
+     * @throws IOException thrown when there is an error.
+     */
+    DeleteOperationResponse deleteById(String index, String type, List<String> ids) throws IOException;
+
+    /**
+     * Delete documents by query.
+     *
+     * @param query A valid JSON query to be used for finding documents to delete.
+     * @param index The index to target.
+     * @param type The type to target within the index. Optional.
+     * @return A DeleteOperationResponse object if successful.
+     */
+    DeleteOperationResponse deleteByQuery(String query, String index, String type) throws IOException;
+
+    /**
+     * Get a document by ID.
+     *
+     * @param index The index that holds the document.
+     * @param type The document type. Optional.
+     * @param id The document ID
+     * @return Map if successful, null if not found.
+     * @throws IOException thrown when there is an error.
+     */
+    Map<String, Object> get(String index, String type, String id) throws IOException;
+
     /**
      * Perform a search using the JSON DSL.
      *

nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/IndexOperationRequest.java

@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.elasticsearch;
+
+import java.util.Map;
+
+public class IndexOperationRequest {
+    private String index;
+    private String type;
+    private String id;
+    private Map<String, Object> fields;
+
+    public IndexOperationRequest(String index, String type, String id, Map<String, Object> fields) {
+        this.index = index;
+        this.type = type;
+        this.id = id;
+        this.fields = fields;
+    }
+
+    public String getIndex() {
+        return index;
+    }
+
+    public String getType() {
+        return type;
+    }
+
+    public String getId() {
+        return id;
+    }
+
+    public Map<String, Object> getFields() {
+        return fields;
+    }
+}

nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service-api/src/main/java/org/apache/nifi/elasticsearch/IndexOperationResponse.java

@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.elasticsearch;
+
+public class IndexOperationResponse {
+    private long took;
+    private long ingestTook;
+
+    public IndexOperationResponse(long took, long ingestTook) {
+        this.took = took;
+        this.ingestTook = ingestTook;
+    }
+
+    public long getTook() {
+        return took;
+    }
+
+    public long getIngestTook() {
+        return ingestTook;
+    }
+}

nifi-nar-bundles/nifi-elasticsearch-bundle/nifi-elasticsearch-client-service/pom.xml

@@ -69,11 +69,6 @@
             <version>2.6</version>
         </dependency>
 
-        <dependency>
-            <groupId>org.elasticsearch.client</groupId>
-            <artifactId>rest</artifactId>
-            <version>5.0.1</version>
-        </dependency>
         <dependency>
             <groupId>com.github.stephenc.findbugs</groupId>
             <artifactId>findbugs-annotations</artifactId>
@@ -120,6 +115,18 @@
             <version>1.7.0-SNAPSHOT</version>
             <scope>provided</scope>
         </dependency>
+        <dependency>
+            <groupId>org.elasticsearch</groupId>
+            <artifactId>elasticsearch</artifactId>
+            <version>5.6.8</version>
+            <scope>compile</scope>
+        </dependency>
+        <dependency>
+            <groupId>org.elasticsearch.client</groupId>
+            <artifactId>elasticsearch-rest-high-level-client</artifactId>
+            <version>5.6.8</version>
+            <scope>compile</scope>
+        </dependency>
     </dependencies>
 
     <profiles>
@@ -137,7 +144,6 @@
                             <httpPort>9400</httpPort>
                             <version>5.6.2</version>
                             <timeout>90</timeout>
-                            <pathInitScript>${project.basedir}/src/test/resources/setup.script</pathInitScript>
                         </configuration>
                         <executions>
                             <execution>