Added the option to rollover multiple indices with a wildcard alias

Author: Chris Wiechmann

api-builder-plugin-fn-elasticsearch/CHANGELOG.md

@@ -4,6 +4,10 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## [1.0.16] 2020-12-15
+### Added
+- Added the option to rollover multiple indicies based on a wildcard alias
+
 ## [1.0.15] 2020-12-10
 ### Added
 - Added the option to disable the Elasticsearch connection validation during startup (e.g. environment variable: VALIDATE_ELASTIC_CONNECTION=false)

api-builder-plugin-fn-elasticsearch/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@axway-api-builder-ext/api-builder-plugin-fn-elasticsearch",
-	"version": "1.0.15",
+	"version": "1.0.16",
 	"description": "Integrate Elasticsearch into your API-Builder flow to combine search data for instance with other data available in your flow.",
 	"author": "Chris Wiechmann <cwiechmann@axway.com> (http://www.axway.com)",
 	"license": "Apache-2.0",

api-builder-plugin-fn-elasticsearch/src/actions/indices.js

@@ -17,29 +17,42 @@ const { ElasticsearchClient } = require('./ElasticsearchClient');
  */
 
 async function indicesRollover(params, options) {
-	const elasticSearchConfig = options.pluginConfig.elastic;
-
-	if (typeof elasticSearchConfig.node === 'undefined' && typeof elasticSearchConfig.nodes === 'undefined') {
-		options.logger.error('Elasticsearch configuration is invalid: nodes or node is missing.');
-		throw new Error('Elasticsearch configuration is invalid: nodes or node is missing.');
-	}
-	if (!params.alias) {
+	var { alias, wildcardAlias } = params;
+	var { logger } = options;
+	var client = new ElasticsearchClient().client;
+	debugger;
+	if (!alias) {
 		throw new Error('Missing required parameter: alias');
 	}
+	var aliasesToRollover = [];
 	// The alias might be a string or an object 
-	if(typeof params.alias === 'object') {
-		if(Object.keys(params.alias).length === 0 && params.alias.constructor === Object) {
-			throw new Error(`Given rollover alias object is empty.`);
-		}
-		for(var alias in params.alias) {
-			params.alias = alias;
+	if(typeof alias === 'object') {
+		for(var alias in alias) {
+			aliasesToRollover.push(alias);
 			break; // Only one alias is supported.
 		}
+	} else {
+		aliasesToRollover.push(alias);
+	}
+	// Is the given alias is a wildcard, search for other indicies starting with this alias
+	if(wildcardAlias) {
+		aliasesToRollover = [];
+		logger.debug(`Rolling over all indicies starting with alias: ${alias}`);
+		var indicesFound = await client.indices.getAlias({name: `${alias}*`}, { ignore: [404], maxRetries: 3 });
+		for (const index of Object.values(indicesFound.body)) {
+			var indexAlias = Object.keys(index.aliases)[0];
+			aliasesToRollover.push(indexAlias);
+		}	
+	}
+	if(aliasesToRollover.length == 0) {
+		throw new Error(`No index found to rollover for alias: ${JSON.stringify(alias)}`);
 	}
 	try {
-		var client = new ElasticsearchClient(elasticSearchConfig).client;
-		var result = await client.indices.rollover( params, { ignore: [404], maxRetries: 3 });
-
+		var result = [];
+		for(alias of aliasesToRollover) {
+			var rolloverResult = await client.indices.rollover( { alias: alias }, { ignore: [404], maxRetries: 3 });
+			result.push(rolloverResult);
+		}
 		return result;
 	} catch (e) {
 		if(e instanceof Error) throw e;

api-builder-plugin-fn-elasticsearch/src/flow-nodes.yml

@@ -929,12 +929,21 @@ flow-nodes:
             schema:
               default: false
               type: boolean
+          wildcardAlias:
+            name: Wildcard alias
+            description: Given alias is considered as a prefix. All indicies starting with the alias, will be rolled up.
+            required: false
+            group: Advanced
+            initialType: boolean
+            schema:
+              default: false
+              type: boolean
         returns:
           name: Next
           description: Acknowledge message about the rollover.
           context: $.result
           schema:
-            type: object
+            type: array
         throws:
           name: Error
           description: An unexpected error was encountered.

api-builder-plugin-fn-elasticsearch/test/indices/indices-Test.js

@@ -29,7 +29,7 @@ describe('Indices rollover tests', () => {
 			expect(output).to.equal('error');
 		});
 
-		it('should pass without with a valid alias name', async () => {
+		it('should pass with with a valid alias name', async () => {
 			const mockedFn = setupElasticsearchMock(client, 'indices.rollover', './test/mock/indices/rolloverResponse.json', false);
 
 			const inputParameter = { alias: 'apigw-traffic-summary' };
@@ -53,6 +53,21 @@ describe('Indices rollover tests', () => {
 			expect(mockedFn.lastCall.arg).to.deep.equals({ alias: 'apigw-traffic-summary' });
 		});
 
+		it('should pass if the alias is a wildcard - Should rollover all indicies', async () => {
+			const mockedIndicesGetAliasResponse = setupElasticsearchMock(client, 'indices.getAlias', './test/mock/indices/twoIndicesForAliasFoundResponse.json', false);
+			const mockedRolloverResponse = setupElasticsearchMock(client, 'indices.rollover', './test/mock/indices/rolloverResponse.json', false);
+			// 
+
+			const inputParameter = { alias: 'apigw-traffic-summary', wildcardAlias: true };
+			const { value, output } = await flowNode.indicesRollover(inputParameter);
+			debugger;
+			expect(output).to.equal('next');
+			expect(mockedIndicesGetAliasResponse.callCount).to.equals(1);
+			expect(mockedRolloverResponse.callCount).to.equals(2);
+			// Given object should be translated into a single alias string
+			expect(mockedRolloverResponse.lastCall.arg).to.deep.equals({ alias: 'apigw-traffic-summary-us-dc1' });
+		});
+
 		it('should gracefully handle, if the alias object is empty', async () => {
 			const mockedFn = setupElasticsearchMock(client, 'indices.rollover', './test/mock/indices/rolloverResponse.json', false);
 
@@ -62,7 +77,7 @@ describe('Indices rollover tests', () => {
 			expect(output).to.equal('error');
 			expect(mockedFn.callCount).to.equals(0);
 			expect(value).to.be.instanceOf(Error)
-				.and.to.have.property('message', 'Given rollover alias object is empty.');
+				.and.to.have.property('message', 'No index found to rollover for alias: {}');
 		});
 	});
 

api-builder-plugin-fn-elasticsearch/test/mock/indices/twoIndicesForAliasFoundResponse.json

@@ -0,0 +1,65 @@
+{
+    "body": {
+      "apigw-traffic-summary-000002": {
+        "aliases": {
+          "apigw-traffic-summary": {
+            
+          }
+        }
+      },
+      "apigw-traffic-summary-us-dc1-000001": {
+        "aliases": {
+          "apigw-traffic-summary-us-dc1": {
+            
+          }
+        }
+      }
+    },
+    "statusCode": 200,
+    "headers": {
+      "content-type": "application/json; charset=UTF-8",
+      "content-length": "159"
+    },
+    "meta": {
+      "context": null,
+      "request": {
+        "params": {
+          "method": "GET",
+          "path": "/_alias/apigw-traffic-summary*",
+          "body": null,
+          "querystring": "",
+          "headers": {
+            "user-agent": "elasticsearch-js/7.10.0 (win32 10.0.19041-x64; Node.js v12.13.1)"
+          },
+          "timeout": 30000
+        },
+        "options": {
+          "ignore": [
+            404
+          ],
+          "maxRetries": 3
+        },
+        "id": 1
+      },
+      "name": "elasticsearch-js",
+      "connection": {
+        "url": "http://api-env:9200/",
+        "id": "http://api-env:9200/",
+        "headers": {
+          
+        },
+        "deadCount": 0,
+        "resurrectTimeout": 0,
+        "_openRequests": 0,
+        "status": "alive",
+        "roles": {
+          "master": true,
+          "data": true,
+          "ingest": true,
+          "ml": false
+        }
+      },
+      "attempts": 0,
+      "aborted": false
+    }
+  }