Arguments includes methods for retrieving
+ * the value of the token provided the name for the token, number of
+ * tokens, support for checking if the named argument exists, type of
+ * token as specified by TokenType and helper method for
+ * constructing JsonElement object.
+ *
+ * @see io.cdap.wrangler.api.parser.UsageDefinition
+ */
+public interface Arguments {
+ /**
+ * This method returns the token {@code value} based on the {@code name}
+ * specified in the argument. This method will attempt to convert the token
+ * into the expected return type T.
+ *
+ * If the name doesn't exist in this object, then this
+ * method is expected to return null
Token.
+ */
+ The optional arguments specified during the UsageDefinition
+ * are not included in the size if they are not present in the tokens parsed.
name registered
+ * with this object.
+ *
+ * The name is expected to the same as specified in the UsageDefinition.
+ * There are two reason why the name might not exists in this object :
+ *
+ * name.name exists, false otherwise.
+ */
+ boolean contains(String name);
+
+ /**
+ * Each token is defined as one of the types defined in the class {@link TokenType}.
+ * When the directive is parsed into token, the type of the token is passed through.
+ *
+ * @param name associated with the token.
+ * @return TokenType associated with argument name, else null.
+ */
+ TokenType type(String name);
+
+ /**
+ * Returns the source line number these arguments were parsed from.
+ *
+ * @return the source line number.
+ */
+ int line();
+
+ /**
+ * Returns the source column number these arguments were parsed from.
+ * It takes the start position of the directive as the column number.
+ * + * @return the start of the column number for the start of the directive + * these arguments contain. + */ + int column(); + + /** + * This method returns the original source line of the directive as specified + * the user. It returns theString representation of the directive.
+ *
+ * @return String object representing the original directive
+ * as specified by the user.
+ */
+ String source();
+
+ /**
+ * Returns JsonElement representation of this object.
+ *
+ * @return an instance of JsonElementobject representing all the
+ * named tokens held within this object.
+ */
+ JsonElement toJson();
+}
diff --git a/src/main/java/io/cdap/wrangler/api/CompileException.java b/src/main/java/io/cdap/wrangler/api/CompileException.java
new file mode 100644
index 0000000000..e9e1288249
--- /dev/null
+++ b/src/main/java/io/cdap/wrangler/api/CompileException.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright © 2017-2019 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.wrangler.api;
+
+import io.cdap.wrangler.api.annotations.Public;
+import io.cdap.wrangler.api.parser.SyntaxError;
+
+import java.util.Iterator;
+
+/**
+ * This class CompileException is thrown when there is
+ * any issue with compiling the Recipe. It's content
+ * include the exact line where the error occured and guess of what
+ * the problem is. Often times, the guess is close enough to point
+ * the problem, but, it's fair attempt to detect the exact issue.
+ *
+ * The SyntaxError object embedded within this exception
+ * contains the line number, character position, the raw content and
+ * the formatted content of the line.
+ */
+@Public
+public class CompileException extends Exception {
+ private IteratorCompileStatus contains the status of compilation.
+ * If there are errors - syntax or semnatic it records the details of the errors
+ * as iterator over SyntaxError. If the compilation is successfull,
+ * it contains the ExecutableObject.
+ */
+public final class CompileStatus {
+ private RecipeSymbol symbols = null;
+ private boolean hasError = false;
+ private IteratorCompiler interface provides a way to implement your
+ * own version of compiler for directive or recipe.
+ *
+ * This interface contains methods that provides variants of the source
+ * from which the recipe are read. It support reading from string, HDFS location
+ * and Path.
Each of the methods would return CompileStatus objects that
+ * contains the compiled directives in the form of Executor or
+ * iterator of SyntaxError in case of failure to compile.
String format.
+ *
+ * @param recipe representing the String form of recipe.
+ * @return CompileStatus status of compilation.
+ */
+ CompileStatus compile(String recipe) throws CompileException;
+
+ /**
+ * Compiles the recipe that is supplied in a Location on HDFS.
+ *
+ * @param location Location to the recipe being compiled.
+ * @return CompileStatus status of compilation.
+ */
+ CompileStatus compile(Location location) throws CompileException;
+
+ /**
+ * Compiles the recipe that is supplied in a Path on Filesystem.
+ *
+ * @param path Path to the recipe being compiled.
+ * @return CompileStatus status of compilation.
+ */
+ CompileStatus compile(Path path) throws CompileException;
+}
diff --git a/src/main/java/io/cdap/wrangler/api/Directive.java b/src/main/java/io/cdap/wrangler/api/Directive.java
new file mode 100644
index 0000000000..4f332848aa
--- /dev/null
+++ b/src/main/java/io/cdap/wrangler/api/Directive.java
@@ -0,0 +1,129 @@
+/*
+ * Copyright © 2017-2019 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.wrangler.api;
+
+import io.cdap.wrangler.api.parser.UsageDefinition;
+
+import java.util.List;
+
+/**
+ * Directive is a user defined directive. DIE - Define, Initialize & Execute - Pattern
+ *
+ * Following is a simple example of how to use this interface.
+ *
+ * @Plugin(type = Directive.Type)
+ * @Name("text-reverse")
+ * @Description("Reverses a string value of a column.")
+ * public final class TextReverse implements Directive {
+ * private final ColumnName columnArgs;
+ *
+ * @Override
+ * public UsageDefinition define() {
+ * UsageDefinition.Builder builder = UsageDefinition.builder();
+ * builder.define("col", TokenType.COLUMN_NAME)
+ * return builder.build();
+ * }
+ *
+ * @Override
+ * public void initialize(Argument args) throws DirectiveParseException {
+ * this.columnArgs = args.value("col");
+ * }
+ *
+ * @Override
+ * public List execute(List rows, ExecutorContext context)
+ * throws DirectiveExecutionException, ErrorRowException {
+ * ...
+ * }
+ * }
+ *
+ */
+public interface Directive extends ExecutorThis constant should be used when defining a plugin. An example + * of such usage is as follow:
+ * + *
+ * @Plugin(type = Directive.Type)
+ * @Name("text-reverse")
+ * @Description("Reverses the value of the column.)
+ * public final class TextReverse implements Directive {
+ * ...
+ * }
+ *
+ */
+ String TYPE = "directive";
+
+ /**
+ * This method provides a way for the developer to provide information
+ * about the arguments expected by this directive. The definition of
+ * arguments would provide information to the framework about how each
+ * argument should be parsed and interpretted.
+ *
+ * This method uses {@code UsageDefinition#Builder} to build the token + * definitions. Each token definition consists of {@code name}, {@code TokenType} + * and {@code optional} field that specifies whether the token specified is + * optionally present.
+ * + *The {@code UsageDefinition} provides different methods to {@code define}, + * and as well generate {@code usage} based on the definition.
+ * + *This method is invoked by the framework at the time of creating an executable + * directive that will be added to the {@link RecipePipeline}. It's generally during + * the configuration phase.
. + * + *NOTE: As best practice, developer needs to make sure that this class doesn't + * throw an exception. Also, it should not include external libraries that can + * generate exception unknown to the developer.
+ * + *
+ * Following is an example of how {@code define} could be used.
+ *
+ * public UsageDefinition define() {
+ * UsageDefinition.Builder builder = UsageDefinition.builder();
+ * builder.define("column", TokeType.COLUMN_NAME); // :column
+ * builder.define("number", TokenType.NUMERIC, Optional.TRUE); // 1.0 or 8
+ * builder.define("text", TokenType.TEXT); // 'text'
+ * builder.define("boolean", TokenType.BOOL); // true / false
+ * builder.define("expression", TokenType.EXPRESSOION); // exp: { age < 10.0 }
+ * }
+ *
+ *
This method is invoked only once during the initialization phase of the {@code Executor} + * object. The arguments are constructed based on the definition as provided by the user in + * the method above {@code define}.
+ * + *
+ * Following is an example of how {@code initialize} could be used to accept the
+ * arguments that are tokenized and parsed by the framework.
+ *
+ * public void initialize(Arguments args) throws DirectiveParseException {
+ * ColumnName column = args.value("column");
+ * if (args.contains("number") {
+ * Numeric number = args.value("number");
+ * }
+ * Text text = args.value("text");
+ * Bool bool = args.value("boolean");
+ * Expression expression = args.value("expression");
+ * }
+ *
+ *
Transform#destroy() when the directive is invoked
+ * within a plugin or when during Service#destroy() when invoked in the
+ * service.
+ *
+ * This method is specifically designed not to thrown any exceptions. So, if the
+ * the user code is throws any exception, the system will be unable to react or
+ * correct at this phase of invocation.
+ */
+ void destroy();
+
+ /**
+ * This method is used to get the updated schema of the data after the directive's transformation has been applied.
+ *
+ * @param schemaResolutionContext context containing necessary information for getting output schema
+ * @return output {@link Schema} of the transformed data
+ * @implNote By default, returns a null and the schema is inferred from the data when necessary.
+ * For consistent handling, override for directives that perform column renames, + * column data type changes or column additions with specific schemas.
+ */ + @Nullable + default Schema getOutputSchema(SchemaResolutionContext schemaResolutionContext) { + // no op + return null; + } +} diff --git a/src/main/java/io/cdap/wrangler/api/ExecutorContext.java b/src/main/java/io/cdap/wrangler/api/ExecutorContext.java new file mode 100644 index 0000000000..fe73d96588 --- /dev/null +++ b/src/main/java/io/cdap/wrangler/api/ExecutorContext.java @@ -0,0 +1,84 @@ +/* + * Copyright © 2017-2019 Cask Data, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not + * use this file except in compliance with the License. You may obtain a copy of + * the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +package io.cdap.wrangler.api; + +import io.cdap.cdap.etl.api.LookupProvider; +import io.cdap.cdap.etl.api.StageMetrics; +import io.cdap.wrangler.api.annotations.PublicEvolving; + +import java.io.Serializable; +import java.net.URL; +import java.util.Map; + +/** + * Pipeline Context for passing contextual information to the pipeline being executed. + */ +@PublicEvolving +public interface ExecutorContext extends LookupProvider, Serializable { + /** + * Specifies the environment in which wrangler is running. + */ + enum Environment { + SERVICE, + TRANSFORM, + MICROSERVICE, + TESTING + }; + + /** + * @return Environment this context is prepared for. + */ + Environment getEnvironment(); + + /** + * @return namespace that the program is being executed in + */ + String getNamespace(); + + /** + * @return Measurements handler. + */ + StageMetrics getMetrics(); + + /** + * @return Context name. + */ + String getContextName(); + + /** + * @return Properties associated with run and pipeline. + */ + Mapint.
+ * This may involve rounding or truncation.
+ *
+ * @return the numeric value represented by this object after conversion
+ * to type int.
+ */
+ @Override
+ public int intValue() {
+ try {
+ return Integer.parseInt(value);
+ } catch (NumberFormatException e) {
+ try {
+ return (int) Long.parseLong(value);
+ } catch (NumberFormatException nfe) {
+ return new BigDecimal(value).intValue();
+ }
+ }
+ }
+
+ /**
+ * Returns the value of the specified number as a long.
+ * This may involve rounding or truncation.
+ *
+ * @return the numeric value represented by this object after conversion
+ * to type long.
+ */
+ @Override
+ public long longValue() {
+ try {
+ return Long.parseLong(value);
+ } catch (NumberFormatException e) {
+ return new BigDecimal(value).longValue();
+ }
+ }
+
+ /**
+ * Returns the value of the specified number as a float.
+ * This may involve rounding.
+ *
+ * @return the numeric value represented by this object after conversion
+ * to type float.
+ */
+ @Override
+ public float floatValue() {
+ return Float.parseFloat(value);
+ }
+
+ /**
+ * Returns the value of the specified number as a double.
+ * This may involve rounding.
+ *
+ * @return the numeric value represented by this object after conversion
+ * to type double.
+ */
+ @Override
+ public double doubleValue() {
+ return Double.parseDouble(value);
+ }
+
+ @Override
+ public String toString() {
+ return value;
+ }
+}
diff --git a/src/main/java/io/cdap/wrangler/api/Optional.java b/src/main/java/io/cdap/wrangler/api/Optional.java
new file mode 100644
index 0000000000..11b9e80e34
--- /dev/null
+++ b/src/main/java/io/cdap/wrangler/api/Optional.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright © 2017-2019 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.wrangler.api;
+
+import io.cdap.wrangler.api.annotations.Public;
+
+/**
+ * This class Optional is a helper class used in specifying
+ * whether a argument for the directive is optional or not.
+ *
+ * This class is used when you are defining the usage for a directive + * argument. Following is an example :
+ * + *
+ * UsageDefinition.Builder builder = UsageDefinition.builder(NAME);
+ * builder.define("regex", TokenType.TEXT, Optional.TRUE);
+ * return builder.build();
+ *
+ *
+ * By default, the option is FALSE
TRUE is specified.
+ */
+ public static final boolean TRUE = true;
+
+ /**
+ * When an argument is non optional, FALSE is specified.
+ * The default behavior is false.
+ */
+ public static final boolean FALSE = false;
+}
diff --git a/src/main/java/io/cdap/wrangler/api/Pair.java b/src/main/java/io/cdap/wrangler/api/Pair.java
new file mode 100644
index 0000000000..19b7cbf5e7
--- /dev/null
+++ b/src/main/java/io/cdap/wrangler/api/Pair.java
@@ -0,0 +1,49 @@
+/*
+ * Copyright © 2017-2019 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package io.cdap.wrangler.api;
+
+/**
+ * A pair consisting of two elements - first & second.
+ *
+ * This class provides immutable access to elements of the pair.
+ *
+ * @param RecipeSymbol stores information about all the
+ * TokenGroup ( TokenGroup represents a collection of tokens
+ * generated from parsing a single directive). The object also contains
+ * information about the directives (or plugins) that need to be loaded
+ * at the startup time.
+ *
+ * This class provides some useful methods for accessing the list of + * directives or plugins that need to be loaded, the token groups for + * all the directives tokenized and parsed.
+ * + *This class exposes a builder pattern for constructing the object.
+ * in the RecipeVisitor. The RecipeVisitor
+ * constructs RecipeSymbol using the RecipeSymbol.Builder
RecipePipeline.
+ */
+ private final SetIf there are no directives specified in the recipe, then there would + * be no plugins to be loaded.
+ * + * @return An empty set if there are not directives to be loaded dynamically, + * else the list of directives as specified in the recipe. + */ + public Set#pragma version 2.0;
+ *
+ * @return version of the grammar used in the recipe.
+ */
+ public String getVersion() {
+ return version;
+ }
+
+ /**
+ * Returns number of groups tokenized and parsed. The number returned will
+ * less than or equal to the number of directives specified in the recipe.
+ *
+ * Fewer than number of directives is because of the '#pragma' directives
+ * @return + */ + public int size() { + return tokens.size(); + } + + /** + * Returns an iterator to the list of token groups maintained by this object. + * + * @return iterator to the list of tokens maintained. + */ + public IteratortoJson returns the JsonElement object
+ * representation of this object.
+ *
+ * @return An instance of JsonElement representing this object.
+ */
+ public JsonElement toJson() {
+ JsonObject output = new JsonObject();
+ output.addProperty("class", this.getClass().getSimpleName());
+ output.addProperty("count", tokens.size());
+ JsonArray array = new JsonArray();
+ for (TokenGroup token : tokens) {
+ JsonArray darray = new JsonArray();
+ IteratorRecipeSymbol object. In order to create the
+ * this builder, one has to use the static method defined in
+ * RecipeSymbol.
+ *
+ * Following is an example of how this can be done.
+ *
+ *
+ * RecipeSymbol.Builder builder = RecipeSymbol.builder();
+ * builder.createTokenGroup(...);
+ * builder.addToken(...);
+ * builder.addVersion(...);
+ * builder.addLoadableDirective(...);
+ * RecipeSymbol compiled = builder.build();
+ *
+ */
+ public static final class Builder {
+ private final ListTokenGroup is created for each directive in
+ * the recipe. This method creates a new TokenGroup
+ * by passing the SourceInfo, which represents the
+ * information of the source parsed.
+ *
+ * @param info about the source directive being parsed.
+ */
+ public void createTokenGroup(SourceInfo info) {
+ if (group != null) {
+ groups.add(group);
+ }
+ this.group = new TokenGroup(info);
+ }
+
+ /**
+ * This method provides a way to add a Token to the TokenGroup.
+ *
+ * @param token to be added to the token group.
+ */
+ public void addToken(Token token) {
+ group.add(token);
+ }
+
+ /**
+ * Recipe can specify the version of the grammar. This method
+ * allows one to extract and add the version to the RecipeSymbol.
+ *
+ * @param version of the recipe grammar being used.
+ */
+ public void addVersion(String version) {
+ this.version = version;
+ }
+
+ /**
+ * A Recipe can specify the pragma instructions for loading the directives
+ * dynamically. This method allows adding the new directive to be loaded
+ * as it's parsing through the call graph.
+ *
+ * @param directive to be loaded dynamically.
+ */
+ public void addLoadableDirective(String directive) {
+ loadableDirectives.add(directive);
+ }
+
+ /**
+ * Returns a fully constructed and valid RecipeSymbol object.
+ *
+ * @return An instance of RecipeSymbol
+ */
+ public RecipeSymbol build() {
+ groups.add(group);
+ return new RecipeSymbol(version, loadableDirectives, this.groups);
+ }
+ }
+}
diff --git a/src/main/java/io/cdap/wrangler/api/RemoteDirectiveResponse.java b/src/main/java/io/cdap/wrangler/api/RemoteDirectiveResponse.java
new file mode 100644
index 0000000000..2c92ffc413
--- /dev/null
+++ b/src/main/java/io/cdap/wrangler/api/RemoteDirectiveResponse.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright © 2024 Cask Data, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not
+ * use this file except in compliance with the License. You may obtain a copy of
+ * the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package io.cdap.wrangler.api;
+
+import io.cdap.cdap.api.data.schema.Schema;
+
+import java.io.Serializable;
+import java.util.List;
+
+/**
+ * Response after executing directives remotely
+ * Please make sure all fields are registered with {@link io.cdap.wrangler.utils.KryoSerializer}
+ */
+public class RemoteDirectiveResponse implements Serializable {
+ private final List