MINIFICPP-2765 Move GCP Extension to stable C API

Author: martinzink

behave_framework/src/minifi_behave/steps/checking_steps.py

@@ -119,7 +119,7 @@ def verify_minifi_logs_match_regex(context, regex, duration):
 @step('no errors were generated on the http-proxy regarding "{url}"')
 def verify_no_errors_on_http_proxy(context: MinifiTestContext, url: str):
     http_proxy_container = next(container for container in context.containers.values() if isinstance(container, HttpProxy))
-    assert http_proxy_container.check_http_proxy_access(url) or http_proxy_container.log_app_output()
+    assert http_proxy_container.check_http_proxy_access(url) or log_due_to_failure(context)
 
 
 @then('in the "{container}" container no files are placed in the "{directory}" directory in {duration} of running time')

core-framework/common/include/core/PropertyDefinitionBuilder.h

@@ -27,7 +27,21 @@ namespace org::apache::nifi::minifi::core {
 namespace detail {
 template<typename... Types>
 inline constexpr auto TypeNames = std::array<std::string_view, sizeof...(Types)>{core::className<Types>()...};
-}
+
+template <size_t N>
+struct StringLiteral {
+  char value[N];
+  constexpr StringLiteral(const char (&str)[N]) {  // NOLINT(runtime/explicit)
+    for (size_t i = 0; i < N; ++i) {
+      value[i] = str[i];
+    }
+  }
+};
+
+// A variable template that creates permanent static memory for the span to point to
+template <StringLiteral str>
+inline constexpr auto StaticAllowedType = std::array<std::string_view, 1>{std::string_view{str.value, sizeof(str.value) - 1}};
+}  // namespace detail
 
 template<size_t NumAllowedValues = 0>
 struct PropertyDefinitionBuilder {
@@ -81,6 +95,12 @@ struct PropertyDefinitionBuilder {
     return *this;
   }
 
+  template <detail::StringLiteral TypeName>
+  constexpr PropertyDefinitionBuilder<NumAllowedValues> withAllowedType() {
+    property.allowed_types = detail::StaticAllowedType<TypeName>;
+    return *this;
+  }
+
   constexpr PropertyDefinitionBuilder<NumAllowedValues> withValidator(const PropertyValidator& property_validator) {
     property.validator = gsl::make_not_null(&property_validator);
     return *this;

extension-framework/cpp-extension-lib/include/api/core/ProcessContext.h

@@ -17,10 +17,11 @@
 
 #pragma once
 
-#include <string>
 #include <expected>
+#include <string>
 
 #include "api/core/FlowFile.h"
+#include "api/utils/Proxy.h"
 #include "api/utils/Ssl.h"
 #include "minifi-c.h"
 #include "minifi-cpp/core/PropertyDefinition.h"
@@ -44,7 +45,8 @@ class ProcessContext {
   [[nodiscard]] virtual bool hasNonEmptyProperty(std::string_view name) const = 0;
   [[nodiscard]] virtual std::map<std::string, std::string> getDynamicProperties(const FlowFile* flow_file) const = 0;
 
-  [[nodiscard]] virtual std::expected<utils::net::SslData, std::error_code> getSslData(std::string_view name) const = 0;
+  [[nodiscard]] virtual std::expected<std::optional<utils::net::SslData>, std::error_code> getSslData(const minifi::core::PropertyReference& prop) const = 0;
+  [[nodiscard]] virtual std::expected<std::optional<utils::ProxyData>, std::error_code> getProxyData(const minifi::core::PropertyReference& prop) const = 0;
 };
 
 class CffiProcessContext : public ProcessContext {
@@ -58,7 +60,8 @@ class CffiProcessContext : public ProcessContext {
   [[nodiscard]] std::map<std::string, std::string> getDynamicProperties(const FlowFile* flow_file) const override;
   [[nodiscard]] bool hasNonEmptyProperty(std::string_view name) const override;
 
-  [[nodiscard]] std::expected<utils::net::SslData, std::error_code> getSslData(std::string_view name) const override;
+  [[nodiscard]] std::expected<std::optional<utils::net::SslData>, std::error_code> getSslData(const minifi::core::PropertyReference& prop) const override;
+  [[nodiscard]] std::expected<std::optional<utils::ProxyData>, std::error_code> getProxyData(const minifi::core::PropertyReference& prop) const override;
 
  private:
   [[nodiscard]] std::expected<std::string, std::error_code> getProperty(std::string_view name, const FlowFile* flow_file) const;

extension-framework/cpp-extension-lib/include/api/utils/Proxy.h

@@ -0,0 +1,42 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#pragma once
+
+#include <filesystem>
+#include <optional>
+#include <string>
+
+namespace org::apache::nifi::minifi::api::utils {
+
+enum class ProxyType {
+  DIRECT,
+  HTTP
+};
+
+struct BasicAuthCredentials {
+  std::string username;
+  std::string password;
+};
+
+struct ProxyData {
+  std::string host;
+  uint16_t port;
+  std::optional<BasicAuthCredentials> proxy_credentials;
+  ProxyType proxy_type;
+};
+
+}  // namespace org::apache::nifi::minifi::api::utils

extension-framework/cpp-extension-lib/include/api/utils/Ssl.h

@@ -17,12 +17,8 @@
 #pragma once
 
 #include <string>
-#include <memory>
-#include <optional>
 #include <filesystem>
 
-#include "utils/Enum.h"
-
 namespace org::apache::nifi::minifi::api::utils::net {
 
 enum class ClientAuthOption {

extension-framework/cpp-extension-lib/mocklib/include/MockProcessContext.h

@@ -34,7 +34,8 @@ class MockProcessContext : public api::core::ProcessContext {
   [[nodiscard]] std::map<std::string, std::string> getDynamicProperties(const api::core::FlowFile* flow_file) const override;
   [[nodiscard]] bool hasNonEmptyProperty(std::string_view name) const override;
 
-  [[nodiscard]] std::expected<api::utils::net::SslData, std::error_code> getSslData(std::string_view name) const override;
+  [[nodiscard]] std::expected<std::optional<api::utils::net::SslData>, std::error_code> getSslData(const minifi::core::PropertyReference& prop) const override;
+  [[nodiscard]] std::expected<std::optional<api::utils::ProxyData>, std::error_code> getProxyData(const minifi::core::PropertyReference& prop) const override;
 
   std::map<std::string, std::string, std::less<>> properties_;
 

extension-framework/cpp-extension-lib/mocklib/src/MockProcessContext.cpp

@@ -44,7 +44,11 @@ bool MockProcessContext::hasNonEmptyProperty(const std::string_view name) const
   return properties_.contains(name);
 }
 
-std::expected<api::utils::net::SslData, std::error_code> MockProcessContext::getSslData(std::string_view) const {
-  return api::utils::net::SslData{};
+std::expected<std::optional<api::utils::net::SslData>, std::error_code> MockProcessContext::getSslData(const minifi::core::PropertyReference&) const {
+  return std::nullopt;
+}
+
+[[nodiscard]] std::expected<std::optional<api::utils::ProxyData>, std::error_code> MockProcessContext::getProxyData(const minifi::core::PropertyReference&) const {
+  return std::nullopt;
 }
 }  // namespace org::apache::nifi::minifi::mock

extension-framework/cpp-extension-lib/src/core/ProcessContext.cpp

@@ -69,10 +69,13 @@ std::map<std::string, std::string> CffiProcessContext::getDynamicProperties(cons
   return result;
 }
 
-std::expected<utils::net::SslData, std::error_code> CffiProcessContext::getSslData(const std::string_view name) const {
+std::expected<std::optional<utils::net::SslData>, std::error_code> CffiProcessContext::getSslData(const minifi::core::PropertyReference& prop) const {
+  const auto controller_name = getProperty(prop, nullptr);
+  if (!controller_name) { return std::nullopt; }
+
   auto ssl_data = utils::net::SslData{};
 
-  if (const auto status = MinifiProcessContextGetSslData(impl_, utils::minifiStringView(name), [](void* data, const MinifiSslData* minifi_ssl_data) {
+  if (const auto status = MinifiProcessContextGetSslData(impl_, utils::minifiStringView(*controller_name), [](void* data, const MinifiSslData* minifi_ssl_data) {
       auto* my_ssl_data = static_cast<utils::net::SslData*>(data);
       my_ssl_data->ca_loc = utils::toString(minifi_ssl_data->ca_certificate_file);
       my_ssl_data->cert_loc = utils::toString(minifi_ssl_data->certificate_file);
@@ -86,4 +89,36 @@ std::expected<utils::net::SslData, std::error_code> CffiProcessContext::getSslDa
   return ssl_data;
 }
 
+std::expected<std::optional<utils::ProxyData>, std::error_code> CffiProcessContext::getProxyData(const minifi::core::PropertyReference& prop) const {
+  const auto controller_name = getProperty(prop, nullptr);
+  if (!controller_name) { return std::nullopt; }
+
+  auto proxy_data = utils::ProxyData{};
+  if (const auto status = MinifiProcessContextGetProxyData(
+          impl_,
+          utils::minifiStringView(*controller_name),
+          [](void* data, const MinifiProxyData* minifi_proxy_data) {
+            auto* proxy = static_cast<utils::ProxyData*>(data);
+            proxy->host = utils::toString(minifi_proxy_data->hostname);
+            proxy->port = minifi_proxy_data->port;
+            if (minifi_proxy_data->password && minifi_proxy_data->username) {
+              proxy->proxy_credentials = utils::BasicAuthCredentials{.username = utils::toString(*minifi_proxy_data->username),
+                  .password = utils::toString(*minifi_proxy_data->password)};
+            } else {
+              proxy->proxy_credentials = std::nullopt;
+            }
+            if (minifi_proxy_data->proxy_type == MINIFI_PROXY_TYPE_HTTP) {
+              proxy->proxy_type = utils::ProxyType::HTTP;
+            } else {
+              proxy->proxy_type = utils::ProxyType::DIRECT;
+            }
+          },
+          &proxy_data);
+      status != MINIFI_STATUS_SUCCESS) {
+    return std::unexpected{utils::make_error_code(status)};
+  }
+
+  return proxy_data;
+}
+
 }  // namespace org::apache::nifi::minifi::api::core

extensions/gcp/CMakeLists.txt

@@ -30,8 +30,8 @@ add_minifi_library(minifi-gcp SHARED ${SOURCES})
 if (NOT WIN32)
     target_compile_options(minifi-gcp PRIVATE -Wno-error=deprecated-declarations)  # Suppress deprecation warnings for std::rel_ops usage
 endif()
-target_link_libraries(minifi-gcp ${LIBMINIFI} google-cloud-cpp::storage)
-target_include_directories(minifi-gcp SYSTEM PUBLIC ${google-cloud-cpp_INCLUDE_DIRS})
+target_link_libraries(minifi-gcp minifi-cpp-extension-lib google-cloud-cpp::storage)
 
-register_extension(minifi-gcp "GCP EXTENSIONS" GCP-EXTENSIONS "This enables Google Cloud Platform support" "extensions/gcp/tests")
+target_include_directories(minifi-gcp SYSTEM PUBLIC ${google-cloud-cpp_INCLUDE_DIRS})
 
+register_c_api_extension(minifi-gcp "GCP EXTENSIONS" GCP-EXTENSIONS "This enables Google Cloud Platform support" "extensions/gcp/tests")

extensions/gcp/ExtensionInitializer.cpp

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "../../extension-framework/cpp-extension-lib/include/api/core/Resource.h"
+#include "api/core/Resource.h"
+#include "api/utils/minifi-c-utils.h"
+#include "processors/DeleteGCSObject.h"
+#include "processors/FetchGCSObject.h"
+#include "processors/ListGCSBucket.h"
+#include "processors/PutGCSObject.h"
+
+#define MKSOC(x) #x
+#define MAKESTRING(x) MKSOC(x)  // NOLINT(cppcoreguidelines-macro-usage)
+
+namespace minifi = org::apache::nifi::minifi;
+
+CEXTENSIONAPI const uint32_t MinifiApiVersion = MINIFI_API_VERSION;
+
+CEXTENSIONAPI void MinifiInitExtension(MinifiExtensionContext* extension_context) {
+  const MinifiExtensionDefinition extension_definition{
+    .name = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_NAME)),
+    .version = minifi::api::utils::minifiStringView(MAKESTRING(EXTENSION_VERSION)),
+    .deinit = nullptr,
+    .user_data = nullptr
+  };
+  auto* extension = MinifiRegisterExtension(extension_context, &extension_definition);
+  minifi::api::core::registerProcessors<minifi::extensions::gcp::DeleteGCSObject,
+      minifi::extensions::gcp::FetchGCSObject,
+      minifi::extensions::gcp::ListGCSBucket,
+      minifi::extensions::gcp::PutGCSObject>(extension);
+  minifi::api::core::registerControllerServices<minifi::extensions::gcp::GCPCredentialsControllerService>(extension);
+}