MINIFICPP-2749 Add EncryptContentPGP and DecryptContentPGP

.github/references/ubuntu_22_04_clang_arm_manifest.json

@@ -1159,6 +1159,66 @@
                 "supportsDynamicProperties": "false",
                 "type": "minifi_rs_playground.processors.count_actual_logging.CountActualLogging"
             },
+            {
+                "propertyDescriptors": {
+                    "Decryption Strategy": {
+                        "name": "Decryption Strategy",
+                        "description": "Strategy for writing files to success after decryption",
+                        "validator": "VALID",
+                        "required": "true",
+                        "sensitive": "false",
+                        "expressionLanguageScope": "NONE",
+                        "defaultValue": "DECRYPTED",
+                        "allowableValues": [
+                            {
+                                "value": "DECRYPTED",
+                                "displayName": "DECRYPTED"
+                            },
+                            {
+                                "value": "PACKAGED",
+                                "displayName": "PACKAGED"
+                            }
+                        ]
+                    },
+                    "Private Key Service": {
+                        "typeProvidedByValue": {
+                            "type": "minifi_pgp.controller_services.private_key_service.PGPPrivateKeyService",
+                            "group": "minifi_pgp.controller_services.private_key_service",
+                            "artifact": "minifi_pgp"
+                        },
+                        "name": "Private Key Service",
+                        "description": "PGP Private Key Service for decrypting data encrypted with Public Key Encryption",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "false",
+                        "expressionLanguageScope": "NONE"
+                    },
+                    "Symmetric Password": {
+                        "name": "Symmetric Password",
+                        "description": "Password used for decrypting data encrypted with Password-Based Encryption",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "true",
+                        "expressionLanguageScope": "NONE"
+                    }
+                },
+                "inputRequirement": "INPUT_REQUIRED",
+                "isSingleThreaded": "false",
+                "supportedRelationships": [
+                    {
+                        "name": "failure",
+                        "description": "Decryption Failed"
+                    },
+                    {
+                        "name": "success",
+                        "description": "Decryption Succeeded"
+                    }
+                ],
+                "typeDescription": "Decrypt contents of OpenPGP messages. Using the Packaged Decryption Strategy preserves OpenPGP encoding to support subsequent signature verification.",
+                "supportsDynamicRelationships": "false",
+                "supportsDynamicProperties": "false",
+                "type": "minifi_pgp.processors.decrypt_content.DecryptContentPGP"
+            },
             {
                 "propertyDescriptors": {
                     "Max Buffer Age": {
@@ -1911,6 +1971,74 @@
                 "supportsDynamicProperties": "false",
                 "type": "minifi_rs_playground.processors.duplicate_text.DuplicateStreamText"
             },
+            {
+                "propertyDescriptors": {
+                    "File Encoding": {
+                        "name": "File Encoding",
+                        "description": "File Encoding for encryption",
+                        "validator": "VALID",
+                        "required": "true",
+                        "sensitive": "false",
+                        "expressionLanguageScope": "NONE",
+                        "defaultValue": "BINARY",
+                        "allowableValues": [
+                            {
+                                "value": "ASCII",
+                                "displayName": "ASCII"
+                            },
+                            {
+                                "value": "BINARY",
+                                "displayName": "BINARY"
+                            }
+                        ]
+                    },
+                    "Public Key Search": {
+                        "name": "Public Key Search",
+                        "description": "PGP Public Key Search will be used to match against the User ID or Key ID when formatted as uppercase hexadecimal string of 16 characters",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "false",
+                        "expressionLanguageScope": "FLOWFILE_ATTRIBUTES"
+                    },
+                    "Public Key Service": {
+                        "typeProvidedByValue": {
+                            "type": "minifi_pgp.controller_services.public_key_service.PGPPublicKeyService",
+                            "group": "minifi_pgp.controller_services.public_key_service",
+                            "artifact": "minifi_pgp"
+                        },
+                        "name": "Public Key Service",
+                        "description": "PGP Public Key Service for encrypting data with Public Key Encryption",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "false",
+                        "expressionLanguageScope": "NONE"
+                    },
+                    "Symmetric Password": {
+                        "name": "Symmetric Password",
+                        "description": "Password used for encrypting data with Password-Based Encryption",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "true",
+                        "expressionLanguageScope": "NONE"
+                    }
+                },
+                "inputRequirement": "INPUT_REQUIRED",
+                "isSingleThreaded": "false",
+                "supportedRelationships": [
+                    {
+                        "name": "failure",
+                        "description": "Encryption Failed"
+                    },
+                    {
+                        "name": "success",
+                        "description": "Encryption Succeeded"
+                    }
+                ],
+                "typeDescription": "Encrypt contents using OpenPGP.",
+                "supportsDynamicRelationships": "false",
+                "supportsDynamicProperties": "false",
+                "type": "minifi_pgp.processors.encrypt_content.EncryptContentPGP"
+            },
             {
                 "propertyDescriptors": {
                     "Destination": {
@@ -12046,6 +12174,62 @@
                 "supportsDynamicProperties": "false",
                 "type": "org.apache.nifi.minifi.controllers.PersistentMapStateStorage"
             },
+            {
+                "propertyDescriptors": {
+                    "Key": {
+                        "name": "Key",
+                        "description": "Secret Key encoded in ASCII Armor",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "true",
+                        "expressionLanguageScope": "NONE"
+                    },
+                    "Key File": {
+                        "name": "Key File",
+                        "description": "File path to PGP Secret Key encoded in binary or ASCII Armor",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "false",
+                        "expressionLanguageScope": "FLOWFILE_ATTRIBUTES"
+                    },
+                    "Key Passphrase": {
+                        "name": "Key Passphrase",
+                        "description": "Passphrase used for decrypting Private Keys",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "true",
+                        "expressionLanguageScope": "NONE"
+                    }
+                },
+                "typeDescription": "PGP Private Key Service provides Private Keys loaded from files or properties",
+                "supportsDynamicRelationships": "false",
+                "supportsDynamicProperties": "false",
+                "type": "minifi_pgp.controller_services.private_key_service.PGPPrivateKeyService"
+            },
+            {
+                "propertyDescriptors": {
+                    "Keyring": {
+                        "name": "Keyring",
+                        "description": "PGP Keyring or Secret Key encoded in ASCII Armor",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "true",
+                        "expressionLanguageScope": "NONE"
+                    },
+                    "Keyring File": {
+                        "name": "Keyring File",
+                        "description": "File path to PGP Keyring or Secret Key encoded in binary or ASCII Armor",
+                        "validator": "VALID",
+                        "required": "false",
+                        "sensitive": "false",
+                        "expressionLanguageScope": "FLOWFILE_ATTRIBUTES"
+                    }
+                },
+                "typeDescription": "PGP Public Key Service providing Public Keys loaded from files",
+                "supportsDynamicRelationships": "false",
+                "supportsDynamicProperties": "false",
+                "type": "minifi_pgp.controller_services.public_key_service.PGPPublicKeyService"
+            },
             {
                 "propertyDescriptors": {
                     "Proxy Server Host": {

minifi_rust/extensions/minifi_pgp/.cargo/config.toml

@@ -0,0 +1,5 @@
+[target.aarch64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
+
+[target.x86_64-apple-darwin]
+rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]

minifi_rust/extensions/minifi_pgp/.gitignore

@@ -0,0 +1,7 @@
+target
+output
+features/.venv
+features/output
+integration_tests/features/.venv
+integration_tests/features/linux_so
+integration_tests/.venv

minifi_rust/extensions/minifi_pgp/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "minifi_pgp"
+version = "0.1.0"
+edition = "2024"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+minifi_native = { path = "../../minifi_native" }
+strum_macros = "0.28.0"
+strum = "0.28.0"
+pgp = "0.20.0"
+rand = "0.8.6" # pgp 0.20.0 doesnt support >= 0.9 rand yet
+

minifi_rust/extensions/minifi_pgp/features/encrypt_decrypt.feature

@@ -0,0 +1,54 @@
+@SUPPORTS_WINDOWS
+Feature: Test PGP extension's encryption and decryption capabilities
+
+  Background: The pgp library is successfully built on linux
+
+  Scenario: The pgp library is loaded into minifi
+    Given log property "logger.org::apache::nifi::minifi::core::extension::ExtensionManager" is set to "TRACE,stderr"
+    And log property "logger.org::apache::nifi::minifi::core::ClassLoader" is set to "TRACE,stderr"
+
+    When the MiNiFi instance starts up
+
+    Then the Minifi logs contain the following message: "Registering class 'EncryptContentPGP' at '/minifi_pgp'" in less than 10 seconds
+    And the Minifi logs contain the following message: "Registering class 'DecryptContentPGP' at '/minifi_pgp'" in less than 1 seconds
+    And the Minifi logs contain the following message: "Registering class 'PGPPublicKeyService' at '/minifi_pgp'" in less than 1 seconds
+    And the Minifi logs contain the following message: "Registering class 'PGPPrivateKeyService' at '/minifi_pgp'" in less than 1 seconds
+    And the Minifi logs do not contain errors
+    And the Minifi logs do not contain warnings
+
+  Scenario: Encrypted for Alice but not for Bob
+    Given log property "logger.minifi_pgp::processors::decrypt_content::DecryptContentPGP" is set to "TRACE,stderr"
+    And log property "logger.minifi_pgp::processors::encrypt_content::EncryptContentPGP" is set to "TRACE,stderr"
+
+    And a GetFile processor with the "Input Directory" property set to "/tmp/input"
+    And an EncryptContentPGP processor with a PGPPublicKeyService is set up
+    And a DecryptContentPGP processor named DecryptAlice with a PGPPrivateKeyService is set up for Alice
+    And a DecryptContentPGP processor named DecryptBob with a PGPPrivateKeyService is set up for Bob
+    And a PutFile processor with the name "AliceSuccess"
+    And a PutFile processor with the name "BobFailure"
+
+    And these processor properties are set
+      | processor name    | property name     | property value       |
+      | EncryptContentPGP | File Encoding     | ASCII                |
+      | EncryptContentPGP | Public Key Search | Alice                |
+      | AliceSuccess      | Directory         | /tmp/output/alice_ok |
+      | BobFailure        | Directory         | /tmp/output/bob_fail |
+
+    And the processors are connected up as described here
+      | source name       | relationship name | destination name  |
+      | GetFile           | success           | EncryptContentPGP |
+      | EncryptContentPGP | success           | DecryptAlice      |
+      | EncryptContentPGP | success           | DecryptBob        |
+      | DecryptAlice      | success           | AliceSuccess      |
+      | DecryptBob        | failure           | BobFailure        |
+
+    And AliceSuccess's success relationship is auto-terminated
+    And BobFailure's success relationship is auto-terminated
+
+    And a directory at "/tmp/input" has a file "test_file.log" with the content "test content"
+
+    When the MiNiFi instance starts up
+
+    Then at least one file with the content "test content" is placed in the "/tmp/output/alice_ok" directory in less than 5 seconds
+    And an encrypted armored pgp file is placed in the "/tmp/output/bob_fail" directory in less than 5 seconds
+    And the Minifi logs do not contain errors

minifi_rust/extensions/minifi_pgp/features/environment.py

@@ -0,0 +1,73 @@
+import os
+from typing import List
+
+from minifi_behave.containers.docker_image_builder import DockerImageBuilder
+from minifi_behave.core.hooks import common_after_scenario
+from minifi_behave.core.hooks import common_before_scenario, get_minifi_container_image
+from minifi_behave.core.minifi_test_context import MinifiTestContext
+
+
+def add_extension_to_minifi_container(
+    extension_name: str, possible_paths: List[str], context: MinifiTestContext
+):
+    new_container_name = f"apacheminificpp:{extension_name}"
+    is_windows = os.name == "nt"
+    if is_windows:
+        lib_filename = f"{extension_name}.dll"
+        container_extension_dir = (
+            "C:/Program Files/ApacheNiFiMiNiFi/nifi-minifi-cpp/extensions"
+        )
+    else:
+        lib_filename = f"lib{extension_name}.so"
+        container_extension_dir = "/opt/minifi/minifi-current/extensions/"
+
+    host_path = None
+    for path in possible_paths:
+        if os.path.exists(os.path.join(path, lib_filename)):
+            host_path = os.path.join(path, lib_filename)
+            break
+
+    assert host_path is not None, (
+        f"Could not find {lib_filename} in {[p for p in possible_paths]}"
+    )
+
+    with open(host_path, "rb") as f:
+        lib_content = f.read()
+
+    base_img = get_minifi_container_image()
+
+    if is_windows:
+        dockerfile = f"""
+FROM {base_img}
+COPY ["{lib_filename}", "{container_extension_dir}/{lib_filename}"]
+"""
+    else:
+        dockerfile = f"""
+FROM {base_img}
+COPY --chown=minificpp:minificpp {lib_filename} {container_extension_dir}
+RUN chmod 755 {container_extension_dir}{lib_filename}
+"""
+
+    builder = DockerImageBuilder(
+        image_tag=new_container_name,
+        dockerfile_content=dockerfile,
+        files_on_context={lib_filename: lib_content},
+    )
+
+    builder.build()
+    return new_container_name
+
+
+def before_all(context):
+    dir_path = os.path.dirname(os.path.realpath(__file__))
+    build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/"))
+    add_extension_to_minifi_container("minifi_pgp", [build_path], context)
+
+
+def before_scenario(context, scenario):
+    context.minifi_container_image = "apacheminificpp:minifi_pgp"
+    common_before_scenario(context, scenario)
+
+
+def after_scenario(context, scenario):
+    common_after_scenario(context, scenario)