Skip to content

Commit cd1d2df

Browse files
martinzinkmartinzink
committed
MINIFICPP-2749 Add EncryptContentPGP and DecryptContentPGP
1 parent 0dddb32 commit cd1d2df

53 files changed

Lines changed: 2743 additions & 2 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/references/ubuntu_22_04_clang_arm_manifest.json‎

Lines changed: 184 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1159,6 +1159,66 @@
11591159
"supportsDynamicProperties": "false",
11601160
"type": "minifi_rs_playground.processors.count_actual_logging.CountActualLogging"
11611161
},
1162+
{
1163+
"propertyDescriptors": {
1164+
"Decryption Strategy": {
1165+
"name": "Decryption Strategy",
1166+
"description": "Strategy for writing files to success after decryption",
1167+
"validator": "VALID",
1168+
"required": "true",
1169+
"sensitive": "false",
1170+
"expressionLanguageScope": "NONE",
1171+
"defaultValue": "DECRYPTED",
1172+
"allowableValues": [
1173+
{
1174+
"value": "DECRYPTED",
1175+
"displayName": "DECRYPTED"
1176+
},
1177+
{
1178+
"value": "PACKAGED",
1179+
"displayName": "PACKAGED"
1180+
}
1181+
]
1182+
},
1183+
"Private Key Service": {
1184+
"typeProvidedByValue": {
1185+
"type": "minifi_pgp.controller_services.private_key_service.PGPPrivateKeyService",
1186+
"group": "minifi_pgp.controller_services.private_key_service",
1187+
"artifact": "minifi_pgp"
1188+
},
1189+
"name": "Private Key Service",
1190+
"description": "PGP Private Key Service for decrypting data encrypted with Public Key Encryption",
1191+
"validator": "VALID",
1192+
"required": "false",
1193+
"sensitive": "false",
1194+
"expressionLanguageScope": "NONE"
1195+
},
1196+
"Symmetric Password": {
1197+
"name": "Symmetric Password",
1198+
"description": "Password used for decrypting data encrypted with Password-Based Encryption",
1199+
"validator": "VALID",
1200+
"required": "false",
1201+
"sensitive": "true",
1202+
"expressionLanguageScope": "NONE"
1203+
}
1204+
},
1205+
"inputRequirement": "INPUT_REQUIRED",
1206+
"isSingleThreaded": "false",
1207+
"supportedRelationships": [
1208+
{
1209+
"name": "failure",
1210+
"description": "Decryption Failed"
1211+
},
1212+
{
1213+
"name": "success",
1214+
"description": "Decryption Succeeded"
1215+
}
1216+
],
1217+
"typeDescription": "Decrypt contents of OpenPGP messages. Using the Packaged Decryption Strategy preserves OpenPGP encoding to support subsequent signature verification.",
1218+
"supportsDynamicRelationships": "false",
1219+
"supportsDynamicProperties": "false",
1220+
"type": "minifi_pgp.processors.decrypt_content.DecryptContentPGP"
1221+
},
11621222
{
11631223
"propertyDescriptors": {
11641224
"Max Buffer Age": {
@@ -1911,6 +1971,74 @@
19111971
"supportsDynamicProperties": "false",
19121972
"type": "minifi_rs_playground.processors.duplicate_text.DuplicateStreamText"
19131973
},
1974+
{
1975+
"propertyDescriptors": {
1976+
"File Encoding": {
1977+
"name": "File Encoding",
1978+
"description": "File Encoding for encryption",
1979+
"validator": "VALID",
1980+
"required": "true",
1981+
"sensitive": "false",
1982+
"expressionLanguageScope": "NONE",
1983+
"defaultValue": "BINARY",
1984+
"allowableValues": [
1985+
{
1986+
"value": "ASCII",
1987+
"displayName": "ASCII"
1988+
},
1989+
{
1990+
"value": "BINARY",
1991+
"displayName": "BINARY"
1992+
}
1993+
]
1994+
},
1995+
"Public Key Search": {
1996+
"name": "Public Key Search",
1997+
"description": "PGP Public Key Search will be used to match against the User ID or Key ID when formatted as uppercase hexadecimal string of 16 characters",
1998+
"validator": "VALID",
1999+
"required": "false",
2000+
"sensitive": "false",
2001+
"expressionLanguageScope": "FLOWFILE_ATTRIBUTES"
2002+
},
2003+
"Public Key Service": {
2004+
"typeProvidedByValue": {
2005+
"type": "minifi_pgp.controller_services.public_key_service.PGPPublicKeyService",
2006+
"group": "minifi_pgp.controller_services.public_key_service",
2007+
"artifact": "minifi_pgp"
2008+
},
2009+
"name": "Public Key Service",
2010+
"description": "PGP Public Key Service for encrypting data with Public Key Encryption",
2011+
"validator": "VALID",
2012+
"required": "false",
2013+
"sensitive": "false",
2014+
"expressionLanguageScope": "NONE"
2015+
},
2016+
"Symmetric Password": {
2017+
"name": "Symmetric Password",
2018+
"description": "Password used for encrypting data with Password-Based Encryption",
2019+
"validator": "VALID",
2020+
"required": "false",
2021+
"sensitive": "true",
2022+
"expressionLanguageScope": "NONE"
2023+
}
2024+
},
2025+
"inputRequirement": "INPUT_REQUIRED",
2026+
"isSingleThreaded": "false",
2027+
"supportedRelationships": [
2028+
{
2029+
"name": "failure",
2030+
"description": "Encryption Failed"
2031+
},
2032+
{
2033+
"name": "success",
2034+
"description": "Encryption Succeeded"
2035+
}
2036+
],
2037+
"typeDescription": "Encrypt contents using OpenPGP.",
2038+
"supportsDynamicRelationships": "false",
2039+
"supportsDynamicProperties": "false",
2040+
"type": "minifi_pgp.processors.encrypt_content.EncryptContentPGP"
2041+
},
19142042
{
19152043
"propertyDescriptors": {
19162044
"Destination": {
@@ -12046,6 +12174,62 @@
1204612174
"supportsDynamicProperties": "false",
1204712175
"type": "org.apache.nifi.minifi.controllers.PersistentMapStateStorage"
1204812176
},
12177+
{
12178+
"propertyDescriptors": {
12179+
"Key": {
12180+
"name": "Key",
12181+
"description": "Secret Key encoded in ASCII Armor",
12182+
"validator": "VALID",
12183+
"required": "false",
12184+
"sensitive": "true",
12185+
"expressionLanguageScope": "NONE"
12186+
},
12187+
"Key File": {
12188+
"name": "Key File",
12189+
"description": "File path to PGP Secret Key encoded in binary or ASCII Armor",
12190+
"validator": "VALID",
12191+
"required": "false",
12192+
"sensitive": "false",
12193+
"expressionLanguageScope": "FLOWFILE_ATTRIBUTES"
12194+
},
12195+
"Key Passphrase": {
12196+
"name": "Key Passphrase",
12197+
"description": "Passphrase used for decrypting Private Keys",
12198+
"validator": "VALID",
12199+
"required": "false",
12200+
"sensitive": "true",
12201+
"expressionLanguageScope": "NONE"
12202+
}
12203+
},
12204+
"typeDescription": "PGP Private Key Service provides Private Keys loaded from files or properties",
12205+
"supportsDynamicRelationships": "false",
12206+
"supportsDynamicProperties": "false",
12207+
"type": "minifi_pgp.controller_services.private_key_service.PGPPrivateKeyService"
12208+
},
12209+
{
12210+
"propertyDescriptors": {
12211+
"Keyring": {
12212+
"name": "Keyring",
12213+
"description": "PGP Keyring or Secret Key encoded in ASCII Armor",
12214+
"validator": "VALID",
12215+
"required": "false",
12216+
"sensitive": "true",
12217+
"expressionLanguageScope": "NONE"
12218+
},
12219+
"Keyring File": {
12220+
"name": "Keyring File",
12221+
"description": "File path to PGP Keyring or Secret Key encoded in binary or ASCII Armor",
12222+
"validator": "VALID",
12223+
"required": "false",
12224+
"sensitive": "false",
12225+
"expressionLanguageScope": "FLOWFILE_ATTRIBUTES"
12226+
}
12227+
},
12228+
"typeDescription": "PGP Public Key Service providing Public Keys loaded from files",
12229+
"supportsDynamicRelationships": "false",
12230+
"supportsDynamicProperties": "false",
12231+
"type": "minifi_pgp.controller_services.public_key_service.PGPPublicKeyService"
12232+
},
1204912233
{
1205012234
"propertyDescriptors": {
1205112235
"Proxy Server Host": {

‎minifi_rust/extensions/minifi_pgp/.cargo/config.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
[target.aarch64-apple-darwin]
2+
rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]
3+
4+
[target.x86_64-apple-darwin]
5+
rustflags = ["-C", "link-arg=-undefined", "-C", "link-arg=dynamic_lookup"]

‎minifi_rust/extensions/minifi_pgp/.gitignore‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
target
2+
output
3+
features/.venv
4+
features/output
5+
integration_tests/features/.venv
6+
integration_tests/features/linux_so
7+
integration_tests/.venv

‎minifi_rust/extensions/minifi_pgp/Cargo.toml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[package]
2+
name = "minifi_pgp"
3+
version = "0.1.0"
4+
edition = "2024"
5+
6+
[lib]
7+
crate-type = ["cdylib"]
8+
9+
[dependencies]
10+
minifi_native = { path = "../../minifi_native" }
11+
strum_macros = "0.28.0"
12+
strum = "0.28.0"
13+
pgp = "0.20.0"
14+
rand = "0.8.6" # pgp 0.20.0 doesnt support >= 0.9 rand yet
15+

‎minifi_rust/extensions/minifi_pgp/features/encrypt_decrypt.feature‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
@SUPPORTS_WINDOWS
2+
Feature: Test PGP extension's encryption and decryption capabilities
3+
4+
Background: The pgp library is successfully built on linux
5+
6+
Scenario: The pgp library is loaded into minifi
7+
Given log property "logger.org::apache::nifi::minifi::core::extension::ExtensionManager" is set to "TRACE,stderr"
8+
And log property "logger.org::apache::nifi::minifi::core::ClassLoader" is set to "TRACE,stderr"
9+
10+
When the MiNiFi instance starts up
11+
12+
Then the Minifi logs contain the following message: "Registering class 'EncryptContentPGP' at '/minifi_pgp'" in less than 10 seconds
13+
And the Minifi logs contain the following message: "Registering class 'DecryptContentPGP' at '/minifi_pgp'" in less than 1 seconds
14+
And the Minifi logs contain the following message: "Registering class 'PGPPublicKeyService' at '/minifi_pgp'" in less than 1 seconds
15+
And the Minifi logs contain the following message: "Registering class 'PGPPrivateKeyService' at '/minifi_pgp'" in less than 1 seconds
16+
And the Minifi logs do not contain errors
17+
And the Minifi logs do not contain warnings
18+
19+
Scenario: Encrypted for Alice but not for Bob
20+
Given log property "logger.minifi_pgp::processors::decrypt_content::DecryptContentPGP" is set to "TRACE,stderr"
21+
And log property "logger.minifi_pgp::processors::encrypt_content::EncryptContentPGP" is set to "TRACE,stderr"
22+
23+
And a GetFile processor with the "Input Directory" property set to "/tmp/input"
24+
And an EncryptContentPGP processor with a PGPPublicKeyService is set up
25+
And a DecryptContentPGP processor named DecryptAlice with a PGPPrivateKeyService is set up for Alice
26+
And a DecryptContentPGP processor named DecryptBob with a PGPPrivateKeyService is set up for Bob
27+
And a PutFile processor with the name "AliceSuccess"
28+
And a PutFile processor with the name "BobFailure"
29+
30+
And these processor properties are set
31+
| processor name | property name | property value |
32+
| EncryptContentPGP | File Encoding | ASCII |
33+
| EncryptContentPGP | Public Key Search | Alice |
34+
| AliceSuccess | Directory | /tmp/output/alice_ok |
35+
| BobFailure | Directory | /tmp/output/bob_fail |
36+
37+
And the processors are connected up as described here
38+
| source name | relationship name | destination name |
39+
| GetFile | success | EncryptContentPGP |
40+
| EncryptContentPGP | success | DecryptAlice |
41+
| EncryptContentPGP | success | DecryptBob |
42+
| DecryptAlice | success | AliceSuccess |
43+
| DecryptBob | failure | BobFailure |
44+
45+
And AliceSuccess's success relationship is auto-terminated
46+
And BobFailure's success relationship is auto-terminated
47+
48+
And a directory at "/tmp/input" has a file "test_file.log" with the content "test content"
49+
50+
When the MiNiFi instance starts up
51+
52+
Then at least one file with the content "test content" is placed in the "/tmp/output/alice_ok" directory in less than 5 seconds
53+
And an encrypted armored pgp file is placed in the "/tmp/output/bob_fail" directory in less than 5 seconds
54+
And the Minifi logs do not contain errors

‎minifi_rust/extensions/minifi_pgp/features/environment.py‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
import os
2+
from typing import List
3+
4+
from minifi_behave.containers.docker_image_builder import DockerImageBuilder
5+
from minifi_behave.core.hooks import common_after_scenario
6+
from minifi_behave.core.hooks import common_before_scenario, get_minifi_container_image
7+
from minifi_behave.core.minifi_test_context import MinifiTestContext
8+
9+
10+
def add_extension_to_minifi_container(
11+
extension_name: str, possible_paths: List[str], context: MinifiTestContext
12+
):
13+
new_container_name = f"apacheminificpp:{extension_name}"
14+
is_windows = os.name == "nt"
15+
if is_windows:
16+
lib_filename = f"{extension_name}.dll"
17+
container_extension_dir = (
18+
"C:/Program Files/ApacheNiFiMiNiFi/nifi-minifi-cpp/extensions"
19+
)
20+
else:
21+
lib_filename = f"lib{extension_name}.so"
22+
container_extension_dir = "/opt/minifi/minifi-current/extensions/"
23+
24+
host_path = None
25+
for path in possible_paths:
26+
if os.path.exists(os.path.join(path, lib_filename)):
27+
host_path = os.path.join(path, lib_filename)
28+
break
29+
30+
assert host_path is not None, (
31+
f"Could not find {lib_filename} in {[p for p in possible_paths]}"
32+
)
33+
34+
with open(host_path, "rb") as f:
35+
lib_content = f.read()
36+
37+
base_img = get_minifi_container_image()
38+
39+
if is_windows:
40+
dockerfile = f"""
41+
FROM {base_img}
42+
COPY ["{lib_filename}", "{container_extension_dir}/{lib_filename}"]
43+
"""
44+
else:
45+
dockerfile = f"""
46+
FROM {base_img}
47+
COPY --chown=minificpp:minificpp {lib_filename} {container_extension_dir}
48+
RUN chmod 755 {container_extension_dir}{lib_filename}
49+
"""
50+
51+
builder = DockerImageBuilder(
52+
image_tag=new_container_name,
53+
dockerfile_content=dockerfile,
54+
files_on_context={lib_filename: lib_content},
55+
)
56+
57+
builder.build()
58+
return new_container_name
59+
60+
61+
def before_all(context):
62+
dir_path = os.path.dirname(os.path.realpath(__file__))
63+
build_path = os.path.normpath(os.path.join(dir_path, "../../../target/release/"))
64+
add_extension_to_minifi_container("minifi_pgp", [build_path], context)
65+
66+
67+
def before_scenario(context, scenario):
68+
context.minifi_container_image = "apacheminificpp:minifi_pgp"
69+
common_before_scenario(context, scenario)
70+
71+
72+
def after_scenario(context, scenario):
73+
common_after_scenario(context, scenario)

0 commit comments

Comments
 (0)