v24.42.0 (#109)

* Moved counter for checking handler classes to outside the condition (#104)

* Add recursive checking when creating a destination directory (#105)

* 106 unexplained errors deleting gpg keystore after decryption (#107)

* Move exception print statement earlier to avoid confusing log messages. Check for directories before trying to delete them.

* Upate changelog

* Add way to enable host key validation for SSH/SFTP (#108)

* bump version v24.37.2 -> v24.42.0

Author: adammcdonagh

.vscode/launch.json

@@ -33,6 +33,16 @@
       "args": ["-t", "scp-basic", "-c", "test/cfg", "-v3"],
       "justMyCode": false
     },
+    {
+      "name": "Python: Transfer - SFTP Basic",
+      "type": "debugpy",
+      "request": "launch",
+      "preLaunchTask": "Build Test containers",
+      "program": "src/opentaskpy/cli/task_run.py",
+      "console": "integratedTerminal",
+      "args": ["-t", "sftp-basic", "-c", "test/cfg", "-v3"],
+      "justMyCode": false
+    },
     {
       "name": "Python: Transfer - Basic - As job",
       "type": "debugpy",

CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+# v24.42.0
+
+- Fix issue where different protocols where not being detected properly, and proxy had to be explicitly defined when it was unnecessary
+- When creating a destination directory with SFTP, it will now check whether lower level directories exist, and create them if not
+- Always check for a directory before trying to delete it and thowing an exception if it doesn't exist.
+- Moved exception printing for transfers to earlier in th code to ensure log messages aren't confusing.
+- Add ability to check for SSH host key and validate it before proceeding with connection
+
 # No release
 
 - Bump `black` to 24.10.0

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "opentaskpy"
-version = "v24.37.2"
+version = "v24.42.0"
 authors = [{ name = "Adam McDonagh", email = "adam@elitemonkey.net" }]
 license = { text = "GPLv3" }
 classifiers = [
@@ -71,7 +71,7 @@ otf-batch-validator = "opentaskpy.cli.batch_validator:main"
 profile = 'black'
 
 [tool.bumpver]
-current_version = "v24.37.2"
+current_version = "v24.42.0"
 version_pattern = "vYY.WW.PATCH[-TAG]"
 commit_message = "bump version {old_version} -> {new_version}"
 commit = true

src/opentaskpy/config/schemas/execution/ssh/protocol.json

@@ -10,6 +10,12 @@
     "port": {
       "type": "integer"
     },
+    "hostKeyValidation": {
+      "type": "boolean"
+    },
+    "knownHostsFile": {
+      "type": "string"
+    },
     "credentials": {
       "type": "object",
       "properties": {

src/opentaskpy/config/schemas/transfer/sftp_destination/protocol.json

@@ -10,6 +10,12 @@
     "port": {
       "type": "integer"
     },
+    "hostKeyValidation": {
+      "type": "boolean"
+    },
+    "knownHostsFile": {
+      "type": "string"
+    },
     "supportsPosixRename": {
       "type": "boolean",
       "default": true

src/opentaskpy/config/schemas/transfer/sftp_source/protocol.json

@@ -10,6 +10,12 @@
     "port": {
       "type": "integer"
     },
+    "hostKeyValidation": {
+      "type": "boolean"
+    },
+    "knownHostsFile": {
+      "type": "string"
+    },
     "supportsPosixRename": {
       "type": "boolean",
       "default": true

src/opentaskpy/config/schemas/transfer/ssh_destination/protocol.json

@@ -10,6 +10,12 @@
     "port": {
       "type": "integer"
     },
+    "hostKeyValidation": {
+      "type": "boolean"
+    },
+    "knownHostsFile": {
+      "type": "string"
+    },
     "credentials": {
       "type": "object",
       "properties": {

src/opentaskpy/config/schemas/transfer/ssh_source/protocol.json

@@ -10,6 +10,12 @@
     "port": {
       "type": "integer"
     },
+    "hostKeyValidation": {
+      "type": "boolean"
+    },
+    "knownHostsFile": {
+      "type": "string"
+    },
     "credentials": {
       "type": "object",
       "properties": {

src/opentaskpy/remotehandlers/sftp.py

@@ -11,12 +11,20 @@
 from io import StringIO
 from shlex import quote
 
-from paramiko import AutoAddPolicy, Channel, RSAKey, SFTPClient, SSHClient
-from tenacity import retry, stop_after_attempt, wait_exponential
+from paramiko import Channel, RSAKey, SFTPClient, SSHClient
+from tenacity import (
+    retry,
+    retry_if_exception,
+    retry_if_not_exception_message,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 import opentaskpy.otflogging
 from opentaskpy.remotehandlers.remotehandler import RemoteTransferHandler
 
+from .ssh_utils import setup_host_key_validation
+
 
 class SFTPTransfer(RemoteTransferHandler):
     """SFTP Transfer Handler."""
@@ -103,6 +111,12 @@ def connect(self, hostname: str) -> None:
         reraise=True,
         stop=stop_after_attempt(6),
         wait=wait_exponential(multiplier=2, min=5, max=60),
+        retry=(
+            retry_if_not_exception_message(
+                match=r".*(not found in known_hosts|Name or service not known).*"
+            )
+            & retry_if_exception(Exception)
+        ),
     )
     def connect_with_retry(self, client_kwargs: dict) -> SSHClient:
         """Connect to the remote host with retry.
@@ -118,7 +132,7 @@ def connect_with_retry(self, client_kwargs: dict) -> SSHClient:
             ssh_client.set_log_channel(
                 f"{__name__}.{ self.spec['task_id']}.paramiko.transport"
             )
-            ssh_client.set_missing_host_key_policy(AutoAddPolicy())
+            setup_host_key_validation(ssh_client, self.spec, self.logger)
             self.logger.info(f"Connecting to {client_kwargs['hostname']}")
 
             # Set additional timeout options to match the standard timeout
@@ -317,7 +331,20 @@ def push_files_from_worker(
                 f"[{self.spec['hostname']}] Creating destination directory:"
                 f" {destination_directory}"
             )
-            self.sftp_client.mkdir(destination_directory)
+
+            # We need to check recursively if the destination directory is nested
+            current_dir = ""
+            for dir_part in destination_directory.split("/"):
+                if not dir_part:
+                    continue
+                current_dir += f"/{dir_part}"
+                try:
+                    self.sftp_client.stat(current_dir)
+                except OSError:
+                    self.logger.info(
+                        f"[{self.spec['hostname']}] Destination directory {current_dir} does not exist. Creating it."
+                    )
+                    self.sftp_client.mkdir(current_dir)
 
         # Transfer the files
         result = 0

src/opentaskpy/remotehandlers/ssh.py

@@ -13,9 +13,15 @@
 from io import StringIO
 from shlex import quote
 
-from paramiko import AutoAddPolicy, RSAKey, SFTPClient, SSHClient, Transport
+from paramiko import RSAKey, SFTPClient, SSHClient, Transport
 from paramiko.channel import ChannelFile, ChannelStderrFile
-from tenacity import retry, stop_after_attempt, wait_exponential
+from tenacity import (
+    retry,
+    retry_if_exception,
+    retry_if_not_exception_message,
+    stop_after_attempt,
+    wait_exponential,
+)
 
 import opentaskpy.otflogging
 from opentaskpy.exceptions import SSHClientError
@@ -24,6 +30,8 @@
     RemoteTransferHandler,
 )
 
+from .ssh_utils import setup_host_key_validation
+
 SSH_OPTIONS: str = "-o StrictHostKeyChecking=no -o BatchMode=yes -o ConnectTimeout=5"
 REMOTE_SCRIPT_BASE_DIR: str = "/tmp"  # nosec B108
 
@@ -52,7 +60,7 @@ def __init__(self, spec: dict):
 
         client = SSHClient()
         client.set_log_channel(f"{__name__}.{ spec['task_id']}.paramiko.transport")
-        client.set_missing_host_key_policy(AutoAddPolicy())
+        setup_host_key_validation(client, spec, self.logger)
         self.ssh_client = client
 
         # Handle default values
@@ -136,6 +144,12 @@ def connect(self, hostname: str, ssh_client: SSHClient | None = None) -> None:
         reraise=True,
         stop=stop_after_attempt(6),
         wait=wait_exponential(multiplier=2, min=5, max=60),
+        retry=(
+            retry_if_not_exception_message(
+                match=r".*(not found in known_hosts|Name or service not known).*"
+            )
+            & retry_if_exception(Exception)
+        ),
     )
     def connect_with_retry(self, ssh_client: SSHClient, kwargs: dict) -> None:
         """Connect to the remote host with retry.
@@ -916,7 +930,8 @@ def __init__(self, remote_host: str, spec: dict):
 
         client = SSHClient()
         client.set_log_channel(f"{__name__}.{ spec['task_id']}.paramiko.transport")
-        client.set_missing_host_key_policy(AutoAddPolicy())
+
+        setup_host_key_validation(client, spec, self.logger)
 
         self.ssh_client = client