diff --git a/admin/admin.py b/admin/admin.py
index 61209688b84..905a1bb0891 100644
--- a/admin/admin.py
+++ b/admin/admin.py
@@ -30,7 +30,6 @@
from lib.cuckoo.common.admin_utils import (
CAPE_PATH,
- POSTPROCESS,
AutoAddPolicy,
bulk_deploy,
compare_hashed_files,
@@ -61,6 +60,7 @@
JUMPBOX_USED = False
jumpbox = False
+RETRY = 3
logging.getLogger("paramiko").setLevel(logging.WARNING)
logging.getLogger("paramiko.transport").setLevel(logging.WARNING)
@@ -226,6 +226,13 @@
required=False,
default=False,
)
+ compare_opt.add_argument(
+ "--remove-ssh-keys",
+ help="Remove servers ssh key from known keys on localhost",
+ action="store_true",
+ required=False,
+ default=False,
+ )
args = parser.parse_args()
@@ -235,12 +242,15 @@
logging.getLogger("paramiko.transport").setLevel(logging.DEBUG)
if args.username:
+ from lib.cuckoo.common import admin_utils
+ admin_utils.JUMP_BOX_USERNAME = args.username
JUMP_BOX_USERNAME = args.username
- # if args.debug:
- # log.setLevel(logging.DEBUG)
+ if args.debug:
+ logging.getLogger().setLevel(logging.DEBUG)
if args.jump_box_second and not args.dry_run:
+
ssh.connect(
JUMP_BOX_SECOND,
username=JUMP_BOX_SECOND_USERNAME,
@@ -286,7 +296,12 @@
print(parameters)
sys.exit(0)
queue.put([servers, file] + list(parameters))
- _ = deploy_file(queue, jumpbox)
+ for i in range(RETRY):
+ try:
+ _ = deploy_file(queue, jumpbox)
+ break
+ except Exception as eee:
+ print(f"Error {eee}, retry {i + 1}/{RETRY}")
elif args.delete_file:
queue = Queue()
@@ -342,7 +357,7 @@
sys.exit()
elif args.enum_all_servers:
- enumerate_files_on_all_servers()
+ enumerate_files_on_all_servers(servers, jumpbox, "/opt/CAPEv2", args.filename)
elif args.generate_files_listing and not args.enum_all_servers:
gen_hashfile(args.generate_files_listing, args.filename)
elif args.check_files_difference:
@@ -355,8 +370,12 @@
bulk_deploy(files, args.yara_category, args.dry_run, servers, jumpbox)
- if args.restart_service and POSTPROCESS:
- execute_command_on_all(POSTPROCESS, servers, jumpbox)
+ if args.restart_service:
+ execute_command_on_all("systemctl restart cape-processor; systemctl status cape-processor", servers, jumpbox)
if args.restart_uwsgi:
execute_command_on_all("touch /tmp/capeuwsgireload", servers, jumpbox)
+
+ if args.remove_ssh_keys:
+ for node in SERVERS_STATIC_LIST:
+ subprocess.run(["ssh-keygen", "-R", node])
diff --git a/lib/cuckoo/common/admin_utils.py b/lib/cuckoo/common/admin_utils.py
index 9808bf72e80..900d80856e7 100644
--- a/lib/cuckoo/common/admin_utils.py
+++ b/lib/cuckoo/common/admin_utils.py
@@ -2,6 +2,7 @@
import logging
import os
import re
+import shlex
# from glob import glob
import shutil
@@ -11,7 +12,7 @@
from pathlib import Path
from queue import Queue
from socket import if_nameindex
-from threading import Thread
+from threading import Lock, Thread
import urllib3
@@ -34,7 +35,6 @@
)
from scp import SCPClient, SCPException
-
conf = SSHConfig()
conf.parse(open(os.path.expanduser("~/.ssh/config")))
@@ -197,7 +197,7 @@ def compare_hashed_files(files: list, servers: list, ssh_proxy: SSHClient, priva
def enumerate_files_on_all_servers(servers: list, ssh_proxy: SSHClient, dir_folder: str, filename: str):
- cmd = f"python3 {CAPE_PATH}/admin/admin.py -gfl {dir_folder} -f /tmp/{filename} -s"
+ cmd = f"python3 {CAPE_PATH}/admin/admin.py -gfl {shlex.quote(dir_folder)} -f /tmp/{shlex.quote(filename)} -s"
execute_command_on_all(cmd, servers, ssh_proxy)
get_file(f"/tmp/{filename}.json", servers, ssh_proxy)
@@ -308,43 +308,52 @@ def file_recon(file, yara_category="CAPE"):
return False
# build command to be executed remotely
- REMOTE_COMMAND = f"chown {OWNER} {TARGET}; chmod 644 {TARGET};"
+ quoted_target = shlex.quote(TARGET)
+ REMOTE_COMMAND = f"chown {OWNER} {quoted_target}; chmod 644 {quoted_target};"
if filename.endswith(".py") and TARGET:
- REMOTE_COMMAND += "rm -f {0}.pyc; ls -la {0}.*".format(TARGET.replace(".py", ""))
+ REMOTE_COMMAND += "rm -f {0}.pyc; ls -la {0}.*".format(shlex.quote(TARGET.replace(".py", "")))
return TARGET, REMOTE_COMMAND, LOCAL_SHA256
# For session reuse
sockets = {}
+sockets_lock = Lock()
def _connect_via_jump_box(server: str, ssh_proxy: SSHClient):
session_checker()
- host = conf.lookup(server)
+ try:
+ host = conf.lookup(server)
+ except NameError:
+ log.error("Missed dependencies/activation of venv")
+ return None
+
+ with sockets_lock:
+ if server in sockets:
+ ssh = sockets[server]
+ if ssh.get_transport() and ssh.get_transport().is_active():
+ return ssh
+ else:
+ del sockets[server]
+
try:
"""
This is SSH pivoting it ssh to host Y via host X, can be used due to different networks
We doing direct-tcpip channel and pasing it as socket to be used
"""
if ssh_proxy and JUMP_BOX_USERNAME:
- if server not in sockets:
- ssh = SSHJumpClient(jump_session=ssh_proxy if ssh_proxy else None)
- ssh.set_missing_host_key_policy(AutoAddPolicy())
- # ssh_port = 22 if ":" not in server else int(server.split(":")[1])
- ssh.connect(
- server,
- username=JUMP_BOX_USERNAME,
- key_filename=host.get("identityfile"),
- banner_timeout=200,
- look_for_keys=False,
- allow_agent=True,
- # disabled_algorithms=dict(pubkeys=["rsa-sha2-512", "rsa-sha2-256"]),
- )
- sockets[server] = ssh
- else:
- # ToDo check if alive and reconnect
- ssh = sockets[server]
-
+ ssh = SSHJumpClient(jump_session=ssh_proxy if ssh_proxy else None)
+ ssh.set_missing_host_key_policy(AutoAddPolicy())
+ # ssh_port = 22 if ":" not in server else int(server.split(":")[1])
+ ssh.connect(
+ server,
+ username=JUMP_BOX_USERNAME,
+ key_filename=host.get("identityfile"),
+ banner_timeout=200,
+ look_for_keys=False,
+ allow_agent=True,
+ # disabled_algorithms=dict(pubkeys=["rsa-sha2-512", "rsa-sha2-256"]),
+ )
else:
ssh = SSHJumpClient()
ssh.load_system_host_keys()
@@ -357,35 +366,53 @@ def _connect_via_jump_box(server: str, ssh_proxy: SSHClient):
banner_timeout=200,
look_for_keys=False,
allow_agent=True,
- sock=ProxyCommand(host.get("proxycommand")),
+ sock=ProxyCommand(host.get("proxycommand")) if host.get("proxycommand") else None,
)
+
+ with sockets_lock:
+ sockets[server] = ssh
+
except (BadHostKeyException, AuthenticationException, PasswordRequiredException) as e:
- sys.exit(
- f"Connect error: {str(e)}. Also pay attention to this log for more details /var/log/auth.log and paramiko might need update.\nAlso ensure that you have added your public ssh key to /root/.ssh/authorized_keys"
+ log.error(
+ "Connect error to %s: %s. Also pay attention to this log for more details /var/log/auth.log and paramiko might need update.\nAlso ensure that you have added your public ssh key to /root/.ssh/authorized_keys", server, str(e)
)
+ return None
except ProxyCommandFailure as e:
- # Todo reconnect
- log.error("Can't connect to server: %s", str(e))
+ log.error("Can't connect to server %s: %s", server, str(e))
+ return None
+ except Exception as e:
+ log.error("Unexpected error connecting to %s: %s", server, str(e))
+ return None
+
return ssh
def execute_command_on_all(remote_command, servers: list, ssh_proxy: SSHClient):
for server in servers:
+ srv = server.split(".")[1] if "." in server else server
+ log.info("[*] Connecting to %s...", server)
try:
ssh = _connect_via_jump_box(server, ssh_proxy)
- _, ssh_stdout, _ = ssh.exec_command(remote_command)
+ if not ssh:
+ continue
+
+ _, ssh_stdout, ssh_stderr = ssh.exec_command(remote_command, get_pty=True)
ssh_out = ssh_stdout.read().decode("utf-8").strip()
+ ssh_err = ssh_stderr.read().decode("utf-8").strip()
+
if "Active: active (running)" in ssh_out and "systemctl status" not in remote_command:
- log.info("[+] Service %s", green("restarted successfully and is UP"))
- else:
- srv = str(server.split(".")[1])
- if ssh_out:
- log.info(green(f"[+] {srv} - {ssh_out}"))
- else:
- log.info(green(f"[+] {srv}"))
- ssh.close()
+ log.info("[+] %s - Service %s", srv, green("restarted successfully and is UP"))
+ elif ssh_out:
+ log.info(green("[+] %s - %s", srv, ssh_out ))
+
+ if ssh_err:
+ log.error(red("[-] %s ERROR - %s", srv, ssh_err))
+
+ if not ssh_out and not ssh_err:
+ log.info(green("[+] %s", srv))
+
except TimeoutError as e:
- sys.exit(f"Did you forget to use jump box? {str(e)}")
+ log.error("Timeout connecting to %s: %s", server, str(e))
except SSHException as e:
log.error("Can't read remote bufffer: %s", str(e))
except Exception as e:
@@ -431,8 +458,10 @@ def bulk_deploy(files, yara_category, dry_run=False, servers: list = [], ssh_pro
def get_file(path, servers: list, ssh_proxy: SSHClient, yara_category: str = "CAPE", dry_run: bool = False):
for server in servers:
try:
- print(server)
+ print(f"[*] Connecting to {server}...")
ssh = _connect_via_jump_box(server, ssh_proxy)
+ if not ssh:
+ continue
with SCPClient(ssh.get_transport()) as scp:
try:
scp.get(path, f"{server}_{os.path.basename(path)}")
@@ -454,6 +483,8 @@ def deploy_file(queue, ssh_proxy: SSHClient):
for server in servers:
try:
ssh = _connect_via_jump_box(server, ssh_proxy)
+ if not ssh:
+ continue
with SCPClient(ssh.get_transport()) as scp:
try:
scp.put(local_file, remote_file)
@@ -461,15 +492,24 @@ def deploy_file(queue, ssh_proxy: SSHClient):
print(e)
if remote_command:
- _, ssh_stdout, _ = ssh.exec_command(remote_command)
+ _, ssh_stdout, ssh_stderr = ssh.exec_command(remote_command, get_pty=True)
- ssh_out = ssh_stdout.read().decode("utf-8")
- log.info(ssh_out)
+ ssh_out = ssh_stdout.read().decode("utf-8").strip()
+ ssh_err = ssh_stderr.read().decode("utf-8").strip()
+ if ssh_out:
+ log.info(ssh_out)
+ if ssh_err:
+ log.error(red("ERROR: %s", ssh_err))
- _, ssh_stdout, _ = ssh.exec_command(f"sha256sum {remote_file} | cut -d' ' -f1")
+ _, ssh_stdout, ssh_stderr = ssh.exec_command(f"sha256sum {shlex.quote(remote_file)} | cut -d' ' -f1", get_pty=True)
remote_sha256 = ssh_stdout.read().strip().decode("utf-8")
+ remote_sha256_err = ssh_stderr.read().strip().decode("utf-8")
+ if remote_sha256_err:
+ log.error(red("sha256sum error: %s", remote_sha256_err))
+
+ srv = server.split(".")[1] if "." in server else server
if local_sha256 == remote_sha256:
- log.info("[+] %s - Hashes are %s: %s - %s", server.split(".")[1], green("correct"), local_sha256, remote_file)
+ log.info("[+] %s - Hashes are %s: %s - %s", srv, green("correct"), local_sha256, remote_file)
else:
log.info(
"[-] %s - Hashes are %s: \n\tLocal: %s\n\tRemote: %s - %s",
@@ -481,14 +521,13 @@ def deploy_file(queue, ssh_proxy: SSHClient):
)
error = 1
error_list.append(remote_file)
- ssh.close()
except TimeoutError as e:
log.error(e)
if not error:
- log.info(green(f"Completed! {remote_file}\n"))
+ log.info(green("Completed! %s\n", remote_file))
else:
- log.info(red(f"Completed with errors. {remote_file}\n"))
+ log.info(red("Completed with errors. %s\n", remote_file))
queue.task_done()
return error_list
@@ -504,11 +543,15 @@ def delete_file(queue, ssh_proxy: SSHClient):
for server in servers:
try:
ssh = _connect_via_jump_box(server, ssh_proxy)
- _, ssh_stdout, _ = ssh.exec_command(f"rm {remote_file}")
- ssh_out = ssh_stdout.read().decode("utf-8")
+ if not ssh:
+ continue
+ _, ssh_stdout, ssh_stderr = ssh.exec_command(f"rm {shlex.quote(remote_file)}", get_pty=True)
+ ssh_out = ssh_stdout.read().decode("utf-8").strip()
+ ssh_err = ssh_stderr.read().decode("utf-8").strip()
if ssh_out:
log.info(ssh_out)
- ssh.close()
+ if ssh_err:
+ log.error(red("ERROR: %s", ssh_err))
except TimeoutError as e:
log.error(e)
error = 1
diff --git a/modules/processing/CAPE.py b/modules/processing/CAPE.py
index 5a13f2e244e..1ae1ba8b302 100644
--- a/modules/processing/CAPE.py
+++ b/modules/processing/CAPE.py
@@ -299,6 +299,10 @@ def process_file(self, file_path, append_file, metadata: dict, *, category: str,
"category": category,
"file": file_info,
}
+
+ if not os.path.exists(self.task["target"]):
+ log.error("Target file doesn't exist anymore. That will prevent data to be shown on webgui")
+
elif processing_conf.CAPE.dropped and category in ("dropped", "package"):
if category == "dropped":
file_info.update(metadata.get(file_info["path"][0], {}))
diff --git a/utils/dist.py b/utils/dist.py
index bf2f36708fa..c71b9e373b6 100644
--- a/utils/dist.py
+++ b/utils/dist.py
@@ -53,6 +53,7 @@
)
from lib.cuckoo.core.database import (
Database,
+ Guest,
_Database,
init_database,
)
@@ -863,6 +864,46 @@ def delete_target_file(self, task_id: int, sample_sha256: str, target: str):
if not sample_still_used:
path_delete(copy_path)
+ def inject_guest_info(self, main_task_id: int, report_path: str):
+ """
+ Inject guest information from report.json into the main database.
+
+ Args:
+ main_task_id (int): The ID of the main task.
+ report_path (str): The path to the analysis folder.
+ """
+ report_json_path = os.path.join(report_path, "reports", "report.json")
+ if not path_exists(report_json_path):
+ return
+
+ try:
+ with open(report_json_path, "r") as f:
+ report_data = json.load(f)
+ machine = report_data.get("info", {}).get("machine", {})
+ if machine and isinstance(machine, dict):
+ with main_db.session.begin():
+ # Check if guest already exists
+ stmt = select(Guest).where(Guest.task_id == main_task_id)
+ if not main_db.session.scalar(stmt):
+ guest = Guest(
+ name=machine.get("name"),
+ label=machine.get("label"),
+ platform=machine.get("platform"),
+ manager=machine.get("manager"),
+ task_id=main_task_id,
+ )
+ # Set optional fields if they exist
+ if "started_on" in machine:
+ with suppress(Exception):
+ guest.started_on = datetime.strptime(machine["started_on"], "%Y-%m-%d %H:%M:%S")
+ if "shutdown_on" in machine:
+ with suppress(Exception):
+ guest.shutdown_on = datetime.strptime(machine["shutdown_on"], "%Y-%m-%d %H:%M:%S")
+
+ main_db.session.add(guest)
+ except Exception as e:
+ log.error("Failed to inject guest info for task %d: %s", main_task_id, e)
+
# This should be executed as external thread as it generates bottle neck
def fetch_latest_reports_nfs(self):
"""
@@ -961,6 +1002,8 @@ def fetch_latest_reports_nfs(self):
t.main_task_id,
)
+ self.inject_guest_info(t.main_task_id, report_path)
+
# this doesn't exist for some reason
if path_exists(t.path):
sample_sha256 = None
@@ -1141,6 +1184,7 @@ def fetch_latest_reports(self):
with zipfile.ZipFile(BytesIO(report.content)) as zf:
try:
zf.extractall(report_path)
+ self.inject_guest_info(t.main_task_id, report_path)
if (node_id, task.get("id")) not in self.cleaner_queue.queue:
self.cleaner_queue.put((node_id, task.get("id")))
except OSError:
diff --git a/web/templates/analysis/overview/_info.html b/web/templates/analysis/overview/_info.html
index e96c2dc2719..9b4e4cf1299 100644
--- a/web/templates/analysis/overview/_info.html
+++ b/web/templates/analysis/overview/_info.html
@@ -196,7 +196,7 @@ During-Script Log
-{% if analysis.info.machine and analysis.info.machine.name %}
+{% if analysis.info.machine %}