-
Notifications
You must be signed in to change notification settings - Fork 573
some fixes #2900
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
some fixes #2900
Changes from all commits
97b894a
229f6fb
cafd227
ef4b57a
2eb3f21
f7cea19
514d882
6539dd4
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -30,7 +30,6 @@ | |
|
|
||
| from lib.cuckoo.common.admin_utils import ( | ||
| CAPE_PATH, | ||
| POSTPROCESS, | ||
| AutoAddPolicy, | ||
| bulk_deploy, | ||
| compare_hashed_files, | ||
|
|
@@ -61,6 +60,7 @@ | |
|
|
||
| JUMPBOX_USED = False | ||
| jumpbox = False | ||
| RETRY = 3 | ||
|
|
||
| logging.getLogger("paramiko").setLevel(logging.WARNING) | ||
| logging.getLogger("paramiko.transport").setLevel(logging.WARNING) | ||
|
|
@@ -226,6 +226,13 @@ | |
| required=False, | ||
| default=False, | ||
| ) | ||
| compare_opt.add_argument( | ||
| "--remove-ssh-keys", | ||
| help="Remove servers ssh key from known keys on localhost", | ||
| action="store_true", | ||
| required=False, | ||
| default=False, | ||
| ) | ||
|
|
||
| args = parser.parse_args() | ||
|
|
||
|
|
@@ -235,12 +242,15 @@ | |
| logging.getLogger("paramiko.transport").setLevel(logging.DEBUG) | ||
|
|
||
| if args.username: | ||
| from lib.cuckoo.common import admin_utils | ||
| admin_utils.JUMP_BOX_USERNAME = args.username | ||
| JUMP_BOX_USERNAME = args.username | ||
|
|
||
| # if args.debug: | ||
| # log.setLevel(logging.DEBUG) | ||
| if args.debug: | ||
| logging.getLogger().setLevel(logging.DEBUG) | ||
|
|
||
| if args.jump_box_second and not args.dry_run: | ||
|
|
||
| ssh.connect( | ||
| JUMP_BOX_SECOND, | ||
| username=JUMP_BOX_SECOND_USERNAME, | ||
|
|
@@ -286,7 +296,12 @@ | |
| print(parameters) | ||
| sys.exit(0) | ||
| queue.put([servers, file] + list(parameters)) | ||
| _ = deploy_file(queue, jumpbox) | ||
| for i in range(RETRY): | ||
| try: | ||
| _ = deploy_file(queue, jumpbox) | ||
| break | ||
| except Exception as eee: | ||
| print(f"Error {eee}, retry {i + 1}/{RETRY}") | ||
|
|
||
| elif args.delete_file: | ||
| queue = Queue() | ||
|
|
@@ -342,7 +357,7 @@ | |
| sys.exit() | ||
|
|
||
| elif args.enum_all_servers: | ||
| enumerate_files_on_all_servers() | ||
| enumerate_files_on_all_servers(servers, jumpbox, "/opt/CAPEv2", args.filename) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The To remediate this, use |
||
| elif args.generate_files_listing and not args.enum_all_servers: | ||
| gen_hashfile(args.generate_files_listing, args.filename) | ||
| elif args.check_files_difference: | ||
|
|
@@ -355,8 +370,12 @@ | |
|
|
||
| bulk_deploy(files, args.yara_category, args.dry_run, servers, jumpbox) | ||
|
|
||
| if args.restart_service and POSTPROCESS: | ||
| execute_command_on_all(POSTPROCESS, servers, jumpbox) | ||
| if args.restart_service: | ||
| execute_command_on_all("systemctl restart cape-processor; systemctl status cape-processor", servers, jumpbox) | ||
|
|
||
| if args.restart_uwsgi: | ||
| execute_command_on_all("touch /tmp/capeuwsgireload", servers, jumpbox) | ||
|
|
||
| if args.remove_ssh_keys: | ||
| for node in SERVERS_STATIC_LIST: | ||
| subprocess.run(["ssh-keygen", "-R", node]) | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -2,6 +2,7 @@ | |
| import logging | ||
| import os | ||
| import re | ||
| import shlex | ||
|
|
||
| # from glob import glob | ||
| import shutil | ||
|
|
@@ -11,7 +12,7 @@ | |
| from pathlib import Path | ||
| from queue import Queue | ||
| from socket import if_nameindex | ||
| from threading import Thread | ||
| from threading import Lock, Thread | ||
|
|
||
| import urllib3 | ||
|
|
||
|
|
@@ -34,7 +35,6 @@ | |
| ) | ||
| from scp import SCPClient, SCPException | ||
|
|
||
|
|
||
| conf = SSHConfig() | ||
| conf.parse(open(os.path.expanduser("~/.ssh/config"))) | ||
|
|
||
|
|
@@ -197,7 +197,7 @@ def compare_hashed_files(files: list, servers: list, ssh_proxy: SSHClient, priva | |
|
|
||
|
|
||
| def enumerate_files_on_all_servers(servers: list, ssh_proxy: SSHClient, dir_folder: str, filename: str): | ||
| cmd = f"python3 {CAPE_PATH}/admin/admin.py -gfl {dir_folder} -f /tmp/{filename} -s" | ||
| cmd = f"python3 {CAPE_PATH}/admin/admin.py -gfl {shlex.quote(dir_folder)} -f /tmp/{shlex.quote(filename)} -s" | ||
| execute_command_on_all(cmd, servers, ssh_proxy) | ||
| get_file(f"/tmp/{filename}.json", servers, ssh_proxy) | ||
|
|
||
|
|
@@ -308,43 +308,52 @@ def file_recon(file, yara_category="CAPE"): | |
| return False | ||
|
|
||
| # build command to be executed remotely | ||
| REMOTE_COMMAND = f"chown {OWNER} {TARGET}; chmod 644 {TARGET};" | ||
| quoted_target = shlex.quote(TARGET) | ||
| REMOTE_COMMAND = f"chown {OWNER} {quoted_target}; chmod 644 {quoted_target};" | ||
| if filename.endswith(".py") and TARGET: | ||
| REMOTE_COMMAND += "rm -f {0}.pyc; ls -la {0}.*".format(TARGET.replace(".py", "")) | ||
| REMOTE_COMMAND += "rm -f {0}.pyc; ls -la {0}.*".format(shlex.quote(TARGET.replace(".py", ""))) | ||
| return TARGET, REMOTE_COMMAND, LOCAL_SHA256 | ||
|
|
||
|
|
||
| # For session reuse | ||
| sockets = {} | ||
| sockets_lock = Lock() | ||
|
|
||
|
|
||
| def _connect_via_jump_box(server: str, ssh_proxy: SSHClient): | ||
| session_checker() | ||
| host = conf.lookup(server) | ||
| try: | ||
| host = conf.lookup(server) | ||
| except NameError: | ||
| log.error("Missed dependencies/activation of venv") | ||
| return None | ||
|
|
||
| with sockets_lock: | ||
| if server in sockets: | ||
| ssh = sockets[server] | ||
| if ssh.get_transport() and ssh.get_transport().is_active(): | ||
| return ssh | ||
| else: | ||
| del sockets[server] | ||
|
|
||
| try: | ||
| """ | ||
| This is SSH pivoting it ssh to host Y via host X, can be used due to different networks | ||
| We doing direct-tcpip channel and pasing it as socket to be used | ||
| """ | ||
| if ssh_proxy and JUMP_BOX_USERNAME: | ||
| if server not in sockets: | ||
| ssh = SSHJumpClient(jump_session=ssh_proxy if ssh_proxy else None) | ||
| ssh.set_missing_host_key_policy(AutoAddPolicy()) | ||
| # ssh_port = 22 if ":" not in server else int(server.split(":")[1]) | ||
| ssh.connect( | ||
| server, | ||
| username=JUMP_BOX_USERNAME, | ||
| key_filename=host.get("identityfile"), | ||
| banner_timeout=200, | ||
| look_for_keys=False, | ||
| allow_agent=True, | ||
| # disabled_algorithms=dict(pubkeys=["rsa-sha2-512", "rsa-sha2-256"]), | ||
| ) | ||
| sockets[server] = ssh | ||
| else: | ||
| # ToDo check if alive and reconnect | ||
| ssh = sockets[server] | ||
|
|
||
| ssh = SSHJumpClient(jump_session=ssh_proxy if ssh_proxy else None) | ||
| ssh.set_missing_host_key_policy(AutoAddPolicy()) | ||
| # ssh_port = 22 if ":" not in server else int(server.split(":")[1]) | ||
| ssh.connect( | ||
| server, | ||
| username=JUMP_BOX_USERNAME, | ||
| key_filename=host.get("identityfile"), | ||
| banner_timeout=200, | ||
| look_for_keys=False, | ||
| allow_agent=True, | ||
| # disabled_algorithms=dict(pubkeys=["rsa-sha2-512", "rsa-sha2-256"]), | ||
| ) | ||
| else: | ||
| ssh = SSHJumpClient() | ||
| ssh.load_system_host_keys() | ||
|
|
@@ -357,35 +366,53 @@ def _connect_via_jump_box(server: str, ssh_proxy: SSHClient): | |
| banner_timeout=200, | ||
| look_for_keys=False, | ||
| allow_agent=True, | ||
| sock=ProxyCommand(host.get("proxycommand")), | ||
| sock=ProxyCommand(host.get("proxycommand")) if host.get("proxycommand") else None, | ||
| ) | ||
|
|
||
| with sockets_lock: | ||
| sockets[server] = ssh | ||
|
|
||
| except (BadHostKeyException, AuthenticationException, PasswordRequiredException) as e: | ||
| sys.exit( | ||
| f"Connect error: {str(e)}. Also pay attention to this log for more details /var/log/auth.log and paramiko might need update.\nAlso ensure that you have added your public ssh key to /root/.ssh/authorized_keys" | ||
| log.error( | ||
| "Connect error to %s: %s. Also pay attention to this log for more details /var/log/auth.log and paramiko might need update.\nAlso ensure that you have added your public ssh key to /root/.ssh/authorized_keys", server, str(e) | ||
| ) | ||
| return None | ||
| except ProxyCommandFailure as e: | ||
| # Todo reconnect | ||
| log.error("Can't connect to server: %s", str(e)) | ||
| log.error("Can't connect to server %s: %s", server, str(e)) | ||
| return None | ||
| except Exception as e: | ||
| log.error("Unexpected error connecting to %s: %s", server, str(e)) | ||
| return None | ||
|
|
||
| return ssh | ||
|
|
||
|
|
||
| def execute_command_on_all(remote_command, servers: list, ssh_proxy: SSHClient): | ||
| for server in servers: | ||
| srv = server.split(".")[1] if "." in server else server | ||
| log.info("[*] Connecting to %s...", server) | ||
| try: | ||
| ssh = _connect_via_jump_box(server, ssh_proxy) | ||
| _, ssh_stdout, _ = ssh.exec_command(remote_command) | ||
| if not ssh: | ||
| continue | ||
|
|
||
| _, ssh_stdout, ssh_stderr = ssh.exec_command(remote_command, get_pty=True) | ||
| ssh_out = ssh_stdout.read().decode("utf-8").strip() | ||
| ssh_err = ssh_stderr.read().decode("utf-8").strip() | ||
|
|
||
| if "Active: active (running)" in ssh_out and "systemctl status" not in remote_command: | ||
| log.info("[+] Service %s", green("restarted successfully and is UP")) | ||
| else: | ||
| srv = str(server.split(".")[1]) | ||
| if ssh_out: | ||
| log.info(green(f"[+] {srv} - {ssh_out}")) | ||
| else: | ||
| log.info(green(f"[+] {srv}")) | ||
| ssh.close() | ||
| log.info("[+] %s - Service %s", srv, green("restarted successfully and is UP")) | ||
| elif ssh_out: | ||
| log.info(green("[+] %s - %s", srv, ssh_out )) | ||
|
|
||
| if ssh_err: | ||
| log.error(red("[-] %s ERROR - %s", srv, ssh_err)) | ||
|
|
||
| if not ssh_out and not ssh_err: | ||
| log.info(green("[+] %s", srv)) | ||
|
|
||
| except TimeoutError as e: | ||
| sys.exit(f"Did you forget to use jump box? {str(e)}") | ||
| log.error("Timeout connecting to %s: %s", server, str(e)) | ||
| except SSHException as e: | ||
| log.error("Can't read remote bufffer: %s", str(e)) | ||
| except Exception as e: | ||
|
|
@@ -431,8 +458,10 @@ def bulk_deploy(files, yara_category, dry_run=False, servers: list = [], ssh_pro | |
| def get_file(path, servers: list, ssh_proxy: SSHClient, yara_category: str = "CAPE", dry_run: bool = False): | ||
| for server in servers: | ||
| try: | ||
| print(server) | ||
| print(f"[*] Connecting to {server}...") | ||
| ssh = _connect_via_jump_box(server, ssh_proxy) | ||
| if not ssh: | ||
| continue | ||
| with SCPClient(ssh.get_transport()) as scp: | ||
| try: | ||
| scp.get(path, f"{server}_{os.path.basename(path)}") | ||
|
|
@@ -454,22 +483,33 @@ def deploy_file(queue, ssh_proxy: SSHClient): | |
| for server in servers: | ||
| try: | ||
| ssh = _connect_via_jump_box(server, ssh_proxy) | ||
| if not ssh: | ||
| continue | ||
| with SCPClient(ssh.get_transport()) as scp: | ||
| try: | ||
| scp.put(local_file, remote_file) | ||
| except SCPException as e: | ||
| print(e) | ||
|
|
||
| if remote_command: | ||
| _, ssh_stdout, _ = ssh.exec_command(remote_command) | ||
| _, ssh_stdout, ssh_stderr = ssh.exec_command(remote_command, get_pty=True) | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The To remediate this, ensure that |
||
|
|
||
| ssh_out = ssh_stdout.read().decode("utf-8") | ||
| log.info(ssh_out) | ||
| ssh_out = ssh_stdout.read().decode("utf-8").strip() | ||
| ssh_err = ssh_stderr.read().decode("utf-8").strip() | ||
| if ssh_out: | ||
| log.info(ssh_out) | ||
| if ssh_err: | ||
| log.error(red("ERROR: %s", ssh_err)) | ||
|
|
||
| _, ssh_stdout, _ = ssh.exec_command(f"sha256sum {remote_file} | cut -d' ' -f1") | ||
| _, ssh_stdout, ssh_stderr = ssh.exec_command(f"sha256sum {shlex.quote(remote_file)} | cut -d' ' -f1", get_pty=True) | ||
| remote_sha256 = ssh_stdout.read().strip().decode("utf-8") | ||
| remote_sha256_err = ssh_stderr.read().strip().decode("utf-8") | ||
| if remote_sha256_err: | ||
| log.error(red("sha256sum error: %s", remote_sha256_err)) | ||
|
|
||
| srv = server.split(".")[1] if "." in server else server | ||
| if local_sha256 == remote_sha256: | ||
| log.info("[+] %s - Hashes are %s: %s - %s", server.split(".")[1], green("correct"), local_sha256, remote_file) | ||
| log.info("[+] %s - Hashes are %s: %s - %s", srv, green("correct"), local_sha256, remote_file) | ||
| else: | ||
| log.info( | ||
| "[-] %s - Hashes are %s: \n\tLocal: %s\n\tRemote: %s - %s", | ||
|
|
@@ -481,14 +521,13 @@ def deploy_file(queue, ssh_proxy: SSHClient): | |
| ) | ||
| error = 1 | ||
| error_list.append(remote_file) | ||
| ssh.close() | ||
| except TimeoutError as e: | ||
| log.error(e) | ||
|
|
||
| if not error: | ||
| log.info(green(f"Completed! {remote_file}\n")) | ||
| log.info(green("Completed! %s\n", remote_file)) | ||
| else: | ||
| log.info(red(f"Completed with errors. {remote_file}\n")) | ||
| log.info(red("Completed with errors. %s\n", remote_file)) | ||
| queue.task_done() | ||
|
|
||
| return error_list | ||
|
|
@@ -504,11 +543,15 @@ def delete_file(queue, ssh_proxy: SSHClient): | |
| for server in servers: | ||
| try: | ||
| ssh = _connect_via_jump_box(server, ssh_proxy) | ||
| _, ssh_stdout, _ = ssh.exec_command(f"rm {remote_file}") | ||
| ssh_out = ssh_stdout.read().decode("utf-8") | ||
| if not ssh: | ||
| continue | ||
| _, ssh_stdout, ssh_stderr = ssh.exec_command(f"rm {shlex.quote(remote_file)}", get_pty=True) | ||
| ssh_out = ssh_stdout.read().decode("utf-8").strip() | ||
| ssh_err = ssh_stderr.read().decode("utf-8").strip() | ||
| if ssh_out: | ||
| log.info(ssh_out) | ||
| ssh.close() | ||
| if ssh_err: | ||
| log.error(red("ERROR: %s", ssh_err)) | ||
| except TimeoutError as e: | ||
| log.error(e) | ||
| error = 1 | ||
|
|
||
Uh oh!
There was an error while loading. Please reload this page.