diff --git a/src/mvt/android/artifacts/dumpsys_adb.py b/src/mvt/android/artifacts/dumpsys_adb.py index 4ed166999..d2a33fbb0 100644 --- a/src/mvt/android/artifacts/dumpsys_adb.py +++ b/src/mvt/android/artifacts/dumpsys_adb.py @@ -131,10 +131,17 @@ def parse(self, content: bytes) -> None: ) return - # TODO: Parse AdbDebuggingManager line in output. - start_of_json = content.find(b"\n{") + 2 - end_of_json = content.rfind(b"}\n") - 2 - json_content = content[start_of_json:end_of_json].rstrip() + start_of_json = content.find(b"\n{") + if start_of_json == -1: + self.log.error("Unable to find ADB manager state in dumpsys output") + return + + end_of_json = content.rfind(b"}\n") + if end_of_json == -1 or end_of_json <= start_of_json: + self.log.error("Unable to find complete ADB manager state in dumpsys output") + return + + json_content = content[start_of_json + 2 : end_of_json - 2].rstrip() parsed = self.indented_dump_parser(json_content) if parsed.get("debugging_manager") is None: diff --git a/src/mvt/android/artifacts/dumpsys_packages.py b/src/mvt/android/artifacts/dumpsys_packages.py index f0cd854b8..81da22efc 100644 --- a/src/mvt/android/artifacts/dumpsys_packages.py +++ b/src/mvt/android/artifacts/dumpsys_packages.py @@ -14,9 +14,12 @@ class DumpsysPackagesArtifact(AndroidArtifact): def check_indicators(self) -> None: + alerted_root_packages = set() for result in self.results: - # XXX: De-duplication Package detections if result["package_name"] in ROOT_PACKAGES: + if result["package_name"] in alerted_root_packages: + continue + alerted_root_packages.add(result["package_name"]) self.alertstore.medium( f'Found an installed package related to rooting/jailbreaking: "{result["package_name"]}"', "", @@ -188,7 +191,7 @@ def parse(self, content: str): package = [] in_package_list = False - for line in content.split("\n"): + for line in content.splitlines(): if line.startswith("Packages:"): in_package_list = True continue diff --git a/src/mvt/android/artifacts/processes.py b/src/mvt/android/artifacts/processes.py index 7cf962e9b..5471eb29e 100644 --- a/src/mvt/android/artifacts/processes.py +++ b/src/mvt/android/artifacts/processes.py @@ -8,7 +8,7 @@ class Processes(AndroidArtifact): def parse(self, entry: str) -> None: - for line in entry.split("\n")[1:]: + for line in entry.splitlines()[1:]: proc = line.split() # Skip empty lines diff --git a/src/mvt/android/artifacts/settings.py b/src/mvt/android/artifacts/settings.py index 4649666ee..17c3bf2c5 100644 --- a/src/mvt/android/artifacts/settings.py +++ b/src/mvt/android/artifacts/settings.py @@ -67,11 +67,14 @@ def check_indicators(self) -> None: # Check if one of the dangerous settings is using an unsafe # value (different than the one specified). if danger["key"] == key and danger["safe_value"] != value: - self.log.warning( - 'Found suspicious "%s" setting "%s = %s" (%s)', - namespace, - key, - value, - danger["description"], + self.alertstore.medium( + f'Found suspicious "{namespace}" setting "{key} = {value}" ({danger["description"]})', + "", + { + "namespace": namespace, + "key": key, + "value": value, + "description": danger["description"], + }, ) break diff --git a/src/mvt/android/artifacts/tombstone_crashes.py b/src/mvt/android/artifacts/tombstone_crashes.py index 7d69f50dc..9a878aa14 100644 --- a/src/mvt/android/artifacts/tombstone_crashes.py +++ b/src/mvt/android/artifacts/tombstone_crashes.py @@ -101,8 +101,7 @@ def check_indicators(self) -> None: continue if result.get("command_line", []): - command_name = result.get("command_line")[0].split("/")[-1] - command_name = result["command_line"][0] + command_name = result["command_line"][0].split("/")[-1] ioc_match = self.indicators.check_process(command_name) if ioc_match: self.alertstore.critical( @@ -200,7 +199,7 @@ def _load_key_value_line( # eg. "Process uptime: 40s" tombstone[destination_key] = int(value_clean.rstrip("s")) elif destination_key == "command_line": - # XXX: Check if command line should be a single string in a list, or a list of strings. + # Wrap in list for consistency with protobuf format (repeated string). tombstone[destination_key] = [value_clean] else: tombstone[destination_key] = value_clean @@ -262,7 +261,7 @@ def _load_timestamp_line(self, line: str, tombstone: dict) -> bool: @staticmethod def _parse_timestamp_string(timestamp: str) -> str: timestamp_parsed = parser.parse(timestamp) - # HACK: Swap the local timestamp to UTC, so keep the original time and avoid timezone conversion. + # Preserve the source wall-clock time while returning the project-wide ISO format. local_timestamp = timestamp_parsed.replace(tzinfo=datetime.timezone.utc) return convert_datetime_to_iso(local_timestamp) diff --git a/src/mvt/android/cli.py b/src/mvt/android/cli.py index 96c1c7d65..12445063a 100644 --- a/src/mvt/android/cli.py +++ b/src/mvt/android/cli.py @@ -283,7 +283,13 @@ def check_androidqf( @click.argument("FOLDER", type=click.Path(exists=True)) @click.pass_context def check_iocs(ctx, iocs, list_modules, module, folder): - cmd = CmdCheckIOCS(target_path=folder, ioc_files=iocs, module_name=module) + cmd = CmdCheckIOCS( + target_path=folder, + ioc_files=iocs, + module_name=module, + disable_version_check=_get_disable_flags(ctx)[0], + disable_indicator_check=_get_disable_flags(ctx)[1], + ) cmd.modules = BACKUP_MODULES + BUGREPORT_MODULES + ANDROIDQF_MODULES if list_modules: diff --git a/src/mvt/android/modules/androidqf/aqf_files.py b/src/mvt/android/modules/androidqf/aqf_files.py index 0200db4e4..de9b44a6c 100644 --- a/src/mvt/android/modules/androidqf/aqf_files.py +++ b/src/mvt/android/modules/androidqf/aqf_files.py @@ -41,7 +41,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -107,16 +107,16 @@ def check_indicators(self) -> None: msg = f'Found {file_type}file at suspicious path "{result["path"]}"' self.alertstore.high(msg, "", result) - if result.get("sha256", "") == "": - continue - - ioc_match = self.indicators.check_file_hash(result.get("sha256") or "") - if ioc_match: - self.alertstore.critical( - ioc_match.message, "", result, matched_indicator=ioc_match.ioc - ) - - # TODO: adds SHA1 and MD5 when available in MVT + for hash_key in ("sha256", "sha1", "md5"): + file_hash = result.get(hash_key, "") + if not file_hash: + continue + ioc_match = self.indicators.check_file_hash(file_hash) + if ioc_match: + self.alertstore.critical( + ioc_match.message, "", result, matched_indicator=ioc_match.ioc + ) + break def run(self) -> None: if timezone := self._get_device_timezone(): @@ -131,7 +131,7 @@ def run(self) -> None: data = json.loads(rawdata) except json.decoder.JSONDecodeError: data = [] - for line in rawdata.split("\n"): + for line in rawdata.splitlines(): if line.strip() == "": continue data.append(json.loads(line)) @@ -142,11 +142,11 @@ def run(self) -> None: utc_timestamp = datetime.datetime.fromtimestamp( file_data[ts], tz=datetime.timezone.utc ) - # Convert the UTC timestamp to local tiem on Android device's local timezone + # Convert the UTC timestamp to local time on Android device's local timezone local_timestamp = utc_timestamp.astimezone(device_timezone) - # HACK: We only output the UTC timestamp in convert_datetime_to_iso, we - # set the timestamp timezone to UTC, to avoid the timezone conversion again. + # Preserve the device-local wall-clock time while using + # the project-wide ISO conversion helper. local_timestamp = local_timestamp.replace( tzinfo=datetime.timezone.utc ) diff --git a/src/mvt/android/modules/androidqf/aqf_getprop.py b/src/mvt/android/modules/androidqf/aqf_getprop.py index f41029ba8..938fb803a 100644 --- a/src/mvt/android/modules/androidqf/aqf_getprop.py +++ b/src/mvt/android/modules/androidqf/aqf_getprop.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -32,7 +32,7 @@ def __init__( log=log, results=results, ) - self.results: list = [] + self.results: list = [] if results is None else results def run(self) -> None: getprop_files = self._get_files_by_pattern("*/getprop.txt") diff --git a/src/mvt/android/modules/androidqf/aqf_log_timestamps.py b/src/mvt/android/modules/androidqf/aqf_log_timestamps.py index 1070d1bf5..305d6be66 100644 --- a/src/mvt/android/modules/androidqf/aqf_log_timestamps.py +++ b/src/mvt/android/modules/androidqf/aqf_log_timestamps.py @@ -27,7 +27,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/androidqf/aqf_packages.py b/src/mvt/android/modules/androidqf/aqf_packages.py index 0ec0122bc..264fd1e68 100644 --- a/src/mvt/android/modules/androidqf/aqf_packages.py +++ b/src/mvt/android/modules/androidqf/aqf_packages.py @@ -30,7 +30,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/androidqf/aqf_processes.py b/src/mvt/android/modules/androidqf/aqf_processes.py index 4c69ca02f..b940e675e 100644 --- a/src/mvt/android/modules/androidqf/aqf_processes.py +++ b/src/mvt/android/modules/androidqf/aqf_processes.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/androidqf/aqf_settings.py b/src/mvt/android/modules/androidqf/aqf_settings.py index 2974932d6..8d5bb518d 100644 --- a/src/mvt/android/modules/androidqf/aqf_settings.py +++ b/src/mvt/android/modules/androidqf/aqf_settings.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -32,7 +32,7 @@ def __init__( log=log, results=results, ) - self.results: dict = {} + self.results: dict = results if results is not None else {} def run(self) -> None: for setting_file in self._get_files_by_pattern("*/settings_*.txt"): @@ -40,7 +40,7 @@ def run(self) -> None: self.results[namespace] = {} data = self._get_file_content(setting_file) - for line in data.decode("utf-8").split("\n"): + for line in data.decode("utf-8").splitlines(): line = line.strip() try: key, value = line.split("=", 1) diff --git a/src/mvt/android/modules/androidqf/base.py b/src/mvt/android/modules/androidqf/base.py index be898fced..b0304d0ce 100644 --- a/src/mvt/android/modules/androidqf/base.py +++ b/src/mvt/android/modules/androidqf/base.py @@ -23,7 +23,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/androidqf/mounts.py b/src/mvt/android/modules/androidqf/mounts.py index fb1274ad8..ea446c2b1 100644 --- a/src/mvt/android/modules/androidqf/mounts.py +++ b/src/mvt/android/modules/androidqf/mounts.py @@ -32,7 +32,7 @@ def __init__( log=log, results=results, ) - self.results: list = [] + self.results: list = [] if results is None else results def run(self) -> None: """ @@ -66,6 +66,9 @@ def run(self) -> None: # AndroidQF format: array of strings like # "/dev/block/dm-12 on / type ext4 (ro,seclabel,noatime)" mount_content = "\n".join(json_data) + else: + self.log.error("Expected mounts.json to contain a list of mount lines") + return self.parse(mount_content) except Exception as exc: self.log.error("Failed to parse mount information: %s", exc) diff --git a/src/mvt/android/modules/androidqf/sms.py b/src/mvt/android/modules/androidqf/sms.py index 46cc3f6ba..bcbd226d5 100644 --- a/src/mvt/android/modules/androidqf/sms.py +++ b/src/mvt/android/modules/androidqf/sms.py @@ -21,10 +21,6 @@ class SMS(AndroidQFModule): """ This module analyse SMS file in backup - - XXX: We should also de-duplicate this AQF module, but first we - need to add tests for loading encrypted SMS backups through the backup - sub-module. """ def __init__( diff --git a/src/mvt/android/modules/backup/base.py b/src/mvt/android/modules/backup/base.py index d49a9456e..6383e4bcf 100644 --- a/src/mvt/android/modules/backup/base.py +++ b/src/mvt/android/modules/backup/base.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/backup/sms.py b/src/mvt/android/modules/backup/sms.py index 1c75587f2..6f8306a51 100644 --- a/src/mvt/android/modules/backup/sms.py +++ b/src/mvt/android/modules/backup/sms.py @@ -20,7 +20,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/base.py b/src/mvt/android/modules/bugreport/base.py index 70ede77d2..e73b11936 100644 --- a/src/mvt/android/modules/bugreport/base.py +++ b/src/mvt/android/modules/bugreport/base.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_accessibility.py b/src/mvt/android/modules/bugreport/dumpsys_accessibility.py index 0c0f294a2..72208c02e 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_accessibility.py +++ b/src/mvt/android/modules/bugreport/dumpsys_accessibility.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_activities.py b/src/mvt/android/modules/bugreport/dumpsys_activities.py index bfceebfcf..2800e95ea 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_activities.py +++ b/src/mvt/android/modules/bugreport/dumpsys_activities.py @@ -24,7 +24,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_adb_state.py b/src/mvt/android/modules/bugreport/dumpsys_adb_state.py index 07d4694bc..506af30d8 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_adb_state.py +++ b/src/mvt/android/modules/bugreport/dumpsys_adb_state.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_appops.py b/src/mvt/android/modules/bugreport/dumpsys_appops.py index f3ab41c29..91122cb57 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_appops.py +++ b/src/mvt/android/modules/bugreport/dumpsys_appops.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_battery_daily.py b/src/mvt/android/modules/bugreport/dumpsys_battery_daily.py index 365d193fb..a7c0c722d 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_battery_daily.py +++ b/src/mvt/android/modules/bugreport/dumpsys_battery_daily.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_battery_history.py b/src/mvt/android/modules/bugreport/dumpsys_battery_history.py index 2e0f468ac..42d395df2 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_battery_history.py +++ b/src/mvt/android/modules/bugreport/dumpsys_battery_history.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_dbinfo.py b/src/mvt/android/modules/bugreport/dumpsys_dbinfo.py index 96b0bf352..13ba8b33b 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_dbinfo.py +++ b/src/mvt/android/modules/bugreport/dumpsys_dbinfo.py @@ -24,7 +24,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_getprop.py b/src/mvt/android/modules/bugreport/dumpsys_getprop.py index 2bb5cd662..198a1d2ab 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_getprop.py +++ b/src/mvt/android/modules/bugreport/dumpsys_getprop.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_packages.py b/src/mvt/android/modules/bugreport/dumpsys_packages.py index 6bc5d27d3..aebec4c26 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_packages.py +++ b/src/mvt/android/modules/bugreport/dumpsys_packages.py @@ -23,7 +23,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py b/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py index 29e58f3ad..968bc25ea 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py +++ b/src/mvt/android/modules/bugreport/dumpsys_platform_compat.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -42,8 +42,10 @@ def run(self) -> None: ) return - data = data.decode("utf-8", errors="replace") - content = self.extract_dumpsys_section(data, "DUMP OF SERVICE platform_compat:") + decoded_data = data.decode("utf-8", errors="replace") + content = self.extract_dumpsys_section( + decoded_data, "DUMP OF SERVICE platform_compat:" + ) self.parse(content) self.log.info("Found %d uninstalled apps", len(self.results)) diff --git a/src/mvt/android/modules/bugreport/dumpsys_receivers.py b/src/mvt/android/modules/bugreport/dumpsys_receivers.py index 1c4a02878..ded9069d7 100644 --- a/src/mvt/android/modules/bugreport/dumpsys_receivers.py +++ b/src/mvt/android/modules/bugreport/dumpsys_receivers.py @@ -22,7 +22,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/fs_timestamps.py b/src/mvt/android/modules/bugreport/fs_timestamps.py index 000d07617..5a2ca485f 100644 --- a/src/mvt/android/modules/bugreport/fs_timestamps.py +++ b/src/mvt/android/modules/bugreport/fs_timestamps.py @@ -24,7 +24,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/modules/bugreport/tombstones.py b/src/mvt/android/modules/bugreport/tombstones.py index 58ef254fd..c4a7afb4a 100644 --- a/src/mvt/android/modules/bugreport/tombstones.py +++ b/src/mvt/android/modules/bugreport/tombstones.py @@ -23,7 +23,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/android/parsers/backup.py b/src/mvt/android/parsers/backup.py index 105b4f2ed..c81ecd0b6 100644 --- a/src/mvt/android/parsers/backup.py +++ b/src/mvt/android/parsers/backup.py @@ -29,9 +29,6 @@ class InvalidBackupPassword(AndroidBackupParsingError): pass -# TODO: Need to clean all the following code and conform it to the coding style. - - def to_utf8_bytes(input_bytes): output = [] for byte in input_bytes: @@ -157,13 +154,13 @@ def decrypt_backup_data(encrypted_backup, password, encryption_algo, format_vers checksum_salt=checksum_salt, ) - # Decrypt and unpad backup data using derivied key. + # Decrypt and unpad backup data using derived key. cipher = Cipher(algorithms.AES(master_key), modes.CBC(master_iv)) decryptor = cipher.decryptor() decrypted_tar = decryptor.update(encrypted_data) + decryptor.finalize() unpadder = padding.PKCS7(128).unpadder() - return unpadder.update(decrypted_tar) + return unpadder.update(decrypted_tar) + unpadder.finalize() def parse_backup_file(data, password=None): @@ -210,6 +207,8 @@ def parse_tar_for_sms(data): or member.name.endswith("_mms_backup") ): dhandler = tar.extractfile(member) + if not dhandler: + continue res.extend(parse_sms_file(dhandler.read())) return res diff --git a/src/mvt/common/artifact.py b/src/mvt/common/artifact.py index af0ba98c4..8d7b60ddf 100644 --- a/src/mvt/common/artifact.py +++ b/src/mvt/common/artifact.py @@ -8,5 +8,6 @@ class Artifact(MVTModule): """Base class for artifacts. - XXX: Inheriting from MVTModule to have the same signature as other modules. Not sure if this is a good idea. + Artifacts share the MVTModule lifecycle so commands can run artifacts and + extraction modules through the same interface. """ diff --git a/src/mvt/common/command.py b/src/mvt/common/command.py index 03d2b00ca..8a43d17e5 100644 --- a/src/mvt/common/command.py +++ b/src/mvt/common/command.py @@ -273,7 +273,6 @@ def run(self) -> None: ): continue - # FIXME: do we need the logger here module_logger = logging.getLogger(module.__module__) m = module( diff --git a/src/mvt/common/module.py b/src/mvt/common/module.py index f555e409b..2d8a3950b 100644 --- a/src/mvt/common/module.py +++ b/src/mvt/common/module.py @@ -51,7 +51,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[Dict[str, Any]] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: """Initialize module. @@ -71,15 +71,13 @@ def __init__( self.file_path: Optional[str] = file_path self.target_path: Optional[str] = target_path self.results_path: Optional[str] = results_path - self.module_options: Optional[Dict[str, Any]] = ( - module_options if module_options else {} - ) + self.module_options: Dict[str, Any] = module_options if module_options else {} self.log = log self.indicators: Optional[Indicators] = None self.alertstore: AlertStore = AlertStore(log=log) - self.results: ModuleResults = results if results else [] + self.results: ModuleResults = results if results is not None else [] self.timeline: ModuleTimeline = [] @classmethod @@ -109,11 +107,14 @@ def save_to_json(self) -> None: name = self.get_slug() if self.results: + converted_results: Any if isinstance(self.results, dict): converted_results = self.results else: converted_results = [ - asdict(result) if is_dataclass(result) else result + asdict(result) + if is_dataclass(result) and not isinstance(result, type) + else result for result in self.results ] results_file_name = f"{name}.json" diff --git a/src/mvt/common/module_types.py b/src/mvt/common/module_types.py index 41bbdaab1..06fdc12e4 100644 --- a/src/mvt/common/module_types.py +++ b/src/mvt/common/module_types.py @@ -16,7 +16,10 @@ ModuleAtomicResult = Dict[str, Any] -ModuleResults = List[ModuleAtomicResult] +# Extraction modules historically use either a list of records or grouped +# dictionaries keyed by source path. Keep this alias broad until those shapes +# are modeled per module. +ModuleResults = Any @dataclass diff --git a/src/mvt/common/updates.py b/src/mvt/common/updates.py index c95f9ceb8..001a5c21e 100644 --- a/src/mvt/common/updates.py +++ b/src/mvt/common/updates.py @@ -180,10 +180,8 @@ def update(self) -> None: def _get_remote_file_latest_commit( self, owner: str, repo: str, branch: str, path: str ) -> int: - # TODO: The branch is currently not taken into consideration. - # How do we specify which branch to look up to the API? file_commit_url = ( - f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}" + f"https://api.github.com/repos/{owner}/{repo}/commits?path={path}&sha={branch}" ) try: res = requests.get(file_commit_url, timeout=5) diff --git a/src/mvt/common/utils.py b/src/mvt/common/utils.py index 78352d44d..30de15957 100644 --- a/src/mvt/common/utils.py +++ b/src/mvt/common/utils.py @@ -123,10 +123,9 @@ def convert_mactime_to_datetime(timestamp: Union[int, float], from_2001: bool = if from_2001: timestamp = timestamp + 978307200 - # TODO: This is rather ugly. Happens sometimes with invalid timestamps. try: return convert_unix_to_utc_datetime(timestamp) - except Exception: + except (OSError, OverflowError, ValueError): return None diff --git a/src/mvt/ios/modules/backup/backup_info.py b/src/mvt/ios/modules/backup/backup_info.py index 9e0720086..6aadd45fd 100644 --- a/src/mvt/ios/modules/backup/backup_info.py +++ b/src/mvt/ios/modules/backup/backup_info.py @@ -25,7 +25,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -36,7 +36,7 @@ def __init__( results=results, ) - self.results: dict = {} + self.results: dict = results if results is not None else {} def run(self) -> None: if not self.target_path: diff --git a/src/mvt/ios/modules/backup/configuration_profiles.py b/src/mvt/ios/modules/backup/configuration_profiles.py index 07a9c0c5e..9c0343e83 100644 --- a/src/mvt/ios/modules/backup/configuration_profiles.py +++ b/src/mvt/ios/modules/backup/configuration_profiles.py @@ -33,7 +33,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -85,6 +85,35 @@ def check_indicators(self) -> None: self.alertstore.medium(warning_message, "", result) continue + @staticmethod + def _b64encode_key(d: dict, key: str) -> None: + if key in d: + d[key] = b64encode(d[key]) + + @staticmethod + def _b64encode_keys(d: dict, keys: list) -> None: + for key in keys: + if key in d: + d[key] = b64encode(d[key]) + + def _b64encode_plist_bytes(self, plist: dict) -> None: + """Encode binary plist values to base64 for JSON serialization.""" + if "SignerCerts" in plist: + plist["SignerCerts"] = [b64encode(x) for x in plist["SignerCerts"]] + + self._b64encode_keys(plist, ["PushTokenDataSentToServerKey", "LastPushTokenHash"]) + + if "OTAProfileStub" in plist: + stub = plist["OTAProfileStub"] + if "SignerCerts" in stub: + stub["SignerCerts"] = [b64encode(x) for x in stub["SignerCerts"]] + if "PayloadContent" in stub: + self._b64encode_key(stub["PayloadContent"], "EnrollmentIdentityPersistentID") + + if "PayloadContent" in plist: + for entry in plist["PayloadContent"]: + self._b64encode_keys(entry, ["PERSISTENT_REF", "IdentityPersistentRef"]) + def run(self) -> None: for conf_file in self._get_backup_files_from_manifest( domain=CONF_PROFILES_DOMAIN @@ -113,65 +142,7 @@ def run(self) -> None: except Exception: conf_plist = {} - # TODO: Tidy up the following code hell. - - if "SignerCerts" in conf_plist: - conf_plist["SignerCerts"] = [ - b64encode(x) for x in conf_plist["SignerCerts"] - ] - - if "OTAProfileStub" in conf_plist: - if "SignerCerts" in conf_plist["OTAProfileStub"]: - conf_plist["OTAProfileStub"]["SignerCerts"] = [ - b64encode(x) - for x in conf_plist["OTAProfileStub"]["SignerCerts"] - ] - - if "PayloadContent" in conf_plist["OTAProfileStub"]: - if ( - "EnrollmentIdentityPersistentID" - in conf_plist["OTAProfileStub"]["PayloadContent"] - ): - conf_plist["OTAProfileStub"]["PayloadContent"][ - "EnrollmentIdentityPersistentID" - ] = b64encode( - conf_plist["OTAProfileStub"]["PayloadContent"][ - "EnrollmentIdentityPersistentID" - ] - ) - - if "PushTokenDataSentToServerKey" in conf_plist: - conf_plist["PushTokenDataSentToServerKey"] = b64encode( - conf_plist["PushTokenDataSentToServerKey"] - ) - - if "LastPushTokenHash" in conf_plist: - conf_plist["LastPushTokenHash"] = b64encode( - conf_plist["LastPushTokenHash"] - ) - - if "PayloadContent" in conf_plist: - for content_entry in range(len(conf_plist["PayloadContent"])): - if "PERSISTENT_REF" in conf_plist["PayloadContent"][content_entry]: - conf_plist["PayloadContent"][content_entry][ - "PERSISTENT_REF" - ] = b64encode( - conf_plist["PayloadContent"][content_entry][ - "PERSISTENT_REF" - ] - ) - - if ( - "IdentityPersistentRef" - in conf_plist["PayloadContent"][content_entry] - ): - conf_plist["PayloadContent"][content_entry][ - "IdentityPersistentRef" - ] = b64encode( - conf_plist["PayloadContent"][content_entry][ - "IdentityPersistentRef" - ] - ) + self._b64encode_plist_bytes(conf_plist) self.results.append( { diff --git a/src/mvt/ios/modules/backup/manifest.py b/src/mvt/ios/modules/backup/manifest.py index 0eaa6bd1f..cc74cbb69 100644 --- a/src/mvt/ios/modules/backup/manifest.py +++ b/src/mvt/ios/modules/backup/manifest.py @@ -33,7 +33,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/backup/profile_events.py b/src/mvt/ios/modules/backup/profile_events.py index 7bd7b94ec..1fc6d7ed5 100644 --- a/src/mvt/ios/modules/backup/profile_events.py +++ b/src/mvt/ios/modules/backup/profile_events.py @@ -34,7 +34,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/base.py b/src/mvt/ios/modules/base.py index 3ff550977..5ac907158 100644 --- a/src/mvt/ios/modules/base.py +++ b/src/mvt/ios/modules/base.py @@ -30,7 +30,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -52,7 +52,7 @@ def _recover_sqlite_db_if_needed( :param file_path: Path to the malformed database file. """ - # TODO: Find a better solution. + # SQLite's immutable mode cannot open databases with active WAL files. if not forced: # If the database is open, do not use immutable if os.path.isfile(file_path + "-shm"): diff --git a/src/mvt/ios/modules/fs/analytics.py b/src/mvt/ios/modules/fs/analytics.py index bf5374586..0a6e50cdc 100644 --- a/src/mvt/ios/modules/fs/analytics.py +++ b/src/mvt/ios/modules/fs/analytics.py @@ -34,7 +34,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -44,7 +44,7 @@ def __init__( log=log, results=results, ) - self.results: list = [] + self.results: list = [] if results is None else results def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: return { diff --git a/src/mvt/ios/modules/fs/analytics_ios_versions.py b/src/mvt/ios/modules/fs/analytics_ios_versions.py index 5fb300ed7..783bb4550 100644 --- a/src/mvt/ios/modules/fs/analytics_ios_versions.py +++ b/src/mvt/ios/modules/fs/analytics_ios_versions.py @@ -30,7 +30,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/fs/cache_files.py b/src/mvt/ios/modules/fs/cache_files.py index 021924a0c..54a34eb18 100644 --- a/src/mvt/ios/modules/fs/cache_files.py +++ b/src/mvt/ios/modules/fs/cache_files.py @@ -25,7 +25,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -99,6 +99,10 @@ def _process_cache_file(self, file_path): def run(self) -> None: self.results: dict = {} + if not self.target_path: + self.log.error("No filesystem dump path provided") + return + for root, _, files in os.walk(self.target_path): for file_name in files: if file_name != "Cache.db": diff --git a/src/mvt/ios/modules/fs/filesystem.py b/src/mvt/ios/modules/fs/filesystem.py index de3aa650c..563d03948 100644 --- a/src/mvt/ios/modules/fs/filesystem.py +++ b/src/mvt/ios/modules/fs/filesystem.py @@ -29,7 +29,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -73,6 +73,10 @@ def check_indicators(self) -> None: ) def run(self) -> None: + if not self.target_path: + self.log.error("No filesystem dump path provided") + return + for root, dirs, files in os.walk(self.target_path): for dir_name in dirs: try: diff --git a/src/mvt/ios/modules/fs/net_netusage.py b/src/mvt/ios/modules/fs/net_netusage.py index 23b97f3fb..91451d6ac 100644 --- a/src/mvt/ios/modules/fs/net_netusage.py +++ b/src/mvt/ios/modules/fs/net_netusage.py @@ -30,7 +30,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/fs/safari_favicon.py b/src/mvt/ios/modules/fs/safari_favicon.py index c5c078b3d..a9c0b65fb 100644 --- a/src/mvt/ios/modules/fs/safari_favicon.py +++ b/src/mvt/ios/modules/fs/safari_favicon.py @@ -31,7 +31,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -41,7 +41,7 @@ def __init__( log=log, results=results, ) - self.results: list = [] + self.results: list = [] if results is None else results def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: return { diff --git a/src/mvt/ios/modules/fs/shutdownlog.py b/src/mvt/ios/modules/fs/shutdownlog.py index ac028093d..6c1fca7d4 100644 --- a/src/mvt/ios/modules/fs/shutdownlog.py +++ b/src/mvt/ios/modules/fs/shutdownlog.py @@ -30,7 +30,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -78,7 +78,7 @@ def process_shutdownlog(self, content): recent_processes = [] times_delayed = 0 delay = 0.0 - for line in content.split("\n"): + for line in content.splitlines(): line = line.strip() if line.startswith("remaining client pid:"): diff --git a/src/mvt/ios/modules/fs/version_history.py b/src/mvt/ios/modules/fs/version_history.py index bd5515c49..c3c583c32 100644 --- a/src/mvt/ios/modules/fs/version_history.py +++ b/src/mvt/ios/modules/fs/version_history.py @@ -32,7 +32,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -42,7 +42,7 @@ def __init__( log=log, results=results, ) - self.results: list = [] + self.results: list = [] if results is None else results def serialize(self, record: ModuleAtomicResult) -> ModuleSerializedResult: return { diff --git a/src/mvt/ios/modules/fs/webkit_indexeddb.py b/src/mvt/ios/modules/fs/webkit_indexeddb.py index 58cea427d..42f0895d2 100644 --- a/src/mvt/ios/modules/fs/webkit_indexeddb.py +++ b/src/mvt/ios/modules/fs/webkit_indexeddb.py @@ -34,7 +34,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/fs/webkit_localstorage.py b/src/mvt/ios/modules/fs/webkit_localstorage.py index 2b94fd1f4..d1ad05f9f 100644 --- a/src/mvt/ios/modules/fs/webkit_localstorage.py +++ b/src/mvt/ios/modules/fs/webkit_localstorage.py @@ -32,7 +32,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/fs/webkit_safariviewservice.py b/src/mvt/ios/modules/fs/webkit_safariviewservice.py index caa7eef14..62c948171 100644 --- a/src/mvt/ios/modules/fs/webkit_safariviewservice.py +++ b/src/mvt/ios/modules/fs/webkit_safariviewservice.py @@ -28,7 +28,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/applications.py b/src/mvt/ios/modules/mixed/applications.py index d508910f4..45b88cd00 100644 --- a/src/mvt/ios/modules/mixed/applications.py +++ b/src/mvt/ios/modules/mixed/applications.py @@ -41,7 +41,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/calendar.py b/src/mvt/ios/modules/mixed/calendar.py index dddd11c02..40f8ad9ab 100644 --- a/src/mvt/ios/modules/mixed/calendar.py +++ b/src/mvt/ios/modules/mixed/calendar.py @@ -31,7 +31,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/calls.py b/src/mvt/ios/modules/mixed/calls.py index 4d411bf69..197f8bf93 100644 --- a/src/mvt/ios/modules/mixed/calls.py +++ b/src/mvt/ios/modules/mixed/calls.py @@ -27,7 +27,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: list = [], + results: Optional[list] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/chrome_favicon.py b/src/mvt/ios/modules/mixed/chrome_favicon.py index 297d48d00..af3df2ed5 100644 --- a/src/mvt/ios/modules/mixed/chrome_favicon.py +++ b/src/mvt/ios/modules/mixed/chrome_favicon.py @@ -16,7 +16,6 @@ from ..base import IOSExtraction CHROME_FAVICON_BACKUP_IDS = ["55680ab883d0fdcffd94f959b1632e5fbbb18c5b"] -# TODO: Confirm Chrome database path. CHROME_FAVICON_ROOT_PATHS = [ "private/var/mobile/Containers/Data/Application/*/Library/Application Support/Google/Chrome/Default/Favicons", ] @@ -32,7 +31,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/chrome_history.py b/src/mvt/ios/modules/mixed/chrome_history.py index d9d50f326..4c5afae23 100644 --- a/src/mvt/ios/modules/mixed/chrome_history.py +++ b/src/mvt/ios/modules/mixed/chrome_history.py @@ -18,7 +18,6 @@ CHROME_HISTORY_BACKUP_IDS = [ "faf971ce92c3ac508c018dce1bef2a8b8e9838f1", ] -# TODO: Confirm Chrome database path. CHROME_HISTORY_ROOT_PATHS = [ "private/var/mobile/Containers/Data/Application/*/Library/Application Support/Google/Chrome/Default/History", # pylint: disable=line-too-long ] @@ -34,7 +33,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/contacts.py b/src/mvt/ios/modules/mixed/contacts.py index e2ed3ddf6..ca0470c90 100644 --- a/src/mvt/ios/modules/mixed/contacts.py +++ b/src/mvt/ios/modules/mixed/contacts.py @@ -29,7 +29,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/firefox_favicon.py b/src/mvt/ios/modules/mixed/firefox_favicon.py index 8df6165ec..ad92e7347 100644 --- a/src/mvt/ios/modules/mixed/firefox_favicon.py +++ b/src/mvt/ios/modules/mixed/firefox_favicon.py @@ -33,7 +33,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/firefox_history.py b/src/mvt/ios/modules/mixed/firefox_history.py index 8b13279d9..7ab1eba48 100644 --- a/src/mvt/ios/modules/mixed/firefox_history.py +++ b/src/mvt/ios/modules/mixed/firefox_history.py @@ -37,7 +37,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/global_preferences.py b/src/mvt/ios/modules/mixed/global_preferences.py index 04b11bcb1..97a2aef0f 100644 --- a/src/mvt/ios/modules/mixed/global_preferences.py +++ b/src/mvt/ios/modules/mixed/global_preferences.py @@ -27,7 +27,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/idstatuscache.py b/src/mvt/ios/modules/mixed/idstatuscache.py index 34d5bda0b..c841f1ee5 100644 --- a/src/mvt/ios/modules/mixed/idstatuscache.py +++ b/src/mvt/ios/modules/mixed/idstatuscache.py @@ -36,7 +36,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/interactionc.py b/src/mvt/ios/modules/mixed/interactionc.py index ba07bcda4..81a67e26e 100644 --- a/src/mvt/ios/modules/mixed/interactionc.py +++ b/src/mvt/ios/modules/mixed/interactionc.py @@ -228,7 +228,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/locationd.py b/src/mvt/ios/modules/mixed/locationd.py index b022bc87e..86de85b01 100644 --- a/src/mvt/ios/modules/mixed/locationd.py +++ b/src/mvt/ios/modules/mixed/locationd.py @@ -36,7 +36,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -147,7 +147,6 @@ def _extract_locationd_entries(self, file_path): # Some migration information are int and not dicts if not isinstance(file_plist[key], dict): continue - # FIXME: unclear key format in iOS 17 result = file_plist[key] result["package"] = key.rstrip(":") for timestamp in self.timestamps: diff --git a/src/mvt/ios/modules/mixed/net_datausage.py b/src/mvt/ios/modules/mixed/net_datausage.py index 713a7f9d6..61b0eef4c 100644 --- a/src/mvt/ios/modules/mixed/net_datausage.py +++ b/src/mvt/ios/modules/mixed/net_datausage.py @@ -31,7 +31,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/osanalytics_addaily.py b/src/mvt/ios/modules/mixed/osanalytics_addaily.py index 7f3243f50..8a5db3f64 100644 --- a/src/mvt/ios/modules/mixed/osanalytics_addaily.py +++ b/src/mvt/ios/modules/mixed/osanalytics_addaily.py @@ -35,7 +35,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/safari_browserstate.py b/src/mvt/ios/modules/mixed/safari_browserstate.py index 20a594c21..48998f1a4 100644 --- a/src/mvt/ios/modules/mixed/safari_browserstate.py +++ b/src/mvt/ios/modules/mixed/safari_browserstate.py @@ -36,7 +36,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/safari_history.py b/src/mvt/ios/modules/mixed/safari_history.py index 4583ffbb1..9ba32ad50 100644 --- a/src/mvt/ios/modules/mixed/safari_history.py +++ b/src/mvt/ios/modules/mixed/safari_history.py @@ -38,7 +38,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/shortcuts.py b/src/mvt/ios/modules/mixed/shortcuts.py index 9ca273912..f8899a42f 100644 --- a/src/mvt/ios/modules/mixed/shortcuts.py +++ b/src/mvt/ios/modules/mixed/shortcuts.py @@ -37,7 +37,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/sms.py b/src/mvt/ios/modules/mixed/sms.py index bf6890b31..76e004876 100644 --- a/src/mvt/ios/modules/mixed/sms.py +++ b/src/mvt/ios/modules/mixed/sms.py @@ -35,7 +35,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/sms_attachments.py b/src/mvt/ios/modules/mixed/sms_attachments.py index 981bf39b1..d02bd2713 100644 --- a/src/mvt/ios/modules/mixed/sms_attachments.py +++ b/src/mvt/ios/modules/mixed/sms_attachments.py @@ -35,7 +35,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/tcc.py b/src/mvt/ios/modules/mixed/tcc.py index b0c54d4f4..ed878c5a7 100644 --- a/src/mvt/ios/modules/mixed/tcc.py +++ b/src/mvt/ios/modules/mixed/tcc.py @@ -56,7 +56,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/mixed/webkit_resource_load_statistics.py b/src/mvt/ios/modules/mixed/webkit_resource_load_statistics.py index 6499a6a52..55cd740b5 100644 --- a/src/mvt/ios/modules/mixed/webkit_resource_load_statistics.py +++ b/src/mvt/ios/modules/mixed/webkit_resource_load_statistics.py @@ -37,7 +37,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -85,32 +85,55 @@ def _process_observations_db(self, db_path: str, domain: str, path: str) -> None try: try: - # FIXME: table contains extra fields with timestamp here cur.execute( """ SELECT domainID, registrableDomain, lastSeen, - hadUserInteraction + hadUserInteraction, + mostRecentUserInteractionTime, + mostRecentWebPushInteractionTime from ObservedDomains; """ ) + has_extra_timestamps = True except sqlite3.OperationalError: - return + try: + cur.execute( + """ + SELECT + domainID, + registrableDomain, + lastSeen, + hadUserInteraction + from ObservedDomains; + """ + ) + has_extra_timestamps = False + except sqlite3.OperationalError: + return for row in cur: - self.results.append( - { - "domain_id": row[0], - "registrable_domain": row[1], - "last_seen": row[2], - "had_user_interaction": bool(row[3]), - "last_seen_isodate": convert_unix_to_iso(row[2]), - "domain": domain, - "path": path, - } - ) + result = { + "domain_id": row[0], + "registrable_domain": row[1], + "last_seen": row[2], + "had_user_interaction": bool(row[3]), + "last_seen_isodate": convert_unix_to_iso(row[2]), + "domain": domain, + "path": path, + } + if has_extra_timestamps: + result["most_recent_user_interaction_time"] = row[4] + result["most_recent_user_interaction_time_isodate"] = ( + convert_unix_to_iso(row[4]) + ) + result["most_recent_web_push_interaction_time"] = row[5] + result["most_recent_web_push_interaction_time_isodate"] = ( + convert_unix_to_iso(row[5]) + ) + self.results.append(result) finally: cur.close() conn.close() diff --git a/src/mvt/ios/modules/mixed/webkit_session_resource_log.py b/src/mvt/ios/modules/mixed/webkit_session_resource_log.py index b56cc5e3f..d8b9a85cd 100644 --- a/src/mvt/ios/modules/mixed/webkit_session_resource_log.py +++ b/src/mvt/ios/modules/mixed/webkit_session_resource_log.py @@ -39,7 +39,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -50,7 +50,7 @@ def __init__( results=results, ) - self.results: dict = {} + self.results: dict = results if results is not None else {} @staticmethod def _extract_domains(entries): @@ -77,14 +77,21 @@ def check_indicators(self) -> None: entry["redirect_destination"] ) - # TODO: Currently not used. - # subframe_origins = self._extract_domains( - # entry["subframe_under_origin"]) - # subresource_domains = self._extract_domains( - # entry["subresource_under_origin"]) + subframe_origins = self._extract_domains( + entry["subframe_under_origin"] + ) + subresource_domains = self._extract_domains( + entry["subresource_under_origin"] + ) all_origins = list( - set([entry["origin"]] + source_domains + destination_domains) + set( + [entry["origin"]] + + source_domains + + destination_domains + + subframe_origins + + subresource_domains + ) ) ioc_match = self.indicators.check_urls(all_origins) diff --git a/src/mvt/ios/modules/mixed/whatsapp.py b/src/mvt/ios/modules/mixed/whatsapp.py index e28d4cb1a..2a72c6f48 100644 --- a/src/mvt/ios/modules/mixed/whatsapp.py +++ b/src/mvt/ios/modules/mixed/whatsapp.py @@ -33,7 +33,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, diff --git a/src/mvt/ios/modules/net_base.py b/src/mvt/ios/modules/net_base.py index 302123eb6..99a851d9c 100644 --- a/src/mvt/ios/modules/net_base.py +++ b/src/mvt/ios/modules/net_base.py @@ -30,7 +30,7 @@ def __init__( results_path: Optional[str] = None, module_options: Optional[dict] = None, log: logging.Logger = logging.getLogger(__name__), - results: ModuleResults = [], + results: Optional[ModuleResults] = None, ) -> None: super().__init__( file_path=file_path, @@ -322,14 +322,11 @@ def find_deleted(self): self.results = sorted(self.results, key=operator.itemgetter("first_isodate")) def check_indicators(self) -> None: - # Check for manipulated process records. - # TODO: Catching KeyError for live_isodate for retro-compatibility. - # This is not very good. - try: + # check_manipulated/find_deleted require "live_isodate" and + # "live_proc_id" keys which may be absent in older result formats. + if self.results and "live_isodate" in self.results[0]: self.check_manipulated() self.find_deleted() - except KeyError: - pass if not self.indicators: return diff --git a/tests/android_androidqf/test_settings.py b/tests/android_androidqf/test_settings.py index ef7386a71..ce14460ff 100644 --- a/tests/android_androidqf/test_settings.py +++ b/tests/android_androidqf/test_settings.py @@ -21,4 +21,5 @@ def test_parsing(self): run_module(m) assert len(m.results) == 1 assert "random" in m.results.keys() - assert len(m.alertstore.alerts) == 0 + assert len(m.alertstore.alerts) == 1 + assert "samsung_errorlog_agree" in m.alertstore.alerts[0].message