From 515336c507f575f9ade6b7bb883983728b825264 Mon Sep 17 00:00:00 2001 From: Dylan McCall Date: Mon, 14 Nov 2022 14:44:49 -0800 Subject: [PATCH 1/2] Organize application code into modules https://phabricator.endlessm.com/T34007 --- src/initialization.py | 80 ----- src/kolibri_android/__init__.py | 0 src/{ => kolibri_android}/android_utils.py | 11 +- .../android_whitenoise.py | 13 +- src/kolibri_android/application.py | 42 +++ src/kolibri_android/globals.py | 21 ++ src/kolibri_android/kolibri_extra/__init__.py | 0 .../kolibri_extra}/middleware.py | 0 .../kolibri_extra/settings.py} | 2 +- src/kolibri_android/kolibri_utils.py | 139 +++++++++ src/kolibri_android/main_activity/__init__.py | 0 src/kolibri_android/main_activity/__main__.py | 14 + src/kolibri_android/main_activity/activity.py | 185 +++++++++++ .../main_activity/kolibri_bus.py | 36 +++ .../remoteshell_service/__init__.py | 0 .../remoteshell_service/__main__.py | 14 + .../remoteshell_service/service.py | 125 ++++++++ src/{ => kolibri_android}/runnable.py | 0 .../workers_service/__init__.py | 0 .../workers_service/__main__.py | 14 + .../workers_service/service.py | 30 ++ src/main.py | 286 +----------------- src/remoteshell.py | 118 +------- src/workers.py | 26 +- 24 files changed, 644 insertions(+), 512 deletions(-) delete mode 100644 src/initialization.py create mode 100644 src/kolibri_android/__init__.py rename src/{ => kolibri_android}/android_utils.py (98%) rename src/{ => kolibri_android}/android_whitenoise.py (97%) create mode 100644 src/kolibri_android/application.py create mode 100644 src/kolibri_android/globals.py create mode 100644 src/kolibri_android/kolibri_extra/__init__.py rename src/{ => kolibri_android/kolibri_extra}/middleware.py (100%) rename src/{kolibri_app_settings.py => kolibri_android/kolibri_extra/settings.py} (80%) create mode 100644 src/kolibri_android/kolibri_utils.py create mode 100644 src/kolibri_android/main_activity/__init__.py create mode 100644 src/kolibri_android/main_activity/__main__.py create mode 100644 src/kolibri_android/main_activity/activity.py create mode 100644 src/kolibri_android/main_activity/kolibri_bus.py create mode 100644 src/kolibri_android/remoteshell_service/__init__.py create mode 100644 src/kolibri_android/remoteshell_service/__main__.py create mode 100644 src/kolibri_android/remoteshell_service/service.py rename src/{ => kolibri_android}/runnable.py (100%) create mode 100644 src/kolibri_android/workers_service/__init__.py create mode 100644 src/kolibri_android/workers_service/__main__.py create mode 100644 src/kolibri_android/workers_service/service.py diff --git a/src/initialization.py b/src/initialization.py deleted file mode 100644 index 44d82854..00000000 --- a/src/initialization.py +++ /dev/null @@ -1,80 +0,0 @@ -import logging -import os -import re -import sys - -from android_utils import apply_android_workarounds -from android_utils import get_activity -from android_utils import get_endless_key_uris -from android_utils import get_home_folder -from android_utils import get_signature_key_issuing_organization -from android_utils import get_timezone_name -from android_utils import get_version_name -from jnius import autoclass - -# initialize logging before loading any third-party modules, as they may cause logging to get configured. -logging.basicConfig(level=logging.DEBUG) -jnius_logger = logging.getLogger("jnius") -jnius_logger.setLevel(logging.INFO) - -apply_android_workarounds() - -script_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(script_dir) -sys.path.append(os.path.join(script_dir, "kolibri", "dist")) -sys.path.append(os.path.join(script_dir, "extra-packages")) - -from android_whitenoise import DynamicWhiteNoise # noqa: E402 -from android_whitenoise import monkeypatch_whitenoise # noqa: E402 - -monkeypatch_whitenoise() - - -def set_content_fallback_dirs_env(): - endless_key_uris = get_endless_key_uris() - if endless_key_uris is not None: - content_uri = DynamicWhiteNoise.encode_root(endless_key_uris["content"]) - logging.info("Setting KOLIBRI_CONTENT_FALLBACK_DIRS to %s", content_uri) - os.environ["KOLIBRI_CONTENT_FALLBACK_DIRS"] = content_uri - - -signing_org = get_signature_key_issuing_organization() -if signing_org == "Learning Equality": - runmode = "android-testing" -elif signing_org == "Android": - runmode = "android-debug" -elif signing_org == "Google Inc.": - runmode = "" # Play Store! -else: - runmode = "android-" + re.sub(r"[^a-z ]", "", signing_org.lower()).replace(" ", "-") - -os.environ["KOLIBRI_RUN_MODE"] = runmode -os.environ["KOLIBRI_PROJECT"] = "endless-key-android" - -os.environ["TZ"] = get_timezone_name() -os.environ["LC_ALL"] = "en_US.UTF-8" - -os.environ["KOLIBRI_HOME"] = get_home_folder() - -set_content_fallback_dirs_env() - -os.environ["KOLIBRI_APK_VERSION_NAME"] = get_version_name() -os.environ["DJANGO_SETTINGS_MODULE"] = "kolibri_app_settings" - -AUTOPROVISION_FILE = os.path.join(script_dir, "automatic_provision.json") -if os.path.exists(AUTOPROVISION_FILE): - os.environ["KOLIBRI_AUTOMATIC_PROVISION_FILE"] = AUTOPROVISION_FILE - -os.environ["KOLIBRI_CHERRYPY_THREAD_POOL"] = "2" - -os.environ["KOLIBRI_APPS_BUNDLE_PATH"] = os.path.join(script_dir, "apps-bundle", "apps") -os.environ["KOLIBRI_CONTENT_COLLECTIONS_PATH"] = os.path.join(script_dir, "collections") - -Secure = autoclass("android.provider.Settings$Secure") - -node_id = Secure.getString(get_activity().getContentResolver(), Secure.ANDROID_ID) - -# Don't set this if the retrieved id is falsy, too short, or a specific -# id that is known to be hardcoded in many devices. -if node_id and len(node_id) >= 16 and node_id != "9774d56d682e549c": - os.environ["MORANGO_NODE_ID"] = node_id diff --git a/src/kolibri_android/__init__.py b/src/kolibri_android/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/android_utils.py b/src/kolibri_android/android_utils.py similarity index 98% rename from src/android_utils.py rename to src/kolibri_android/android_utils.py index 4bc5b82f..17313fa8 100644 --- a/src/android_utils.py +++ b/src/kolibri_android/android_utils.py @@ -22,7 +22,8 @@ from jnius import cast from jnius import JavaException from jnius import jnius -from runnable import Runnable + +from .runnable import Runnable logger = logging.getLogger(__name__) @@ -41,6 +42,8 @@ PackageManager = autoclass("android.content.pm.PackageManager") PendingIntent = autoclass("android.app.PendingIntent") PythonActivity = autoclass("org.kivy.android.PythonActivity") +Secure = autoclass("android.provider.Settings$Secure") +Settings = autoclass("android.provider.Settings") Timezone = autoclass("java.util.TimeZone") Toast = autoclass("android.widget.Toast") Uri = autoclass("android.net.Uri") @@ -104,6 +107,10 @@ def get_timezone_name(): return Timezone.getDefault().getDisplayName() +def get_android_node_id(): + return Secure.getString(get_activity().getContentResolver(), Secure.ANDROID_ID) + + def start_service(service_name, service_args=None): service_args = service_args or {} service = autoclass("org.endlessos.Key.Service{}".format(service_name.title())) @@ -154,6 +161,8 @@ def is_app_installed(app_id): # TODO: check for storage availability, allow user to chose sd card or internal def get_home_folder(): kolibri_home_file = get_activity().getExternalFilesDir(None) + if not kolibri_home_file: + return None return os.path.join(kolibri_home_file.toString(), "KOLIBRI_DATA") diff --git a/src/android_whitenoise.py b/src/kolibri_android/android_whitenoise.py similarity index 97% rename from src/android_whitenoise.py rename to src/kolibri_android/android_whitenoise.py index d3838651..cd1d125a 100644 --- a/src/android_whitenoise.py +++ b/src/kolibri_android/android_whitenoise.py @@ -5,12 +5,6 @@ from urllib.parse import urlparse from wsgiref.headers import Headers -from android_utils import document_exists -from android_utils import document_tree_join -from android_utils import get_activity -from android_utils import is_document_uri -from android_utils import open_file -from android_utils import stat_file from django.contrib.staticfiles import finders from django.utils._os import safe_join from jnius import autoclass @@ -26,6 +20,13 @@ from whitenoise.responders import Response from whitenoise.string_utils import decode_path_info +from .android_utils import document_exists +from .android_utils import document_tree_join +from .android_utils import get_activity +from .android_utils import is_document_uri +from .android_utils import open_file +from .android_utils import stat_file + logger = logging.getLogger(__name__) Uri = autoclass("android.net.Uri") diff --git a/src/kolibri_android/application.py b/src/kolibri_android/application.py new file mode 100644 index 00000000..0f81331c --- /dev/null +++ b/src/kolibri_android/application.py @@ -0,0 +1,42 @@ +import logging + +from android.activity import register_activity_lifecycle_callbacks + + +class BaseActivity(object): + def __init__(self): + register_activity_lifecycle_callbacks( + onActivityStarted=self.on_activity_started, + onActivityPaused=self.on_activity_paused, + onActivityResumed=self.on_activity_resumed, + onActivityStopped=self.on_activity_stopped, + onActivityDestroyed=self.on_activity_destroyed, + ) + + def run(self): + raise NotImplementedError() + + def start_service(self, service_name, service_args=None): + from .android_utils import start_service + + start_service(service_name, service_args) + + def on_activity_started(self, activity): + logging.info("onActivityStarted") + + def on_activity_paused(self, activity): + logging.info("onActivityPaused") + + def on_activity_resumed(self, activity): + logging.info("onActivityResumed") + + def on_activity_stopped(self, activity): + logging.info("onActivityStopped") + + def on_activity_destroyed(self, activity): + logging.info("onActivityDestroyed") + + +class BaseService(object): + def run(self): + raise NotImplementedError() diff --git a/src/kolibri_android/globals.py b/src/kolibri_android/globals.py new file mode 100644 index 00000000..f662eabc --- /dev/null +++ b/src/kolibri_android/globals.py @@ -0,0 +1,21 @@ +import logging +import sys +from pathlib import Path + + +SCRIPT_PATH = Path(__file__).absolute().parent.parent + + +def initialize(): + from .android_utils import apply_android_workarounds + + # initialize logging before loading any third-party modules, as they may cause logging to get configured. + logging.basicConfig(level=logging.DEBUG) + jnius_logger = logging.getLogger("jnius") + jnius_logger.setLevel(logging.INFO) + + apply_android_workarounds() + + sys.path.append(SCRIPT_PATH.as_posix()) + sys.path.append(SCRIPT_PATH.joinpath("kolibri", "dist").as_posix()) + sys.path.append(SCRIPT_PATH.joinpath("extra-packages").as_posix()) diff --git a/src/kolibri_android/kolibri_extra/__init__.py b/src/kolibri_android/kolibri_extra/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/middleware.py b/src/kolibri_android/kolibri_extra/middleware.py similarity index 100% rename from src/middleware.py rename to src/kolibri_android/kolibri_extra/middleware.py diff --git a/src/kolibri_app_settings.py b/src/kolibri_android/kolibri_extra/settings.py similarity index 80% rename from src/kolibri_app_settings.py rename to src/kolibri_android/kolibri_extra/settings.py index b79ad914..593f56af 100644 --- a/src/kolibri_app_settings.py +++ b/src/kolibri_android/kolibri_extra/settings.py @@ -9,5 +9,5 @@ MIDDLEWARE = list(MIDDLEWARE) + [ # noqa F405 - "middleware.AlwaysAuthenticatedMiddleware" + "kolibri_android.kolibri_extra.middleware.AlwaysAuthenticatedMiddleware" ] diff --git a/src/kolibri_android/kolibri_utils.py b/src/kolibri_android/kolibri_utils.py new file mode 100644 index 00000000..5086e6b4 --- /dev/null +++ b/src/kolibri_android/kolibri_utils.py @@ -0,0 +1,139 @@ +import logging +import os +import re +from importlib.util import find_spec + +from .globals import SCRIPT_PATH + + +# These Kolibri plugins conflict with the plugins listed in REQUIRED_PLUGINS +# or OPTIONAL_PLUGINS: +DISABLED_PLUGINS = [ + "kolibri.plugins.learn", +] + +# These Kolibri plugins must be enabled for the application to function +# correctly: +REQUIRED_PLUGINS = [ + "kolibri.plugins.app", +] + +# These Kolibri plugins will be dynamically enabled if they are available: +OPTIONAL_PLUGINS = [ + "kolibri_explore_plugin", + "kolibri_zim_plugin", +] + + +def init_kolibri(content_fallback_dirs=None, **kwargs): + logging.info("Initializing Kolibri and running any upgrade routines") + + _monkeypatch_whitenoise() + _init_kolibri_env(content_fallback_dirs) + + from kolibri.utils.main import initialize + + for plugin_name in DISABLED_PLUGINS: + _disable_kolibri_plugin(plugin_name) + + for plugin_name in REQUIRED_PLUGINS: + _enable_kolibri_plugin(plugin_name) + + for plugin_name in OPTIONAL_PLUGINS: + _enable_kolibri_plugin(plugin_name, optional=True) + + initialize(**kwargs) + + +def _monkeypatch_whitenoise(): + from .android_whitenoise import monkeypatch_whitenoise + + monkeypatch_whitenoise() + + +def _init_kolibri_env(content_fallback_dirs=None): + from .android_utils import get_endless_key_uris + from .android_utils import get_android_node_id + from .android_utils import get_home_folder + from .android_utils import get_signature_key_issuing_organization + from .android_utils import get_timezone_name + from .android_utils import get_version_name + from .android_whitenoise import DynamicWhiteNoise + + signing_org = get_signature_key_issuing_organization() + if signing_org == "Learning Equality": + runmode = "android-testing" + elif signing_org == "Android": + runmode = "android-debug" + elif signing_org == "Google Inc.": + runmode = "" # Play Store! + else: + runmode = "android-" + re.sub(r"[^a-z ]", "", signing_org.lower()).replace( + " ", "-" + ) + os.environ["KOLIBRI_RUN_MODE"] = runmode + os.environ["KOLIBRI_PROJECT"] = "endless-key-android" + + os.environ["TZ"] = get_timezone_name() + os.environ["LC_ALL"] = "en_US.UTF-8" + + os.environ["KOLIBRI_HOME"] = get_home_folder() + + endless_key_uris = get_endless_key_uris() + + if endless_key_uris is not None: + content_fallback_dirs = DynamicWhiteNoise.encode_root( + endless_key_uris["content"] + ) + logging.info( + "Setting KOLIBRI_CONTENT_FALLBACK_DIRS to %s", content_fallback_dirs + ) + os.environ["KOLIBRI_CONTENT_FALLBACK_DIRS"] = content_fallback_dirs + + os.environ["KOLIBRI_APK_VERSION_NAME"] = get_version_name() + os.environ["DJANGO_SETTINGS_MODULE"] = "kolibri_android.kolibri_extra.settings" + + AUTOPROVISION_PATH = SCRIPT_PATH.joinpath("automatic_provision.json") + if AUTOPROVISION_PATH.is_file(): + os.environ["KOLIBRI_AUTOMATIC_PROVISION_FILE"] = AUTOPROVISION_PATH.as_posix() + + os.environ["KOLIBRI_CHERRYPY_THREAD_POOL"] = "2" + + os.environ["KOLIBRI_APPS_BUNDLE_PATH"] = SCRIPT_PATH.joinpath( + "apps-bundle", "apps" + ).as_posix() + os.environ["KOLIBRI_CONTENT_COLLECTIONS_PATH"] = SCRIPT_PATH.joinpath( + "collections" + ).as_posix() + + node_id = get_android_node_id() + + # Don't set this if the retrieved id is falsy, too short, or a specific + # id that is known to be hardcoded in many devices. + if node_id and len(node_id) >= 16 and node_id != "9774d56d682e549c": + os.environ["MORANGO_NODE_ID"] = node_id + + +def _disable_kolibri_plugin(plugin_name: str) -> bool: + from kolibri.main import disable_plugin + from kolibri.plugins import config as plugins_config + + if plugin_name in plugins_config.ACTIVE_PLUGINS: + logging.info(f"Disabling plugin {plugin_name}") + disable_plugin(plugin_name) + + return True + + +def _enable_kolibri_plugin(plugin_name: str, optional=False) -> bool: + from kolibri.main import enable_plugin + from kolibri.plugins import config as plugins_config + + if optional and not find_spec(plugin_name): + return False + + if plugin_name not in plugins_config.ACTIVE_PLUGINS: + logging.info(f"Enabling plugin {plugin_name}") + enable_plugin(plugin_name) + + return True diff --git a/src/kolibri_android/main_activity/__init__.py b/src/kolibri_android/main_activity/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/kolibri_android/main_activity/__main__.py b/src/kolibri_android/main_activity/__main__.py new file mode 100644 index 00000000..96997f69 --- /dev/null +++ b/src/kolibri_android/main_activity/__main__.py @@ -0,0 +1,14 @@ +from ..globals import initialize + +initialize() + + +def main(): + from .activity import MainActivity + + activity = MainActivity() + activity.run() + + +if __name__ == "__main__": + main() diff --git a/src/kolibri_android/main_activity/activity.py b/src/kolibri_android/main_activity/activity.py new file mode 100644 index 00000000..ab3f1bbe --- /dev/null +++ b/src/kolibri_android/main_activity/activity.py @@ -0,0 +1,185 @@ +import logging +import time + +from jnius import autoclass + +from ..android_utils import choose_endless_key_uris +from ..android_utils import get_endless_key_uris +from ..android_utils import has_any_external_storage_device +from ..android_utils import PermissionsCancelledError +from ..android_utils import PermissionsWrongFolderError +from ..android_utils import provision_endless_key_database +from ..android_utils import set_endless_key_uris +from ..android_utils import share_by_intent +from ..android_utils import StartupState +from ..android_utils import stat_file +from ..application import BaseActivity +from ..kolibri_utils import init_kolibri +from ..runnable import Runnable + + +PythonActivity = autoclass("org.kivy.android.PythonActivity") +FullScreen = autoclass("org.learningequality.FullScreen") + + +@Runnable +def configure_webview( + activity, load_runnable, load_with_usb_runnable, loading_ready_runnable +): + FullScreen.configureWebview( + activity, load_runnable, load_with_usb_runnable, loading_ready_runnable + ) + + +@Runnable +def load_url_in_webview(url): + PythonActivity.mWebView.loadUrl(url) + + +@Runnable +def evaluate_javascript(js_code): + PythonActivity.mWebView.evaluateJavascript(js_code, None) + + +def is_endless_key_reachable(): + """ + Check if the KOLIBRI_HOME db file is reachable. + + This only works after the user has granted permissions correctly. In other + case it always return False. + """ + + key_uris = get_endless_key_uris() + if not key_uris: + return False + try: + # Check if the USB is connected + stat_file(key_uris.get("db")) + evaluate_javascript("setHasUSB(true)") + return True + except FileNotFoundError: + evaluate_javascript("setHasUSB(false)") + return False + + +def wait_until_endless_key_is_reachable(): + repeat = not is_endless_key_reachable() + return repeat + + +def _build_kolibri_process_bus(application): + from .kolibri_bus import AppPlugin + from .kolibri_bus import KolibriAppProcessBus + + AppPlugin.register_share_file_interface(share_by_intent) + + kolibri_bus = KolibriAppProcessBus(enable_zeroconf=True) + AppPlugin(kolibri_bus, application).subscribe() + + return kolibri_bus + + +class MainActivity(BaseActivity): + TO_RUN_IN_MAIN = None + _last_has_any_check = None + _kolibri_bus = None + + def __init__(self): + super().__init__() + + configure_webview( + PythonActivity.mActivity, + Runnable(self._on_load), + Runnable(self._on_load_with_usb), + Runnable(self._on_loading_ready), + ) + + def run(self): + self.load_url("file:///android_asset/_load.html") + + while True: + if callable(self.TO_RUN_IN_MAIN): + repeat = self.TO_RUN_IN_MAIN() + if not repeat: + self.TO_RUN_IN_MAIN = None + # Wait a bit after each main function call + time.sleep(0.5) + time.sleep(0.05) + + def load_url(self, url): + load_url_in_webview(url) + + def start_kolibri(self): + # TODO: Wait until external storage is available + # + + init_kolibri(debug=True) + + self._kolibri_bus = _build_kolibri_process_bus(self) + + # start kolibri server + logging.info("Starting kolibri server.") + + self._kolibri_bus.run() + + def start_kolibri_with_usb(self): + key_uris = get_endless_key_uris() + + if key_uris is None: + try: + key_uris = choose_endless_key_uris() + except PermissionsWrongFolderError: + evaluate_javascript("show_wrong_folder()") + return + except PermissionsCancelledError: + evaluate_javascript("show_permissions_cancelled()") + return + + provision_endless_key_database(key_uris) + set_endless_key_uris(key_uris) + + self.start_kolibri() + + def _on_load(self): + self.TO_RUN_IN_MAIN = self.start_kolibri + + def _on_load_with_usb(self): + # TODO: Show grant access view + self.TO_RUN_IN_MAIN = self.start_kolibri_with_usb + + def _on_loading_ready(self): + startup_state = StartupState.get_current_state() + if startup_state == StartupState.FIRST_TIME: + logging.info("First time") + evaluate_javascript("show_welcome()") + + self.TO_RUN_IN_MAIN = self.check_has_any_external_storage_device + + elif startup_state == StartupState.USB_USER: + logging.info("Starting USB mode") + # If it's USB we should have the permissions here so it's not needed to + # ask again + + if not is_endless_key_reachable(): + evaluate_javascript("show_endless_key_required()") + self.TO_RUN_IN_MAIN = wait_until_endless_key_is_reachable + else: + self.TO_RUN_IN_MAIN = self.start_kolibri_with_usb + + else: + logging.info("Starting network mode") + self.TO_RUN_IN_MAIN = self.start_kolibri + + def check_has_any_external_storage_device(self): + # Memorize the last check to avoid evaluating javascript when not + # needed: + has_any = has_any_external_storage_device() + if has_any != self._last_has_any_check: + self._last_has_any_check = has_any + if has_any: + evaluate_javascript("setHasUSB(true)") + else: + evaluate_javascript("setHasUSB(false)") + # By returning True the main loop calls this function over and + # over again, until another function TO_RUN_IN_MAIN is set. + return True diff --git a/src/kolibri_android/main_activity/kolibri_bus.py b/src/kolibri_android/main_activity/kolibri_bus.py new file mode 100644 index 00000000..28c60f3d --- /dev/null +++ b/src/kolibri_android/main_activity/kolibri_bus.py @@ -0,0 +1,36 @@ +from kolibri.plugins.app.utils import interface +from kolibri.utils.server import BaseKolibriProcessBus +from kolibri.utils.server import KolibriServerPlugin +from kolibri.utils.server import ZeroConfPlugin +from kolibri.utils.server import ZipContentServerPlugin +from magicbus.plugins import SimplePlugin + + +class KolibriAppProcessBus(BaseKolibriProcessBus): + def __init__(self, *args, enable_zeroconf=True, **kwargs): + super(KolibriAppProcessBus, self).__init__(*args, **kwargs) + + if enable_zeroconf: + ZeroConfPlugin(self, self.port).subscribe() + + KolibriServerPlugin(self, self.port).subscribe() + + ZipContentServerPlugin(self, self.zip_port).subscribe() + + +class AppPlugin(SimplePlugin): + def __init__(self, bus, application): + self.application = application + self.bus = bus + self.bus.subscribe("SERVING", self.SERVING) + + @staticmethod + def register_share_file_interface(share_file): + interface.register(share_file=share_file) + + def SERVING(self, port): + start_url = ( + "http://127.0.0.1:{port}".format(port=port) + interface.get_initialize_url() + ) + self.application.load_url(start_url) + self.application.start_service("workers") diff --git a/src/kolibri_android/remoteshell_service/__init__.py b/src/kolibri_android/remoteshell_service/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/kolibri_android/remoteshell_service/__main__.py b/src/kolibri_android/remoteshell_service/__main__.py new file mode 100644 index 00000000..c92c724b --- /dev/null +++ b/src/kolibri_android/remoteshell_service/__main__.py @@ -0,0 +1,14 @@ +from kolibri_android.globals import initialize + +initialize() + + +def main(): + from .service import RemoteShellService + + service = RemoteShellService() + service.run() + + +if __name__ == "__main__": + main() diff --git a/src/kolibri_android/remoteshell_service/service.py b/src/kolibri_android/remoteshell_service/service.py new file mode 100644 index 00000000..a44d5667 --- /dev/null +++ b/src/kolibri_android/remoteshell_service/service.py @@ -0,0 +1,125 @@ +import logging +import os + +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from kolibri.main import initialize +from twisted.conch import manhole +from twisted.conch import manhole_ssh +from twisted.conch.ssh import keys +from twisted.cred import checkers +from twisted.cred import credentials +from twisted.cred import error +from twisted.cred import portal +from twisted.internet import defer +from twisted.internet import reactor +from zope.interface import implementer + +from ..application import BaseService + + +class RemoteShellService(BaseService): + def run(self): + logging.info("Starting remoteshell service") + + self._launch_remoteshell() + + def _launch_remoteshell(self, port=4242): + reactor.listenTCP(port, _get_manhole_factory(globals())) + reactor.run() + + +def get_key_pair(refresh=False): + + # calculate paths where we'll store our SSH server keys + KEYPATH = os.path.join(os.environ.get("KOLIBRI_HOME", "."), "ssh_host_key") + PUBKEYPATH = KEYPATH + ".pub" + + # check whether we already have keys there, and use them if so + if os.path.isfile(KEYPATH) and os.path.isfile(PUBKEYPATH) and not refresh: + with open(KEYPATH) as f, open(PUBKEYPATH) as pf: + return f.read(), pf.read() + + # otherwise, generate a new key pair and serialize it + key = rsa.generate_private_key( + backend=default_backend(), public_exponent=65537, key_size=2048 + ) + private_key = key.private_bytes( + serialization.Encoding.PEM, + serialization.PrivateFormat.TraditionalOpenSSL, + serialization.NoEncryption(), + ).decode() + public_key = ( + key.public_key() + .public_bytes( + serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH + ) + .decode() + ) + + # store the keys to disk for use again later + with open(KEYPATH, "w") as f, open(PUBKEYPATH, "w") as pf: + f.write(private_key) + pf.write(public_key) + + return private_key, public_key + + +@implementer(checkers.ICredentialsChecker) +class KolibriSuperAdminCredentialsChecker(object): + """ + Check that the device is unprovisioned, or the credentials are for a super admin, + or the password matches the temp password set over ADB. + """ + + credentialInterfaces = (credentials.IUsernamePassword,) + + def requestAvatarId(self, creds): + from kolibri.core.auth.models import FacilityUser + + # if a temporary password was set over ADB, allow login with it + TEMP_ADMIN_PASS_PATH = os.path.join( + os.environ.get("KOLIBRI_HOME", "."), "temp_admin_pass" + ) + if os.path.isfile(TEMP_ADMIN_PASS_PATH): + with open(TEMP_ADMIN_PASS_PATH) as f: + provided_password = creds.password.decode() + if provided_password and provided_password == f.read().strip(): + return creds.username + + # if there are no users yet (not yet provisioned), allow anon + if FacilityUser.objects.count() == 0: + return creds.username + + # check whether there are any super admins with these credentials + users = FacilityUser.objects.filter(username=creds.username) + for user in users: + if user.is_superuser and user.check_password(creds.password): + return creds.username + + # no matching users were found, so fail + return defer.fail(error.UnauthorizedLogin()) + + +def _get_manhole_factory(namespace): + + # ensure django has been set up so we can use the ORM etc in the shell + initialize(skip_update=True) + + # set up the twisted manhole with Kolibri-based authentication + def get_manhole(_): + return manhole.Manhole(namespace) + + realm = manhole_ssh.TerminalRealm() + realm.chainedProtocolFactory.protocolFactory = get_manhole + p = portal.Portal(realm) + p.registerChecker(KolibriSuperAdminCredentialsChecker()) + f = manhole_ssh.ConchFactory(p) + + # get the SSH server key pair to use + private_rsa, public_rsa = get_key_pair() + f.publicKeys[b"ssh-rsa"] = keys.Key.fromString(public_rsa) + f.privateKeys[b"ssh-rsa"] = keys.Key.fromString(private_rsa) + + return f diff --git a/src/runnable.py b/src/kolibri_android/runnable.py similarity index 100% rename from src/runnable.py rename to src/kolibri_android/runnable.py diff --git a/src/kolibri_android/workers_service/__init__.py b/src/kolibri_android/workers_service/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/src/kolibri_android/workers_service/__main__.py b/src/kolibri_android/workers_service/__main__.py new file mode 100644 index 00000000..00419d48 --- /dev/null +++ b/src/kolibri_android/workers_service/__main__.py @@ -0,0 +1,14 @@ +from kolibri_android.globals import initialize + +initialize() + + +def main(): + from .service import WorkersService + + service = WorkersService() + service.run() + + +if __name__ == "__main__": + main() diff --git a/src/kolibri_android/workers_service/service.py b/src/kolibri_android/workers_service/service.py new file mode 100644 index 00000000..2e6f208e --- /dev/null +++ b/src/kolibri_android/workers_service/service.py @@ -0,0 +1,30 @@ +import logging + +from ..android_utils import make_service_foreground +from ..application import BaseService +from ..kolibri_utils import init_kolibri + + +class WorkersService(BaseService): + def run(self): + logging.info("Starting Kolibri task workers") + + # ensure the service stays running by "foregrounding" it with a persistent notification + make_service_foreground("Kolibri service", "Running tasks.") + + init_kolibri(debug=True, skip_update=True) + self._run_kolibri_workers() + + def _run_kolibri_workers(self): + from kolibri.core.tasks.main import initialize_workers + from kolibri.core.analytics.tasks import schedule_ping + from kolibri.core.deviceadmin.tasks import schedule_vacuum + + schedule_ping() + + schedule_vacuum() + + # Initialize the iceqube engine to handle queued tasks + worker = initialize_workers() + # Join the job checker thread to loop forever + worker.job_checker.join() diff --git a/src/main.py b/src/main.py index fd1f1641..8dc74560 100644 --- a/src/main.py +++ b/src/main.py @@ -1,285 +1,3 @@ -import importlib -import logging -import time +from kolibri_android.main_activity.__main__ import main -import initialization # noqa: F401 keep this first, to ensure we're set up for other imports -from android.activity import register_activity_lifecycle_callbacks -from android_utils import choose_endless_key_uris -from android_utils import get_endless_key_uris -from android_utils import has_any_external_storage_device -from android_utils import PermissionsCancelledError -from android_utils import PermissionsWrongFolderError -from android_utils import provision_endless_key_database -from android_utils import set_endless_key_uris -from android_utils import share_by_intent -from android_utils import start_service -from android_utils import StartupState -from android_utils import stat_file -from jnius import autoclass -from kolibri.main import disable_plugin -from kolibri.main import enable_plugin -from kolibri.plugins import config as plugins_config -from kolibri.plugins.app.utils import interface -from kolibri.utils.cli import initialize -from kolibri.utils.server import BaseKolibriProcessBus -from kolibri.utils.server import KolibriServerPlugin -from kolibri.utils.server import ZeroConfPlugin -from kolibri.utils.server import ZipContentServerPlugin -from magicbus.plugins import SimplePlugin -from runnable import Runnable - -# These Kolibri plugins conflict with the plugins listed in REQUIRED_PLUGINS -# or OPTIONAL_PLUGINS: -DISABLED_PLUGINS = [ - "kolibri.plugins.learn", -] - -# These Kolibri plugins must be enabled for the application to function -# correctly: -REQUIRED_PLUGINS = [ - "kolibri.plugins.app", -] - -# These Kolibri plugins will be dynamically enabled if they are available: -OPTIONAL_PLUGINS = [ - "kolibri_explore_plugin", - "kolibri_zim_plugin", -] - - -TO_RUN_IN_MAIN = None -_last_has_any_check = None - - -def _disable_kolibri_plugin(plugin_name: str) -> bool: - if plugin_name in plugins_config.ACTIVE_PLUGINS: - logging.info(f"Disabling plugin {plugin_name}") - disable_plugin(plugin_name) - - return True - - -def _enable_kolibri_plugin(plugin_name: str, optional=False) -> bool: - if optional and not importlib.util.find_spec(plugin_name): - return False - - if plugin_name not in plugins_config.ACTIVE_PLUGINS: - logging.info(f"Enabling plugin {plugin_name}") - enable_plugin(plugin_name) - - return True - - -def load(): - global TO_RUN_IN_MAIN - TO_RUN_IN_MAIN = start_kolibri - - -def load_with_usb(): - global TO_RUN_IN_MAIN - # TODO: Show grant access view - TO_RUN_IN_MAIN = start_kolibri_with_usb - - -def evaluate_javascript(js_code): - PythonActivity.mWebView.evaluateJavascript(js_code, None) - - -evaluate_javascript = Runnable(evaluate_javascript) - - -def check_has_any_external_storage_device(): - # Memorize the last check to avoid evaluating javascript when not - # needed: - global _last_has_any_check - has_any = has_any_external_storage_device() - if has_any != _last_has_any_check: - _last_has_any_check = has_any - if has_any: - evaluate_javascript("setHasUSB(true)") - else: - evaluate_javascript("setHasUSB(false)") - # By returning True the main loop calls this function over and - # over again, until another function TO_RUN_IN_MAIN is set. - return True - - -def is_endless_key_reachable(): - """ - Check if the KOLIBRI_HOME db file is reachable. - - This only works after the user has granted permissions correctly. In other - case it always return False. - """ - - key_uris = get_endless_key_uris() - if not key_uris: - return False - try: - # Check if the USB is connected - stat_file(key_uris.get("db")) - evaluate_javascript("setHasUSB(true)") - return True - except FileNotFoundError: - evaluate_javascript("setHasUSB(false)") - return False - - -def wait_until_endless_key_is_reachable(): - repeat = not is_endless_key_reachable() - return repeat - - -def on_loading_ready(): - global TO_RUN_IN_MAIN - - startup_state = StartupState.get_current_state() - if startup_state == StartupState.FIRST_TIME: - logging.info("First time") - evaluate_javascript("show_welcome()") - - TO_RUN_IN_MAIN = check_has_any_external_storage_device - - elif startup_state == StartupState.USB_USER: - logging.info("Starting USB mode") - # If it's USB we should have the permissions here so it's not needed to - # ask again - - if not is_endless_key_reachable(): - evaluate_javascript("show_endless_key_required()") - TO_RUN_IN_MAIN = wait_until_endless_key_is_reachable - else: - TO_RUN_IN_MAIN = start_kolibri_with_usb - - else: - logging.info("Starting network mode") - TO_RUN_IN_MAIN = start_kolibri - - -def on_activity_started(activity): - logging.info("onActivityStarted") - - -def on_activity_paused(activity): - logging.info("onActivityPaused") - - -def on_activity_resumed(activity): - logging.info("onActivityResumed") - - -def on_activity_stopped(activity): - logging.info("onActivityStopped") - - -def on_activity_destroyed(activity): - logging.info("onActivityDestroyed") - - -register_activity_lifecycle_callbacks( - onActivityStarted=on_activity_started, - onActivityPaused=on_activity_paused, - onActivityResumed=on_activity_resumed, - onActivityStopped=on_activity_stopped, - onActivityDestroyed=on_activity_destroyed, -) - -PythonActivity = autoclass("org.kivy.android.PythonActivity") - -FullScreen = autoclass("org.learningequality.FullScreen") -configureWebview = Runnable(FullScreen.configureWebview) - -configureWebview( - PythonActivity.mActivity, - Runnable(load), - Runnable(load_with_usb), - Runnable(on_loading_ready), -) - -loadUrl = Runnable(PythonActivity.mWebView.loadUrl) - - -class AppPlugin(SimplePlugin): - def __init__(self, bus): - self.bus = bus - self.bus.subscribe("SERVING", self.SERVING) - - def SERVING(self, port): - start_url = ( - "http://127.0.0.1:{port}".format(port=port) + interface.get_initialize_url() - ) - loadUrl(start_url) - start_service("workers") - - -logging.info("Initializing Kolibri and running any upgrade routines") - -loadUrl("file:///android_asset/_load.html") - -for plugin_name in DISABLED_PLUGINS: - _disable_kolibri_plugin(plugin_name) - -for plugin_name in REQUIRED_PLUGINS: - _enable_kolibri_plugin(plugin_name) - -for plugin_name in OPTIONAL_PLUGINS: - _enable_kolibri_plugin(plugin_name, optional=True) - - -def start_kolibri_with_usb(): - key_uris = get_endless_key_uris() - - if key_uris is None: - try: - key_uris = choose_endless_key_uris() - except PermissionsWrongFolderError: - evaluate_javascript("show_wrong_folder()") - return - except PermissionsCancelledError: - evaluate_javascript("show_permissions_cancelled()") - return - - provision_endless_key_database(key_uris) - set_endless_key_uris(key_uris) - initialization.set_content_fallback_dirs_env() - start_kolibri() - - -def start_kolibri(): - # we need to initialize Kolibri to allow us to access the app key - initialize(debug=True) - - interface.register(share_file=share_by_intent) - - # start kolibri server - logging.info("Starting kolibri server.") - - kolibri_bus = BaseKolibriProcessBus() - # Setup zeroconf plugin - zeroconf_plugin = ZeroConfPlugin(kolibri_bus, kolibri_bus.port) - zeroconf_plugin.subscribe() - kolibri_server = KolibriServerPlugin( - kolibri_bus, - kolibri_bus.port, - ) - - alt_port_server = ZipContentServerPlugin( - kolibri_bus, - kolibri_bus.zip_port, - ) - # Subscribe these servers - kolibri_server.subscribe() - alt_port_server.subscribe() - app_plugin = AppPlugin(kolibri_bus) - app_plugin.subscribe() - kolibri_bus.run() - - -while True: - if callable(TO_RUN_IN_MAIN): - repeat = TO_RUN_IN_MAIN() - if not repeat: - TO_RUN_IN_MAIN = None - # Wait a bit after each main function call - time.sleep(0.5) - time.sleep(0.05) +main() diff --git a/src/remoteshell.py b/src/remoteshell.py index 8dd3142f..99b7df71 100644 --- a/src/remoteshell.py +++ b/src/remoteshell.py @@ -1,117 +1,3 @@ -import os +from kolibri_android.remoteshell_service.__main__ import main -import initialization # noqa: F401 keep this first, to ensure we're set up for other imports -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from kolibri.main import initialize -from twisted.conch import manhole -from twisted.conch import manhole_ssh -from twisted.conch.ssh import keys -from twisted.cred import checkers -from twisted.cred import credentials -from twisted.cred import error -from twisted.cred import portal -from twisted.internet import defer -from twisted.internet import reactor -from zope.interface import implementer - - -def get_key_pair(refresh=False): - - # calculate paths where we'll store our SSH server keys - KEYPATH = os.path.join(os.environ.get("KOLIBRI_HOME", "."), "ssh_host_key") - PUBKEYPATH = KEYPATH + ".pub" - - # check whether we already have keys there, and use them if so - if os.path.isfile(KEYPATH) and os.path.isfile(PUBKEYPATH) and not refresh: - with open(KEYPATH) as f, open(PUBKEYPATH) as pf: - return f.read(), pf.read() - - # otherwise, generate a new key pair and serialize it - key = rsa.generate_private_key( - backend=default_backend(), public_exponent=65537, key_size=2048 - ) - private_key = key.private_bytes( - serialization.Encoding.PEM, - serialization.PrivateFormat.TraditionalOpenSSL, - serialization.NoEncryption(), - ).decode() - public_key = ( - key.public_key() - .public_bytes( - serialization.Encoding.OpenSSH, serialization.PublicFormat.OpenSSH - ) - .decode() - ) - - # store the keys to disk for use again later - with open(KEYPATH, "w") as f, open(PUBKEYPATH, "w") as pf: - f.write(private_key) - pf.write(public_key) - - return private_key, public_key - - -@implementer(checkers.ICredentialsChecker) -class KolibriSuperAdminCredentialsChecker(object): - """ - Check that the device is unprovisioned, or the credentials are for a super admin, - or the password matches the temp password set over ADB. - """ - - credentialInterfaces = (credentials.IUsernamePassword,) - - def requestAvatarId(self, creds): - from kolibri.core.auth.models import FacilityUser - - # if a temporary password was set over ADB, allow login with it - TEMP_ADMIN_PASS_PATH = os.path.join( - os.environ.get("KOLIBRI_HOME", "."), "temp_admin_pass" - ) - if os.path.isfile(TEMP_ADMIN_PASS_PATH): - with open(TEMP_ADMIN_PASS_PATH) as f: - provided_password = creds.password.decode() - if provided_password and provided_password == f.read().strip(): - return creds.username - - # if there are no users yet (not yet provisioned), allow anon - if FacilityUser.objects.count() == 0: - return creds.username - - # check whether there are any super admins with these credentials - users = FacilityUser.objects.filter(username=creds.username) - for user in users: - if user.is_superuser and user.check_password(creds.password): - return creds.username - - # no matching users were found, so fail - return defer.fail(error.UnauthorizedLogin()) - - -def _get_manhole_factory(namespace): - - # ensure django has been set up so we can use the ORM etc in the shell - initialize(skip_update=True) - - # set up the twisted manhole with Kolibri-based authentication - def get_manhole(_): - return manhole.Manhole(namespace) - - realm = manhole_ssh.TerminalRealm() - realm.chainedProtocolFactory.protocolFactory = get_manhole - p = portal.Portal(realm) - p.registerChecker(KolibriSuperAdminCredentialsChecker()) - f = manhole_ssh.ConchFactory(p) - - # get the SSH server key pair to use - private_rsa, public_rsa = get_key_pair() - f.publicKeys[b"ssh-rsa"] = keys.Key.fromString(public_rsa) - f.privateKeys[b"ssh-rsa"] = keys.Key.fromString(private_rsa) - - return f - - -def launch_remoteshell(port=4242): - reactor.listenTCP(port, _get_manhole_factory(globals())) - reactor.run() +main() diff --git a/src/workers.py b/src/workers.py index 5b0be6ba..e998aade 100644 --- a/src/workers.py +++ b/src/workers.py @@ -1,25 +1,3 @@ -import logging +from kolibri_android.workers_service.__main__ import main -import initialization # noqa: F401 keep this first, to ensure we're set up for other imports -from android_utils import make_service_foreground -from kolibri.main import initialize - -logging.info("Starting Kolibri task workers") - -# ensure the service stays running by "foregrounding" it with a persistent notification -make_service_foreground("Kolibri service", "Running tasks.") - -initialize(debug=True, skip_update=True) - -from kolibri.core.tasks.main import initialize_workers # noqa: E402 -from kolibri.core.analytics.tasks import schedule_ping # noqa: E402 -from kolibri.core.deviceadmin.tasks import schedule_vacuum # noqa: E402 - -schedule_ping() - -schedule_vacuum() - -# Initialize the iceqube engine to handle queued tasks -worker = initialize_workers() -# Join the job checker thread to loop forever -worker.job_checker.join() +main() From 981dda8512d1eebb7926b717510bdc6066c65d5d Mon Sep 17 00:00:00 2001 From: Dylan McCall Date: Wed, 16 Nov 2022 13:50:48 -0800 Subject: [PATCH 2/2] WIP: Add more logging for channel_import --- Makefile | 1 + ...P-Add-more-logging-in-channel_import.patch | 50 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 patches/0001-WIP-Add-more-logging-in-channel_import.patch diff --git a/Makefile b/Makefile index e57e5804..7f17c883 100644 --- a/Makefile +++ b/Makefile @@ -109,6 +109,7 @@ src/kolibri: clean sed -i 's/if name.endswith(".py"):/if name.endswith(".py") or name.endswith(".pyc"):/g' src/kolibri/dist/django/db/migrations/loader.py # Apply kolibri patches patch -d src/ -p1 < patches/0001-Add-track-progress-information-to-channelimport.patch + patch -d src/ -p1 < patches/0001-WIP-Add-more-logging-in-channel_import.patch .PHONY: apps-bundle.zip apps-bundle.zip: diff --git a/patches/0001-WIP-Add-more-logging-in-channel_import.patch b/patches/0001-WIP-Add-more-logging-in-channel_import.patch new file mode 100644 index 00000000..5f853680 --- /dev/null +++ b/patches/0001-WIP-Add-more-logging-in-channel_import.patch @@ -0,0 +1,50 @@ +From f812f6e11517337f7724a77ce745efe87cf53e4c Mon Sep 17 00:00:00 2001 +From: Dylan McCall +Date: Wed, 16 Nov 2022 13:49:03 -0800 +Subject: [PATCH] WIP: Add more logging in channel_import + +--- + kolibri/core/content/utils/channel_import.py | 8 +++++++- + 1 file changed, 7 insertions(+), 1 deletion(-) + +diff --git a/kolibri/core/content/utils/channel_import.py b/kolibri/core/content/utils/channel_import.py +index f61e843482..23cc652b15 100644 +--- a/kolibri/core/content/utils/channel_import.py ++++ b/kolibri/core/content/utils/channel_import.py +@@ -655,11 +655,15 @@ class ChannelImport(object): + # keep track of which model is currently being imported + self.current_model_being_imported = model + ++ logger.info("ChannelImport table_import: {}".format(model.__name__)) ++ + if self.destination.engine.name == "postgresql": + result = self.postgres_table_import(model, row_mapper, table_mapper) + elif self.can_use_sqlite_attach_method(model, table_mapper): ++ logger.info("ChannelImport using raw_attached_sqlite_table_import") + result = self.raw_attached_sqlite_table_import(model, table_mapper) + else: ++ logger.warning("ChannelImport using sqlite_table_import") + result = self.sqlite_table_import(model, row_mapper, table_mapper) + + self.current_model_being_imported = None +@@ -845,6 +849,7 @@ class ChannelImport(object): + def try_attaching_sqlite_database(self): + # attach the external content database to our primary database so we can directly transfer records en masse + if self.destination.engine.name == "sqlite": ++ logger.info("ChannelImport attached sqlite db") + try: + self.destination.execute( + text( +@@ -852,7 +857,8 @@ class ChannelImport(object): + ) + ) + self._sqlite_db_attached = True +- except OperationalError: ++ except OperationalError as error: ++ logger.warning("ChannelImport failed to attach sqlite db: {}".format(error)) + # silently ignore if we were unable to attach the database; we'll just fall back to other methods + pass + +-- +2.38.1 +