Wip channel import performance

Makefile

@@ -109,6 +109,7 @@ src/kolibri: clean
 	sed -i 's/if name.endswith(".py"):/if name.endswith(".py") or name.endswith(".pyc"):/g' src/kolibri/dist/django/db/migrations/loader.py
 	# Apply kolibri patches
 	patch -d src/ -p1 < patches/0001-Add-track-progress-information-to-channelimport.patch
+	patch -d src/ -p1 < patches/0001-WIP-Add-more-logging-in-channel_import.patch
 
 .PHONY: apps-bundle.zip
 apps-bundle.zip:

patches/0001-WIP-Add-more-logging-in-channel_import.patch

@@ -0,0 +1,50 @@
+From f812f6e11517337f7724a77ce745efe87cf53e4c Mon Sep 17 00:00:00 2001
+From: Dylan McCall <dylan@endlessos.org>
+Date: Wed, 16 Nov 2022 13:49:03 -0800
+Subject: [PATCH] WIP: Add more logging in channel_import
+
+---
+ kolibri/core/content/utils/channel_import.py | 8 +++++++-
+ 1 file changed, 7 insertions(+), 1 deletion(-)
+
+diff --git a/kolibri/core/content/utils/channel_import.py b/kolibri/core/content/utils/channel_import.py
+index f61e843482..23cc652b15 100644
+--- a/kolibri/core/content/utils/channel_import.py
++++ b/kolibri/core/content/utils/channel_import.py
+@@ -655,11 +655,15 @@ class ChannelImport(object):
+         # keep track of which model is currently being imported
+         self.current_model_being_imported = model
+
++        logger.info("ChannelImport table_import: {}".format(model.__name__))
++
+         if self.destination.engine.name == "postgresql":
+             result = self.postgres_table_import(model, row_mapper, table_mapper)
+         elif self.can_use_sqlite_attach_method(model, table_mapper):
++            logger.info("ChannelImport using raw_attached_sqlite_table_import")
+             result = self.raw_attached_sqlite_table_import(model, table_mapper)
+         else:
++            logger.warning("ChannelImport using sqlite_table_import")
+             result = self.sqlite_table_import(model, row_mapper, table_mapper)
+
+         self.current_model_being_imported = None
+@@ -845,6 +849,7 @@ class ChannelImport(object):
+     def try_attaching_sqlite_database(self):
+         # attach the external content database to our primary database so we can directly transfer records en masse
+         if self.destination.engine.name == "sqlite":
++            logger.info("ChannelImport attached sqlite db")
+             try:
+                 self.destination.execute(
+                     text(
+@@ -852,7 +857,8 @@ class ChannelImport(object):
+                     )
+                 )
+                 self._sqlite_db_attached = True
+-            except OperationalError:
++            except OperationalError as error:
++                logger.warning("ChannelImport failed to attach sqlite db: {}".format(error))
+                 # silently ignore if we were unable to attach the database; we'll just fall back to other methods
+                 pass
+
+-- 
+2.38.1
+

src/android_utils.py → src/kolibri_android/android_utils.py

@@ -22,7 +22,8 @@
 from jnius import cast
 from jnius import JavaException
 from jnius import jnius
-from runnable import Runnable
+
+from .runnable import Runnable
 
 
 logger = logging.getLogger(__name__)
@@ -41,6 +42,8 @@
 PackageManager = autoclass("android.content.pm.PackageManager")
 PendingIntent = autoclass("android.app.PendingIntent")
 PythonActivity = autoclass("org.kivy.android.PythonActivity")
+Secure = autoclass("android.provider.Settings$Secure")
+Settings = autoclass("android.provider.Settings")
 Timezone = autoclass("java.util.TimeZone")
 Toast = autoclass("android.widget.Toast")
 Uri = autoclass("android.net.Uri")
@@ -104,6 +107,10 @@ def get_timezone_name():
     return Timezone.getDefault().getDisplayName()
 
 
+def get_android_node_id():
+    return Secure.getString(get_activity().getContentResolver(), Secure.ANDROID_ID)
+
+
 def start_service(service_name, service_args=None):
     service_args = service_args or {}
     service = autoclass("org.endlessos.Key.Service{}".format(service_name.title()))
@@ -154,6 +161,8 @@ def is_app_installed(app_id):
 # TODO: check for storage availability, allow user to chose sd card or internal
 def get_home_folder():
     kolibri_home_file = get_activity().getExternalFilesDir(None)
+    if not kolibri_home_file:
+        return None
     return os.path.join(kolibri_home_file.toString(), "KOLIBRI_DATA")
 
 

src/android_whitenoise.py → src/kolibri_android/android_whitenoise.py

@@ -5,12 +5,6 @@
 from urllib.parse import urlparse
 from wsgiref.headers import Headers
 
-from android_utils import document_exists
-from android_utils import document_tree_join
-from android_utils import get_activity
-from android_utils import is_document_uri
-from android_utils import open_file
-from android_utils import stat_file
 from django.contrib.staticfiles import finders
 from django.utils._os import safe_join
 from jnius import autoclass
@@ -26,6 +20,13 @@
 from whitenoise.responders import Response
 from whitenoise.string_utils import decode_path_info
 
+from .android_utils import document_exists
+from .android_utils import document_tree_join
+from .android_utils import get_activity
+from .android_utils import is_document_uri
+from .android_utils import open_file
+from .android_utils import stat_file
+
 logger = logging.getLogger(__name__)
 
 Uri = autoclass("android.net.Uri")

src/kolibri_android/application.py

@@ -0,0 +1,42 @@
+import logging
+
+from android.activity import register_activity_lifecycle_callbacks
+
+
+class BaseActivity(object):
+    def __init__(self):
+        register_activity_lifecycle_callbacks(
+            onActivityStarted=self.on_activity_started,
+            onActivityPaused=self.on_activity_paused,
+            onActivityResumed=self.on_activity_resumed,
+            onActivityStopped=self.on_activity_stopped,
+            onActivityDestroyed=self.on_activity_destroyed,
+        )
+
+    def run(self):
+        raise NotImplementedError()
+
+    def start_service(self, service_name, service_args=None):
+        from .android_utils import start_service
+
+        start_service(service_name, service_args)
+
+    def on_activity_started(self, activity):
+        logging.info("onActivityStarted")
+
+    def on_activity_paused(self, activity):
+        logging.info("onActivityPaused")
+
+    def on_activity_resumed(self, activity):
+        logging.info("onActivityResumed")
+
+    def on_activity_stopped(self, activity):
+        logging.info("onActivityStopped")
+
+    def on_activity_destroyed(self, activity):
+        logging.info("onActivityDestroyed")
+
+
+class BaseService(object):
+    def run(self):
+        raise NotImplementedError()

src/kolibri_android/globals.py

@@ -0,0 +1,21 @@
+import logging
+import sys
+from pathlib import Path
+
+
+SCRIPT_PATH = Path(__file__).absolute().parent.parent
+
+
+def initialize():
+    from .android_utils import apply_android_workarounds
+
+    # initialize logging before loading any third-party modules, as they may cause logging to get configured.
+    logging.basicConfig(level=logging.DEBUG)
+    jnius_logger = logging.getLogger("jnius")
+    jnius_logger.setLevel(logging.INFO)
+
+    apply_android_workarounds()
+
+    sys.path.append(SCRIPT_PATH.as_posix())
+    sys.path.append(SCRIPT_PATH.joinpath("kolibri", "dist").as_posix())
+    sys.path.append(SCRIPT_PATH.joinpath("extra-packages").as_posix())

src/kolibri_app_settings.py → src/kolibri_android/kolibri_extra/settings.py

@@ -9,5 +9,5 @@
 
 
 MIDDLEWARE = list(MIDDLEWARE) + [  # noqa F405
-    "middleware.AlwaysAuthenticatedMiddleware"
+    "kolibri_android.kolibri_extra.middleware.AlwaysAuthenticatedMiddleware"
 ]