Shared scene libraries (#202)

* Add shared scene libraries

* Remove legacy driver build mode naming

Author: mariusandra

backend/app/api/frames.py

@@ -104,7 +104,7 @@
 from app.utils.tls import generate_frame_tls_material, parse_certificate_not_valid_after
 from app.utils.ssh_authorized_keys import _install_authorized_keys, resolve_authorized_keys_update
 from app.tasks.binary_builder import FrameBinaryBuilder
-from app.codegen.drivers_nim import frame_driver_build_mode
+from app.codegen.drivers_nim import frame_compilation_mode
 from app.utils.local_exec import exec_local_command
 from app.utils.jwt_tokens import validate_scoped_token
 from . import api_with_auth, api_no_auth
@@ -770,7 +770,7 @@ async def api_frame_local_build_zip(                 # noqa: D401
         source_dir = deployer.create_local_source_folder(tmp)
 
         # Apply all frame‑specific code generation (scenes, drivers, …)
-        await deployer.make_local_modifications(source_dir, driver_build_mode=frame_driver_build_mode(frame))
+        await deployer.make_local_modifications(source_dir, compilation_mode=frame_compilation_mode(frame))
         await copy_custom_fonts_to_local_source_folder(db, source_dir)
 
         # Package → .zip
@@ -833,13 +833,13 @@ async def api_frame_local_c_source_zip(
             )
 
         source_dir = deployer.create_local_source_folder(tmp)
-        driver_build_mode = frame_driver_build_mode(frame)
-        await deployer.make_local_modifications(source_dir, driver_build_mode=driver_build_mode)
+        compilation_mode = frame_compilation_mode(frame)
+        await deployer.make_local_modifications(source_dir, compilation_mode=compilation_mode)
         await copy_custom_fonts_to_local_source_folder(db, source_dir)
 
         build_dir = os.path.join(tmp, f"build_{deployer.build_id}")
         os.makedirs(build_dir, exist_ok=True)
-        await deployer.create_local_build_archive(build_dir, source_dir, arch, driver_build_mode=driver_build_mode)
+        await deployer.create_local_build_archive(build_dir, source_dir, arch, compilation_mode=compilation_mode)
 
         zip_path = os.path.join(tmp, f"frameos_{deployer.build_id}_c_source.zip")
         with zipfile.ZipFile(
@@ -922,6 +922,16 @@ async def api_frame_local_binary_zip(
                         detail=f"Shared driver library missing after build: {driver_library_path}",
                     )
                 shutil.copy2(driver_library_path, os.path.join(driver_dir, os.path.basename(driver_library_path)))
+        if build_result.scene_library_paths:
+            scene_dir = os.path.join(dist_dir, "scenes")
+            os.makedirs(scene_dir, exist_ok=True)
+            for scene_library_path in build_result.scene_library_paths:
+                if not os.path.isfile(scene_library_path):
+                    raise HTTPException(
+                        status_code=HTTPStatus.INTERNAL_SERVER_ERROR,
+                        detail=f"Shared scene library missing after build: {scene_library_path}",
+                    )
+                shutil.copy2(scene_library_path, os.path.join(scene_dir, os.path.basename(scene_library_path)))
 
         zip_path = os.path.join(tmp, f"frameos_{deployer.build_id}_binary.zip")
         with zipfile.ZipFile(

backend/app/codegen/drivers_nim.py

@@ -5,33 +5,33 @@
 
 from app.drivers.drivers import Driver
 
-DRIVER_BUILD_MODE_STATIC = "static"
-DRIVER_BUILD_MODE_SHARED = "shared"
-DRIVER_BUILD_MODE_PRECOMPILED = "precompiled"
-DEFAULT_DRIVER_BUILD_MODE = DRIVER_BUILD_MODE_STATIC
-VALID_DRIVER_BUILD_MODES = {
-    DRIVER_BUILD_MODE_STATIC,
-    DRIVER_BUILD_MODE_SHARED,
-    DRIVER_BUILD_MODE_PRECOMPILED,
+COMPILATION_MODE_STATIC = "static"
+COMPILATION_MODE_SHARED = "shared"
+COMPILATION_MODE_PRECOMPILED = "precompiled"
+DEFAULT_COMPILATION_MODE = COMPILATION_MODE_STATIC
+VALID_COMPILATION_MODES = {
+    COMPILATION_MODE_STATIC,
+    COMPILATION_MODE_SHARED,
+    COMPILATION_MODE_PRECOMPILED,
 }
 
 
-def normalize_driver_build_mode(value: str | None) -> str:
-    normalized = (value or DEFAULT_DRIVER_BUILD_MODE).strip().lower()
-    if normalized not in VALID_DRIVER_BUILD_MODES:
-        return DEFAULT_DRIVER_BUILD_MODE
+def normalize_compilation_mode(value: str | None) -> str:
+    normalized = (value or DEFAULT_COMPILATION_MODE).strip().lower()
+    if normalized not in VALID_COMPILATION_MODES:
+        return DEFAULT_COMPILATION_MODE
     return normalized
 
 
-def frame_driver_build_mode(frame) -> str:
+def frame_compilation_mode(frame) -> str:
     rpios_settings = getattr(frame, "rpios", None) or {}
-    return normalize_driver_build_mode(rpios_settings.get("driverBuildMode"))
+    return normalize_compilation_mode(rpios_settings.get("compilationMode"))
 
 
-def driver_build_mode_uses_shared_libraries(value: str | None) -> bool:
-    return normalize_driver_build_mode(value) in {
-        DRIVER_BUILD_MODE_SHARED,
-        DRIVER_BUILD_MODE_PRECOMPILED,
+def compilation_mode_uses_shared_libraries(value: str | None) -> bool:
+    return normalize_compilation_mode(value) in {
+        COMPILATION_MODE_SHARED,
+        COMPILATION_MODE_PRECOMPILED,
     }
 
 
@@ -241,9 +241,9 @@ def driver_library_context_helpers_nim() -> str:
 
 def write_drivers_nim(
     drivers: dict[str, Driver],
-    driver_build_mode: str = DEFAULT_DRIVER_BUILD_MODE,
+    compilation_mode: str = DEFAULT_COMPILATION_MODE,
 ) -> str:
-    if driver_build_mode_uses_shared_libraries(driver_build_mode):
+    if compilation_mode_uses_shared_libraries(compilation_mode):
         return write_shared_drivers_nim(drivers)
     return write_static_drivers_nim(drivers)
 

backend/app/codegen/scene_nim.py

@@ -7,6 +7,10 @@
 
 from app.models.frame import Frame
 from app.models.apps import get_local_frame_apps, get_local_app_path, get_scene_app_id
+from app.codegen.drivers_nim import (
+    DEFAULT_COMPILATION_MODE,
+    compilation_mode_uses_shared_libraries,
+)
 from app.codegen.utils import sanitize_nim_string, natural_keys
 from app.utils.js_apps import find_js_app_source_key
 
@@ -36,6 +40,50 @@ def write_scene_nim(frame: Frame, scene: dict) -> str:
     return SceneWriter(frame, scene).write_scene_nim()
 
 
+def compiled_frame_scenes(frame: Frame) -> list[dict]:
+    scenes = getattr(frame, "scenes", None) or []
+    compiled_scenes = []
+    for scene in scenes:
+        if not isinstance(scene, dict):
+            continue
+        settings = scene.get("settings") or {}
+        if settings.get("execution", "compiled") != "interpreted":
+            compiled_scenes.append(scene)
+    return compiled_scenes
+
+
+def scene_registry_id(scene: dict) -> str:
+    return re.sub(r"[^a-zA-Z0-9\-\_]", "_", scene.get("id", "default"))
+
+
+def scene_module_suffix(scene: dict) -> str:
+    return re.sub(r"\W+", "", scene_registry_id(scene))
+
+
+def scene_module_filename(scene: dict) -> str:
+    return f"scene_{scene_module_suffix(scene)}.nim"
+
+
+def scene_library_filename(scene: dict) -> str:
+    return f"scene_{scene_module_suffix(scene)}.so"
+
+
+def write_scene_library_nim(scene: dict) -> str:
+    scene_module = f"scene_{scene_module_suffix(scene)}"
+    return f"""# This file is autogenerated
+
+import scenes/{scene_module} as sceneModule
+import frameos/channels
+import frameos/driver_abi
+
+proc frameos_scene_init*(logHook: HostLogProc, sendEventHook: HostSendEventProc) {{.cdecl, exportc, dynlib.}} =
+  setSharedHostCallbacks(logHook, sendEventHook)
+
+proc frameos_scene_export*(): pointer {{.cdecl, exportc, dynlib.}} =
+  result = cast[pointer](sceneModule.exportedScene)
+"""
+
+
 def field_type_to_nim_type(field_type: str, required: bool = True) -> str:
     match field_type:
         case 'select':
@@ -1633,47 +1681,46 @@ def wrap_with_cache(self, node_id: str, value_list: list[str], data: dict):
         return value_list
 
 
-def write_scenes_nim(frame: Frame) -> str:
+def _scene_registry_rows(frame: Frame) -> tuple[list[str], list[str], dict | None]:
     rows = []
-    imports = []
     sceneOptionTuples = []
     default_scene = None
-    for scene in frame.scenes:
-        execution = scene.get("settings", {}).get("execution", "compiled")
-        if execution == "interpreted":
-            continue
+    for scene in compiled_frame_scenes(frame):
         if scene.get("default", False):
             default_scene = scene
 
-        scene_id = scene.get("id", "default")
-        scene_id = re.sub(r"[^a-zA-Z0-9\-\_]", "_", scene_id)
-        scene_id_import = re.sub(r"\W+", "", scene_id)
-        imports.append(
-            f"import scenes/scene_{scene_id_import} as scene_{scene_id_import}"
-        )
+        scene_id = scene_registry_id(scene)
         rows.append(
-            f'  result["{scene_id}".SceneId] = scene_{scene_id_import}.exportedScene'
+            f'  result["{scene_id}".SceneId] = scene_{scene_module_suffix(scene)}.exportedScene'
         )
         sceneOptionTuples.append(
-            f"  (\"{scene_id}\".SceneId, \"{scene.get('name', 'Default')}\"),"
+            f'  ("{scene_id}".SceneId, "{sanitize_nim_string(scene.get("name", "Default"))}"),'
         )
+    return rows, sceneOptionTuples, default_scene
+
 
+def _default_scene_line(default_scene: dict | None) -> str:
     default_scene_id = (
         default_scene.get("id", None) if default_scene is not None else None
     )
     if default_scene_id is None:
-        default_scene_line = "let defaultSceneId* = none(SceneId)"
-    else:
-        default_scene_id = re.sub(r"[^a-zA-Z0-9\-\_]", "_", default_scene_id)
-        default_scene_line = f'let defaultSceneId* = some("{default_scene_id}".SceneId)'
+        return "let defaultSceneId* = none(SceneId)"
+    return f'let defaultSceneId* = some("{scene_registry_id(default_scene)}".SceneId)'
+
 
+def write_static_scenes_nim(frame: Frame) -> str:
+    rows, sceneOptionTuples, default_scene = _scene_registry_rows(frame)
+    imports = [
+        f"import scenes/scene_{scene_module_suffix(scene)} as scene_{scene_module_suffix(scene)}"
+        for scene in compiled_frame_scenes(frame)
+    ]
     newline = "\n"
     scenes_source = f"""
 import frameos/types
 import tables, options
 {newline.join(sorted(imports))}
 
-{default_scene_line}
+{_default_scene_line(default_scene)}
 
 const sceneOptions*: array[{len(sceneOptionTuples)}, tuple[id: SceneId, name: string]] = [
 {newline.join(sorted(sceneOptionTuples))}
@@ -1685,3 +1732,117 @@ def write_scenes_nim(frame: Frame) -> str:
 """
 
     return scenes_source
+
+
+def write_shared_scenes_nim(frame: Frame) -> str:
+    compiled_scenes = compiled_frame_scenes(frame)
+    sceneOptionTuples = [
+        f'  ("{scene_registry_id(scene)}".SceneId, "{sanitize_nim_string(scene.get("name", "Default"))}"),'
+        for scene in compiled_scenes
+    ]
+    default_scene = next((scene for scene in compiled_scenes if scene.get("default", False)), None)
+    specs = [
+        "SceneSpec("
+        f'id: "{scene_registry_id(scene)}".SceneId, '
+        f'name: "{sanitize_nim_string(scene.get("name", "Default"))}", '
+        f'libraryName: "{scene_library_filename(scene)}"'
+        ")"
+        for scene in compiled_scenes
+    ]
+
+    newline = "\n"
+    spec_lines = ("," + newline + "  ").join(specs)
+    if spec_lines:
+        spec_lines = newline + "  " + spec_lines + newline
+
+    scenes_source = f"""
+import std/[dynlib, json, options, os, tables]
+import frameos/types
+import frameos/channels as hostChannels
+import frameos/driver_abi
+
+type
+  SceneSpec = object
+    id: SceneId
+    name: string
+    libraryName: string
+
+  LoadedSceneLibrary = object
+    spec: SceneSpec
+    library: LibHandle
+    exportedScene: ExportedScene
+
+  SceneInitProc = proc(logHook: HostLogProc, sendEventHook: HostSendEventProc) {{.cdecl.}}
+  SceneExportProc = proc(): pointer {{.cdecl.}}
+
+let sceneSpecs: seq[SceneSpec] = @[{spec_lines}]
+var loadedSceneLibraries: seq[LoadedSceneLibrary] = @[]
+
+{_default_scene_line(default_scene)}
+
+const sceneOptions*: array[{len(sceneOptionTuples)}, tuple[id: SceneId, name: string]] = [
+{newline.join(sorted(sceneOptionTuples))}
+]
+
+proc hostLog(event: JsonNode) {{.cdecl, gcsafe.}} =
+  hostChannels.log(event)
+
+proc hostSendEvent(scene: Option[SceneId], event: string, payload: JsonNode) {{.cdecl, gcsafe.}} =
+  hostChannels.sendEvent(scene, event, payload)
+
+proc sceneLibraryPath(spec: SceneSpec): string =
+  getAppDir() / "scenes" / spec.libraryName
+
+proc loadRequiredSymbol[T](library: LibHandle, sceneId: SceneId, symbol: string): T =
+  let address = symAddr(library, symbol)
+  if address.isNil:
+    hostChannels.log(%*{{"event": "scene:shared:error", "sceneId": sceneId.string,
+        "error": "Missing symbol", "symbol": symbol}})
+    return nil
+  cast[T](address)
+
+proc loadSharedScene(spec: SceneSpec): Option[ExportedScene] =
+  let path = sceneLibraryPath(spec)
+  let library = loadLib(path)
+  if library.isNil:
+    hostChannels.log(%*{{"event": "scene:shared:error", "sceneId": spec.id.string,
+        "error": "Unable to load scene library", "path": path}})
+    return none(ExportedScene)
+
+  let initProc = loadRequiredSymbol[SceneInitProc](library, spec.id, "frameos_scene_init")
+  if initProc.isNil:
+    unloadLib(library)
+    return none(ExportedScene)
+  initProc(hostLog, hostSendEvent)
+
+  let exportProc = loadRequiredSymbol[SceneExportProc](library, spec.id, "frameos_scene_export")
+  if exportProc.isNil:
+    unloadLib(library)
+    return none(ExportedScene)
+
+  let exportedScene = cast[ExportedScene](exportProc())
+  if exportedScene.isNil:
+    hostChannels.log(%*{{"event": "scene:shared:error", "sceneId": spec.id.string,
+        "error": "Scene library returned nil export", "path": path}})
+    unloadLib(library)
+    return none(ExportedScene)
+
+  loadedSceneLibraries.add(LoadedSceneLibrary(spec: spec, library: library, exportedScene: exportedScene))
+  hostChannels.log(%*{{"event": "scene:shared", "sceneId": spec.id.string, "path": path, "loaded": true}})
+  return some(exportedScene)
+
+proc getExportedScenes*(): Table[SceneId, ExportedScene] =
+  result = initTable[SceneId, ExportedScene]()
+  for spec in sceneSpecs:
+    let exportedScene = loadSharedScene(spec)
+    if exportedScene.isSome:
+      result[spec.id] = exportedScene.get()
+"""
+
+    return scenes_source
+
+
+def write_scenes_nim(frame: Frame, compilation_mode: str = DEFAULT_COMPILATION_MODE) -> str:
+    if compilation_mode_uses_shared_libraries(compilation_mode):
+        return write_shared_scenes_nim(frame)
+    return write_static_scenes_nim(frame)