reckless: add shebang installer for fancy plugins

This allows plugins with a uv run script to install
themselves.  Unfortunately it requires file access to all
potential entrypoints to check if they are installable.

Changelog-Added: reckless can now install plugins executable by shebang.

Author: endothermicdev

tests/data/recklessrepo/lightningd/testplugshebang/manifest.json

@@ -0,0 +1,7 @@
+{
+    "name": "testplugshebang",
+    "short_description": "a plugin to test reckless installation with a UV shebang",
+    "long_description": "This plugin is used in the reckless blackbox tests.  This one manages its own dependency installation with uv invoked by #! from within the plugin.",
+    "entrypoint": "testplugshebang.py",
+    "requirements": ["python3"]
+}

tests/data/recklessrepo/lightningd/testplugshebang/requirements.txt

@@ -0,0 +1,2 @@
+pyln-client
+

tests/data/recklessrepo/lightningd/testplugshebang/testplugshebang.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env -S uv run --script
+
+# /// script
+# requires-python = ">=3.9.2"
+# dependencies = [
+#   "pyln-client>=25.12",
+# ]
+# ///
+
+from pyln.client import Plugin
+
+plugin = Plugin()
+
+__version__ = 'v1'
+
+
+@plugin.init()
+def init(options, configuration, plugin, **kwargs):
+    plugin.log("testplugshebang initialized")
+
+
+@plugin.method("plugintest")
+def plugintest(plugin):
+    return ("success")
+
+
+plugin.run()

tests/data/recklessrepo/rkls_api_lightningd_plugins.json

@@ -34,5 +34,14 @@
     "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugpyproj",
     "download_url": null,
     "type": "dir"
+  },
+  {
+    "name": "testplugshebang",
+    "path": "testplugshebang",
+    "url": "https://api.github.com/repos/lightningd/plugins/contents/webhook?ref=master",
+    "html_url": "https://github.com/lightningd/plugins/tree/master/testplugshebang",
+    "git_url": "https://api.github.com/repos/lightningd/plugins/git/trees/testplugshebang",
+    "download_url": null,
+    "type": "dir"
   }
 ]

tests/test_reckless.py

@@ -419,6 +419,21 @@ def test_reckless_uv_install(node_factory):
     r.check_stderr()
 
 
+@unittest.skipIf(VALGRIND and SLOW_MACHINE, "node too slow for starting plugin under valgrind")
+def test_reckless_shebang_install(node_factory):
+    node = get_reckless_node(node_factory)
+    node.start()
+    r = reckless([f"--network={NETWORK}", "-v", "install", "testplugshebang"],
+                 dir=node.lightning_dir)
+    assert r.returncode == 0
+    installed_path = Path(node.lightning_dir) / 'reckless/testplugshebang'
+    assert installed_path.is_dir()
+    assert node.rpc.plugintest() == 'success'
+
+    assert r.search_stdout('using installer shebang')
+    r.check_stderr()
+
+
 def test_reckless_available(node_factory):
     """list available plugins"""
     n = get_reckless_node(node_factory)

tools/reckless

@@ -157,6 +157,9 @@ class Installer:
         self.manager = manager    # dependency manager (if required)
         self.dependency_file = None
         self.dependency_call = None
+        # extra check routine to see if a source is installable by this Installer
+        self.check = None
+
 
     def __repr__(self):
         return (f'<Installer {self.name}: '
@@ -175,7 +178,7 @@ class Installer:
             return False
         return True
 
-    def installable(self) -> bool:
+    def installable(self, source) -> bool:
         '''Validate the necessary compiler and package manager executables are
         available to install.  If these are defined, they are considered
         mandatory even though the user may have the requisite packages already
@@ -184,6 +187,8 @@ class Installer:
             return False
         if self.manager and not shutil.which(self.manager):
             return False
+        if self.check:
+            return self.check(source)
         return True
 
     def add_entrypoint(self, entry: str):
@@ -484,9 +489,14 @@ class InstInfo:
                         found_entry = sub.find(g, ftype=SourceFile)
                         if found_entry:
                             break
-                    # FIXME: handle a list of dependencies
-                    found_dep = sub.find(inst.dependency_file,
-                                         ftype=SourceFile)
+
+                    if inst.dependency_file:
+                        # FIXME: handle a list of dependencies
+                        found_dep = sub.find(inst.dependency_file,
+                                             ftype=SourceFile)
+                    else:
+                        found_dep = None
+
                     if found_entry:
                         # Success!
                         if found_dep:
@@ -1031,6 +1041,47 @@ def install_to_python_virtual_environment(cloned_plugin: InstInfo):
     return cloned_plugin
 
 
+def have_files(source: SourceDir):
+    """Do we have direct access to the files in this directory?"""
+    if source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO,
+                          Source.GIT_LOCAL_CLONE]:
+        return True
+    log.info(f'no files in {source.name} ({source.srctype})')
+    return False
+
+
+def fetch_manifest(source: SourceDir) -> dict:
+    """read and ingest a manifest from the provided source."""
+    log.debug(f'ingesting manifest from {source.name}: {source.location}/manifest.json ({source.srctype})')
+    # local_path = RECKLESS_DIR / '.remote_sources' / user
+    if source.srctype not in [Source.GIT_LOCAL_CLONE, Source.LOCAL_REPO, Source.DIRECTORY]:
+        log.info(f'oops! {source.srctype}')
+        return None
+    if source.srctype == Source.GIT_LOCAL_CLONE:
+        try:
+            repo = GithubRepository(source.parent_source.original_source)
+            path = RECKLESS_DIR / '.remote_sources' / repo.user / repo.name
+        except AssertionError:
+            log.info(f'could not parse github source {source.parent_source.original_source}')
+            return None
+    elif source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO]:
+        path = Path(source.location)
+    else:
+        raise Exception(f"cannot access manifest in {source.srctype}: {source}")
+    if source.relative:
+        path = path / source.relative
+    path = path / 'manifest.json'
+    if not path.exists():
+        return None
+    with open(path, 'r+') as manifest_file:
+        try:
+            manifest = json.loads(manifest_file.read())
+            return manifest
+        except json.decoder.JSONDecodeError:
+            log.warning(f'{source.name} contains malformed manifest ({source.parent_source.original_source})')
+            return None
+
+
 def cargo_installation(cloned_plugin: InstInfo):
     call = ['cargo', 'build', '--release', '-vv']
     # FIXME: the symlinked Cargo.toml allows the installer to identify a valid
@@ -1144,6 +1195,60 @@ def install_python_uv_legacy(cloned_plugin: InstInfo):
     return cloned_plugin
 
 
+def open_source_entrypoint(source: InstInfo) -> str:
+    if source.srctype not in [Source.GIT_LOCAL_CLONE, Source.LOCAL_REPO, Source.DIRECTORY]:
+        log.info(f'oops! {source.srctype}')
+        return None
+    assert source.entry
+    file = Path(source.source_loc)
+    # if source.subdir:
+    #     file /= source.subdir
+    file /= source.entry
+    log.debug(f'checking entry file {str(file)}')
+    if file.exists():
+        # FIXME: check file encoding
+        try:
+            with open(file, 'r') as f:
+                return f.read()
+        except UnicodeDecodeError:
+            log.debug('failed to read source file')
+            return None
+    else:
+        log.debug('could not find source file')
+
+    return None
+
+def check_for_shebang(source: InstInfo) -> bool:
+    # TODO: match name, open file, look for shebang
+    # FIXME: Just testing that the plugin runs right now
+    log.debug(f'checking for shebang in {source}')
+    if source.source_dir:
+        source.get_inst_details()
+        log.debug(f'given source dir {source.source_dir}')
+        log.debug(f'given entry {source.entry}')
+        if have_files(source.source_dir):
+            log.debug('Have access to files!')
+            entrypoint_file = open_source_entrypoint(source)
+            log.debug(f"first line: {entrypoint_file.split('\n')[0]}")
+            if entrypoint_file.split('\n')[0].startswith('#!'):
+                log.debug('have shebang')
+                # Calling the python interpreter will not manage dependencies.
+                # Leave this to another python installer.
+                for interpreter in ['bin/python', 'env python']:
+                    if interpreter in entrypoint_file.split('\n')[0]:
+                        return False
+                return True
+
+
+        # Open entrypoint file
+        # check for shebang
+        raise Exception('Keep going!')
+    if source.name == 'testplugshebang':
+        raise Exception("hold up!")
+        return True
+    return False
+
+
 python3venv = Installer('python3venv', exe='python3',
                         manager='pip', entry='{name}.py')
 python3venv.add_entrypoint('{name}')
@@ -1186,7 +1291,12 @@ rust_cargo = Installer('rust', manager='cargo', entry='Cargo.toml')
 rust_cargo.add_dependency_file('Cargo.toml')
 rust_cargo.dependency_call = cargo_installation
 
-INSTALLERS = [pythonuv, pythonuvlegacy, python3venv, poetryvenv,
+shebang = Installer('shebang', entry='{name}.py')
+shebang.add_entrypoint('{name}')
+# An extra installable check to see if a #! is present in the file
+shebang.check = check_for_shebang
+
+INSTALLERS = [shebang, pythonuv, pythonuvlegacy, python3venv, poetryvenv,
               pyprojectViaPip, nodejs, rust_cargo]
 
 
@@ -1376,6 +1486,8 @@ def _checkout_commit(orig_src: InstInfo,
 def _install_plugin(src: InstInfo) -> Union[InstInfo, None]:
     """make sure the repo exists and clone it."""
     log.debug(f'Install requested from {src}.')
+    if src.source_dir and src.source_dir.parent_source:
+        log.debug(f'source has parent {src.source_dir.parent_source}')
     if RECKLESS_CONFIG is None:
         log.error('reckless install directory unavailable')
         return None
@@ -1411,6 +1523,8 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]:
     # FIXME: Validate path was cloned successfully.
     # Depending on how we accessed the original source, there may be install
     # details missing. Searching the cloned repo makes sure we have it.
+    # FIXME: This could be cloned to .remotesources and the global sources
+    # could then be updated with this new LoadedSource to save on additional cloning.
     clone = LoadedSource(plugin_path)
     clone.content.populate()
     # Make sure we don't try to fetch again!
@@ -1426,14 +1540,29 @@ def _install_plugin(src: InstInfo) -> Union[InstInfo, None]:
     if not plugin_path:
         return None
 
+    # FIXME: replace src wholesale
+    # We have a hunch it's in this directory/source, so link it here.
+    inst_check_src = copy.copy(src)
+    if not inst_check_src.source_dir:
+        inst_check_src.source_loc = plugin_path
+        inst_check_src.source_dir = clone.content
+        inst_check_src.source_dir.parent_source = clone
+
+        if src.srctype == Source.GITHUB_REPO:
+            inst_check_src.srctype = Source.GIT_LOCAL_CLONE
+        else:
+            inst_check_src.srctype = clone.type
+
     # Find a suitable installer
     INSTALLER = None
     for inst_method in INSTALLERS:
-        if not (inst_method.installable() and inst_method.executable()):
+        if not (inst_method.installable(inst_check_src) and inst_method.executable()):
             continue
         if inst_method.dependency_file is not None:
             if inst_method.dependency_file not in os.listdir(plugin_path):
                 continue
+        if inst_method.check and not inst_method.check(inst_check_src):
+            continue
         log.debug(f"using installer {inst_method.name}")
         INSTALLER = inst_method
         break
@@ -1685,6 +1814,8 @@ def search(plugin_name: str) -> Union[InstInfo, None]:
         if not found:
             continue
         log.info(f"found {found.name} in source: {found.source_loc}")
+        # FIXME: remove debugging
+        log.debug(f"source type: {found.srctype}")
         log.debug(f"entry: {found.entry}")
         if found.subdir:
             log.debug(f'sub-directory: {found.subdir}')
@@ -2086,47 +2217,6 @@ def listinstalled():
     return plugins
 
 
-def have_files(source: SourceDir):
-    """Do we have direct access to the files in this directory?"""
-    if source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO,
-                          Source.GIT_LOCAL_CLONE]:
-        return True
-    log.info(f'no files in {source.name} ({source.srctype})')
-    return False
-
-
-def fetch_manifest(source: SourceDir) -> dict:
-    """read and ingest a manifest from the provided source."""
-    log.debug(f'ingesting manifest from {source.name}: {source.location}/manifest.json ({source.srctype})')
-    # local_path = RECKLESS_DIR / '.remote_sources' / user
-    if source.srctype not in [Source.GIT_LOCAL_CLONE, Source.LOCAL_REPO, Source.DIRECTORY]:
-        log.info(f'oops! {source.srctype}')
-        return None
-    if source.srctype == Source.GIT_LOCAL_CLONE:
-        try:
-            repo = GithubRepository(source.parent_source.original_source)
-            path = RECKLESS_DIR / '.remote_sources' / repo.user / repo.name
-        except AssertionError:
-            log.info(f'could not parse github source {source.parent_source.original_source}')
-            return None
-    elif source.srctype in [Source.DIRECTORY, Source.LOCAL_REPO]:
-        path = Path(source.location)
-    else:
-        raise Exception(f"cannot access manifest in {source.srctype}: {source}")
-    if source.relative:
-        path = path / source.relative
-    path = path / 'manifest.json'
-    if not path.exists():
-        return None
-    with open(path, 'r+') as manifest_file:
-        try:
-            manifest = json.loads(manifest_file.read())
-            return manifest
-        except json.decoder.JSONDecodeError:
-            log.warning(f'{source.name} contains malformed manifest ({source.parent_source.original_source})')
-            return None
-
-
 def find_plugin_candidates(source: Union[LoadedSource, SourceDir], depth=2) -> list:
     """Filter through a source and return any candidates that appear to be
     installable plugins with the registered installers."""