Allow python help to be shown when package not imported (#13501)

See #12897 

This PR allows python to show help pages even when the module hasn't
been imported.

This fixes two issues:

When using `?` in the console, the kernel would throw before passing the
object's name to the help service. The fix is to monkeypatch the `pinfo`
command and if the referenced object is not found fallback to passing it
into the help service anyway. Otherwise, it uses the normal code path.

Second, now that I can easily click the help button I find that package
distribution names don't always map to their import names. So anything
with a dash like `annotated-types` would need to be mapped to
`annotated_types`. And not only that, some packages have entirely
different names like `scikit-learn` becomes `sklearn`.

This PR can be delivered independent of the help button #13499. However,
one big win with this change is that now we can include the help button
on every package, not just the attached ones.

### Release Notes

#### New Features

- Python: Adds help feature `?cowsay` to packages that are not already
imported.

#### Bug Fixes

- N/A


### Validation Steps

When you first launch a python shell, there is probably very little if
anything imported. I always test with cowsay so you can just try typing
`?cowsay` or `??cowsay` in the console.

Author: bricestacey

extensions/positron-python/python_files/posit/positron/help.py

@@ -8,6 +8,7 @@
 import contextlib
 import logging
 import pydoc
+import re
 from types import MappingProxyType
 from typing import TYPE_CHECKING, Any
 
@@ -28,6 +29,41 @@
 logger = logging.getLogger(__name__)
 
 
+def _canonicalize_distribution_name(name: str) -> str:
+    # PEP 503: collapse runs of -_. into a single dash and lowercase.
+    return re.sub(r"[-_.]+", "-", name).lower()
+
+
+def _distribution_to_modules(name: str) -> list[str]:
+    """Top-level modules provided by the PyPI distribution `name`.
+
+    PyPI distribution names don't always match the importable module name
+    (e.g. "scikit-learn" -> "sklearn", "python-dateutil" -> "dateutil").
+    Returns an empty list on Python < 3.10 (where the stdlib API is missing)
+    or when no installed distribution matches.
+    """
+    try:
+        # packages_distributions exists on Python >= 3.10; pyright's stubs for
+        # older versions don't know about it, so suppress the import-symbol
+        # check here. ImportError handles the actual runtime absence.
+        from importlib.metadata import (
+            packages_distributions,  # type: ignore[reportGeneralTypeIssues]
+        )
+    except ImportError:
+        return []
+
+    canonical = _canonicalize_distribution_name(name)
+    modules: list[str] = []
+    for module, distributions in packages_distributions().items():
+        if any(_canonicalize_distribution_name(d) == canonical for d in distributions):
+            modules.append(module)
+    # When a distribution exposes multiple top-level modules (e.g. setuptools
+    # also exposes _distutils_hack, pkg_resources), prefer one whose name
+    # matches the distribution name.
+    modules.sort(key=lambda m: _canonicalize_distribution_name(m) != canonical)
+    return modules
+
+
 def help(topic="help"):  # noqa: A001
     """
     Show help for the given topic.
@@ -119,6 +155,16 @@ def show_help(self, request: str | Any | None) -> None:
         with contextlib.suppress(ImportError):
             result = pydoc.resolve(thing=request)
 
+        # If pydoc can't resolve the request (e.g. a PyPI distribution name like
+        # "scikit-learn" whose import name is "sklearn"), try mapping the
+        # distribution name to its top-level module(s) and resolving those.
+        if result is None and isinstance(request, str):
+            for module_name in _distribution_to_modules(request):
+                with contextlib.suppress(ImportError):
+                    result = pydoc.resolve(thing=module_name)
+                if result is not None:
+                    break
+
         if result is None:
             # We could not resolve to an object, try to get help for the request as a string.
             key = request

extensions/positron-python/python_files/posit/positron/positron_ipkernel.py

@@ -8,6 +8,7 @@
 from __future__ import annotations
 
 import enum
+import importlib.util
 import json
 import logging
 import os
@@ -37,7 +38,7 @@
 from .connections import ConnectionsService
 from .data_explorer import DataExplorerService, DataExplorerWarning
 from .debugger import PositronDebugger
-from .help import HelpService, help  # noqa: A004
+from .help import HelpService, _distribution_to_modules, help  # noqa: A004
 from .lsp import LSPService
 from .patch.bokeh import handle_bokeh_output, patch_bokeh_no_access
 from .patch.haystack import patch_haystack_is_in_jupyter
@@ -112,6 +113,17 @@ def pinfo(
     pinfo.__doc__ = oinspect.Inspector.pinfo.__doc__
 
 
+def _is_module_on_disk(name: str) -> bool:
+    """Return True if the top-level package of `name` is installed on disk."""
+    top_level = name.split(".", 1)[0]
+    if not top_level.isidentifier():
+        return False
+    try:
+        return importlib.util.find_spec(top_level) is not None
+    except (ImportError, ValueError):
+        return False
+
+
 @magics_class
 class PositronMagics(Magics):
     shell: PositronShell
@@ -462,6 +474,19 @@ def init_display_formatter(self):
         self.display_formatter = PositronDisplayFormatter(parent=self)
         self.configurables.append(self.display_formatter)  # type: ignore IPython type annotation is wrong
 
+    def _inspect(self, meth, oname, namespaces=None, **kw):
+        # For `?name`, if the name isn't a live object but is an installed
+        # module or a known PyPI distribution name (e.g. `scikit-learn`,
+        # whose import name is `sklearn`), show its pydoc help instead of
+        # letting IPython print "Object not found". Pre-checking here
+        # (rather than after super()) avoids the duplicate "not found" message.
+        if meth == "pinfo" and (_is_module_on_disk(oname) or _distribution_to_modules(oname)):
+            info = self._object_find(oname, namespaces)
+            if not info.found and not hasattr(info.parent, oinspect.HOOK_NAME):
+                self.kernel.help_service.show_help(oname)
+                return None
+        return super()._inspect(meth, oname, namespaces, **kw)
+
     def _handle_pre_run_cell(self, info: ExecutionInfo) -> None:
         """Prior to execution, reset the user environment watch state."""
         # If an empty cell is being executed, do nothing.

extensions/positron-python/python_files/posit/positron/tests/test_help.py

@@ -128,6 +128,108 @@ def test_show_help(
     ]
 
 
+@pytest.mark.parametrize(
+    ("raw", "canonical"),
+    [
+        ("scikit-learn", "scikit-learn"),
+        ("scikit_learn", "scikit-learn"),
+        ("Scikit_Learn", "scikit-learn"),
+        ("python.dateutil", "python-dateutil"),
+        ("Pillow", "pillow"),
+        ("already-canonical", "already-canonical"),
+    ],
+)
+def test_canonicalize_distribution_name(raw: str, canonical: str) -> None:
+    """PEP 503 normalization collapses runs of -_. and lowercases."""
+    from positron.help import _canonicalize_distribution_name
+
+    assert _canonicalize_distribution_name(raw) == canonical
+
+
+@pytest.mark.parametrize(
+    ("dist_name", "packages_map", "expected_first", "expected_set"),
+    [
+        # PyPI dist "scikit-learn" -> import "sklearn".
+        ("scikit-learn", {"sklearn": ["scikit-learn"]}, "sklearn", {"sklearn"}),
+        # PEP 503 variants of the dist name still match.
+        ("Scikit_Learn", {"sklearn": ["scikit-learn"]}, "sklearn", {"sklearn"}),
+        # When a dist exposes multiple top-levels, the module whose canonical
+        # name matches the dist is sorted first.
+        (
+            "setuptools",
+            {
+                "_distutils_hack": ["setuptools"],
+                "setuptools": ["setuptools"],
+                "pkg_resources": ["setuptools"],
+            },
+            "setuptools",
+            {"_distutils_hack", "setuptools", "pkg_resources"},
+        ),
+        # Unknown dist returns empty.
+        ("nonexistent-dist", {"sklearn": ["scikit-learn"]}, None, set()),
+    ],
+    ids=["differs", "canonical-variants", "multi-top-level-preferred", "unknown"],
+)
+def test_distribution_to_modules(
+    monkeypatch: pytest.MonkeyPatch,
+    dist_name: str,
+    packages_map: dict,
+    expected_first,
+    expected_set: set,
+) -> None:
+    """Map distribution names to their top-level importable modules."""
+    import importlib.metadata
+
+    from positron.help import _distribution_to_modules
+
+    # `raising=False` -- packages_distributions doesn't exist on Python < 3.10,
+    # but our code imports it lazily; we just need the attribute present for
+    # the test's mocked dispatch.
+    monkeypatch.setattr(
+        importlib.metadata,
+        "packages_distributions",
+        lambda: packages_map,
+        raising=False,
+    )
+
+    result = _distribution_to_modules(dist_name)
+    assert (result[0] if result else None, set(result)) == (expected_first, expected_set)
+
+
+def test_distribution_to_modules_missing_api(monkeypatch: pytest.MonkeyPatch) -> None:
+    """On Python < 3.10 (no packages_distributions) we gracefully return []."""
+    import importlib.metadata
+
+    from positron.help import _distribution_to_modules
+
+    monkeypatch.delattr(importlib.metadata, "packages_distributions", raising=False)
+
+    assert _distribution_to_modules("scikit-learn") == []
+
+
+def test_show_help_resolves_distribution_name(
+    help_service: HelpService,
+    help_comm,
+    mock_pydoc_thread,
+    monkeypatch: pytest.MonkeyPatch,
+) -> None:
+    """When pydoc can't resolve a name, fall back via the distribution mapping."""
+    import importlib.metadata
+
+    # numpy is a real module; pretend it's shipped as the dist "fake-dist".
+    # `raising=False` so this works on Python 3.9 where the attribute is absent.
+    monkeypatch.setattr(
+        importlib.metadata,
+        "packages_distributions",
+        lambda: {"numpy": ["fake-dist"]},
+        raising=False,
+    )
+
+    help_service.show_help("fake-dist")
+
+    assert help_comm.messages == [show_help_event(f"{mock_pydoc_thread.url}get?key=numpy")]
+
+
 def test_handle_show_help_topic(help_comm, mock_pydoc_thread) -> None:
     msg = json_rpc_request(
         HelpBackendRequest.ShowHelpTopic, {"topic": "logging"}, comm_id="dummy_comm_id"

extensions/positron-python/python_files/posit/positron/tests/test_positron_ipkernel.py

@@ -394,6 +394,56 @@ def test_pinfo_2(shell: PositronShell, tmp_path: Path, mock_ui_service: Mock) ->
     mock_ui_service.open_editor.assert_called_once_with(expected_file, 4, 0)
 
 
+def test_pinfo_unimported_module(
+    shell: PositronShell, tmp_path: Path, mock_help_service: Mock, capsys
+) -> None:
+    """`?name` should call show_help when name is an installed-but-not-imported module."""
+    module_name = "test_pinfo_unimported_xyz"
+    (tmp_path / f"{module_name}.py").write_text("# empty\n")
+
+    with prepended_to_syspath(str(tmp_path)):
+        assert module_name not in shell.user_ns
+        shell.run_cell(f"{module_name}?")
+
+    mock_help_service.show_help.assert_called_once_with(module_name)
+    # Should not also print IPython's default "Object not found" message.
+    assert "not found" not in capsys.readouterr().out
+
+
+def test_pinfo_nonexistent_name(shell: PositronShell, mock_help_service: Mock) -> None:
+    """`?name` for a name that's neither in scope nor on disk should not call show_help."""
+    shell.run_cell("definitely_not_a_real_module_zzz_123?")
+
+    mock_help_service.show_help.assert_not_called()
+
+
+def test_pinfo_distribution_name(
+    shell: PositronShell,
+    mock_help_service: Mock,
+    monkeypatch: pytest.MonkeyPatch,
+    capsys,
+) -> None:
+    """`?name` should call show_help when name is a PyPI distribution (e.g. `scikit-learn`)."""
+    import importlib.metadata
+
+    # Pretend numpy is shipped as the distribution `fake-dist-xyz`. The import
+    # name (`numpy`) differs from the distribution name, mirroring the real
+    # `scikit-learn` / `sklearn` case. `raising=False` so this works on Python
+    # 3.9 where `packages_distributions` is absent.
+    monkeypatch.setattr(
+        importlib.metadata,
+        "packages_distributions",
+        lambda: {"numpy": ["fake-dist-xyz"]},
+        raising=False,
+    )
+
+    shell.run_cell("?fake-dist-xyz")
+
+    mock_help_service.show_help.assert_called_once_with("fake-dist-xyz")
+    # Should not also print IPython's default "Object not found" message.
+    assert "not found" not in capsys.readouterr().out
+
+
 def test_clear(shell: PositronShell, mock_ui_service: Mock) -> None:
     """Redirect `%clear` to the Positron UI service's `clear_console` method."""
     shell.run_cell("%clear")