DOC: Add file-based listed gallery order for examples

This adds support for a file "gallery_order.txt" in gallery folders for specifying the display order or examples.

Advantages compared to the previous order approach:
- Orders are now defined inside each gallery and not in a central list
- The placeholder '*' allows to put some examples at the front, some at the end, and all others alphabetically sorted in the position of the placeholder
- Checking of entries that do no exist as examples
- Checking that all examples are listed (if no placeholder is specified)

Co-authored-by: Elliott Sales de Andrade <quantum.analyst@gmail.com>

Author: timhoffm

doc/devel/document.rst

@@ -1100,24 +1100,18 @@ reference should always be the second to mention; see the example above.
 Order examples
 --------------
 
-The order of the sections of the :ref:`tutorials` and the :ref:`gallery`, as
-well as the order of the examples within each section are determined in a
-two step process from within the :file:`/doc/sphinxext/gallery_order.py`:
-
-* *Explicit order*: This file contains a list of folders for the section order
-  and a list of examples for the subsection order. The order of the items
-  shown in the doc pages is the order those items appear in those lists.
-* *Implicit order*: If a folder or example is not in those lists, it will be
-  appended after the explicitly ordered items and all of those additional
-  items will be ordered by pathname (for the sections) or by filename
-  (for the subsections).
-
-As a consequence, if you want to let your example appear in a certain
-position in the gallery, extend those lists with your example.
-In case no explicit order is desired or necessary, still make sure
-to name your example consistently, i.e. use the main function or subject
-of the example as first word in the filename; e.g. an image example
-should ideally be named similar to :file:`imshow_mynewexample.py`.
+The order of the sections in :ref:`plot_types`, :ref:`tutorials` and :ref:`gallery`
+are determined through a custom Sphinx Gallery plugin in :file:`/doc/sphinxext/gallery_order.py`
+Configuration happens directly in that file.
+
+The order of examples within each gallery section is determined from a file
+:file:`gallery_order.txt` in each folder. If the file is missing, the examples
+are sorted alphabetically by filename.
+
+:file:`gallery_order.txt` contains a list of example filenames (without the .py extension)
+in the desired order, with an optional '*' to indicate where not-listed examples should be
+placed. If '*' is not present, all examples must be listed, or an error will be raised.
+Use this if you want to ensure that a full order is intentionally maintained.
 
 .. _raw_restructured_gallery:
 

doc/sphinxext/gallery_order.py

@@ -4,6 +4,8 @@
 """
 
 import itertools
+from pathlib import Path
+
 from sphinx_gallery.sorting import ExplicitOrder
 
 # Gallery sections shall be displayed in the following order.
@@ -64,70 +66,87 @@ def __call__(self, item):
         else:
             return f"{self.ordered_list.index(UNSORTED):04d}{item}"
 
-# Subsection order:
-# Subsections are ordered by filename, unless they appear in the following
-# lists in which case the list order determines the order within the section.
-# Examples/tutorials that do not appear in a list will be appended.
-
-list_all = [
-    #  **Tutorials**
-    #  introductory
-    "quick_start", "pyplot", "images", "lifecycle", "customizing",
-    #  intermediate
-    "artists", "legend_guide", "color_cycle",
-    "constrainedlayout_guide", "tight_layout_guide",
-    #  advanced
-    #  text
-    "text_intro", "text_props",
-    #  colors
-    "colors",
-
-    #  **Examples**
-    #  animation
-    "simple_anim",  # Most basic example
-    #  color
-    "color_demo",
-    #  pies
-    "pie_features", "pie_demo2",
-    # scales
-    "scales",  # Scales overview
-
-    # **Plot Types
-    # Basic
-    "plot", "scatter_plot", "bar", "stem", "step", "fill_between",
-    # Arrays
-    "imshow", "pcolormesh", "contour", "contourf",
-    "barbs", "quiver", "streamplot",
-    # Stats
-    "hist_plot", "boxplot_plot", "errorbar_plot", "violin",
-    "eventplot", "hist2d", "hexbin", "pie",
-    # Unstructured
-    "tricontour", "tricontourf", "tripcolor", "triplot",
-    # Spines
-    "spines", "spine_placement_demo", "spines_dropped",
-    "multiple_yaxis_with_spines", "centered_spines_with_arrows",
-    ]
-explicit_subsection_order = [item + ".py" for item in list_all]
-
-
-class MplExplicitSubOrder(ExplicitOrder):
-    """For use within the 'within_subsection_order' key."""
+
+class MplFileExplicitOrder(ExplicitOrder):
+    """
+    An explicit order class that reads the order of examples from 'gallery_order.txt'.
+
+    For use with the sphinx_gallery 'within_subsection_order' key.
+
+    The file contains a list of example filenames (without the .py extension) in the
+    desired order, with an optional '*' to indicate where not-listed examples should be
+    placed.
+
+    If '*' is not present, all examples must be listed, or an error will be raised.
+    Use this if you want to ensure that a full order is intentionally maintained.
+    """
     def __init__(self, src_dir):
-        self.src_dir = src_dir  # src_dir is unused here
-        self.ordered_list = explicit_subsection_order
+        ordered_list = self.read_gallery_order(Path(src_dir)) or []
+        super().__init__(ordered_list)
+
+    @staticmethod
+    def read_gallery_order(src_dir: Path):
+        """Return the list of examples to be sorted; read from 'gallery_order.txt'."""
+        gallery_order_txt = src_dir / "gallery_order.txt"
+        if not gallery_order_txt.exists():
+            return None
+        lines = [
+            line.strip()
+            for line in gallery_order_txt.read_text().splitlines()
+            if line.strip() and not line.startswith("#")
+        ]
+
+        try:
+            placeholder_index = lines.index("*")
+        except ValueError:
+            placeholder_index = None
+
+        lines = [line + ".py" for line in lines]
+
+        if placeholder_index is None:
+            front = lines
+            back = []
+        else:
+            front = lines[:placeholder_index]
+            back = lines[placeholder_index+1:]
+
+        listed_examples = {*front, *back}
+        existing_examples = {
+            file.name for file in src_dir.iterdir() if file.suffix == ".py"
+        }
+
+        non_existing_examples = listed_examples - existing_examples
+        missing_examples = existing_examples - listed_examples
+        print(f"non_existing_examples: {non_existing_examples}")
+        print(f"missing_examples: {missing_examples}")
+
+        if non_existing_examples:
+            raise ValueError(
+                f"The following examples listed in {gallery_order_txt} do not exist: "
+                f"{', '.join(non_existing_examples)}")
+        if placeholder_index is None and missing_examples:
+            raise ValueError(
+                f"The following examples are not listed in {gallery_order_txt}. "
+                f"Either include them or add a '*' to indicate where not listed "
+                f"examples should be placed: "
+                f"{', '.join(missing_examples)}"
+            )
+
+        mid = list(sorted(missing_examples))
+        return front + mid + back
 
     def __call__(self, item):
         """Return a string determining the sort order."""
-        if item in self.ordered_list:
-            return f"{self.ordered_list.index(item):04d}"
-        else:
-            # ensure not explicitly listed items come last.
-            return "zzz" + item
+        if not self.ordered_list:
+            return item
+        return f"{self.ordered_list.index(item):04d}"
 
+    def __repr__(self):
+        return '<%s: %s>' % (self.__class__.__name__, self.ordered_list)
 
 # Provide the above classes for use in conf.py
 sectionorder = MplExplicitOrder(explicit_order_folders)
-subsectionorder = MplExplicitSubOrder
+subsectionorder = MplFileExplicitOrder
 
 _preserve_count = itertools.count()
 

galleries/examples/animation/gallery_order.txt

@@ -0,0 +1,3 @@
+# Explicit example order. See https://matplotlib.org/devdocs/devel/document.html#order-examples
+simple_anim
+*

galleries/examples/color/gallery_order.txt

@@ -0,0 +1,3 @@
+# Explicit example order. See https://matplotlib.org/devdocs/devel/document.html#order-examples
+color_demo
+*

galleries/examples/pie_and_polar_charts/gallery_order.txt

@@ -0,0 +1,3 @@
+# Explicit example order. See https://matplotlib.org/devdocs/devel/document.html#order-examples
+pie_features
+*

galleries/examples/scales/gallery_order.txt

@@ -0,0 +1,3 @@
+# Explicit example order. See https://matplotlib.org/devdocs/devel/document.html#order-examples
+scales
+*

galleries/examples/spines/gallery_order.txt

@@ -0,0 +1,6 @@
+# Explicit example order. See https://matplotlib.org/devdocs/devel/document.html#order-examples
+spines
+spine_placement_demo
+spines_dropped
+multiple_yaxis_with_spines
+centered_spines_with_arrows

galleries/plot_types/basic/gallery_order.txt

@@ -0,0 +1,8 @@
+# Explicit example order. See https://matplotlib.org/devdocs/devel/document.html#order-examples
+plot
+scatter_plot
+bar
+stem
+fill_between
+stackplot
+stairs

galleries/plot_types/stats/gallery_order.txt

@@ -0,0 +1,10 @@
+# Explicit example order. See https://matplotlib.org/devdocs/devel/document.html#order-examples
+hist_plot
+boxplot_plot
+errorbar_plot
+violin
+eventplot
+hist2d
+hexbin
+pie
+ecdf

galleries/plot_types/unstructured/gallery_order.txt

@@ -0,0 +1,5 @@
+# Explicit example order. See https://matplotlib.org/devdocs/devel/document.html#order-examples
+tricontour
+tricontourf
+tripcolor
+triplot