Finalize scripts (#687)

Add support for scripts that are run after all "main" scripts.

Author: jkloetzke

doc/manual/configuration.rst

@@ -682,6 +682,32 @@ the ``checkoutScript``.
 Other than the above differences setup scripts are identical to
 :ref:`configuration-recipes-scripts`.
 
+.. _configuration-recipes-finalize:
+
+{checkout,build,package}Finalize[{Bash,Pwsh}]
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Type: String
+
+Finalize scripts are appended after the regular scripts defined by
+:ref:`configuration-recipes-scripts`. They are intended for classes that need
+to run some cleanup or post-processing after the recipe (and any other
+inherited classes) already did their work, e.g. stripping binaries or fixing up
+permissions regardless of what the actual recipe script did.
+
+The scripts of all inherited classes and the recipe are joined into a single
+script per step. While the regular scripts are joined in inheritance order
+(classes first, recipe last -- see :ref:`configuration-recipes-scripts`), the
+finalize scripts are appended afterwards in *reverse* order, that is the
+recipe's finalize script runs first, followed by the finalize scripts of the
+inherited classes, innermost class last. This mirrors the usual "wrapping"
+pattern where the outermost class is expected to run its finalization last.
+
+Other than the above differences finalize scripts are identical to
+:ref:`configuration-recipes-scripts`. In particular a ``checkoutFinalize``
+script also renders the checkout indeterministic by default -- see
+:ref:`configuration-recipes-checkoutdeterministic`.
+
 .. _configuration-recipes-tools:
 
 {checkout,build,package}Tools
@@ -908,14 +934,14 @@ that extra care must be taken for a script to fetch always the same sources. If
 you are sure that the result of the checkout script is always the same you may
 set this to ``True``.
 
-The ``checkoutDeterministic`` keyword only relates to the ``checkoutScript`` at
-the same level. Each recipe or class must declare the determinism of its
-``checkoutScript``. If there is no ``checkoutScript`` then
-``checkoutDeterministic`` implicitly defaults to ``True``. Everything in
-``checkoutSCM`` is *not* affected by ``checkoutDeterministic``. All SCMs
-included in Bob will determine their determinism based on the configuration
-automatically, e.g. using a commit or tag is considered deterministic while
-using a branch is indeterministic.
+The ``checkoutDeterministic`` keyword only relates to the ``checkoutScript``
+and ``checkoutFinalize`` scripts at the same level. Each recipe or class must
+declare the determinism of its scripts. If neither a ``checkoutScript`` nor a
+``checkoutFinalize`` script is defined then ``checkoutDeterministic``
+implicitly defaults to ``True``. Everything in ``checkoutSCM`` is *not*
+affected by ``checkoutDeterministic``. All SCMs included in Bob will determine
+their determinism based on the configuration automatically, e.g. using a commit
+or tag is considered deterministic while using a branch is indeterministic.
 
 If the checkout is deemed deterministic it enables Bob to apply various
 optimizations. Deterministic checkouts do not need to be executed every time

pym/bob/input.py

@@ -131,26 +131,37 @@ def fetchScripts(recipe, prefix, resolveBash, resolvePwsh):
                         prefix + "Setup[Bash]"),
             resolveBash(recipe.get(prefix + "ScriptBash", recipe.get(prefix + "Script")),
                         prefix + "Script[Bash]"),
+            resolveBash(recipe.get(prefix + "FinalizeBash", recipe.get(prefix + "Finalize")),
+                        prefix + "Finalize[Bash]"),
         ),
         ScriptLanguage.PWSH : (
             resolvePwsh(recipe.get(prefix + "SetupPwsh", recipe.get(prefix + "Setup")),
                         prefix + "Setup[Pwsh]"),
             resolvePwsh(recipe.get(prefix + "ScriptPwsh", recipe.get(prefix + "Script")),
                         prefix + "Script[Pwsh]"),
+            resolvePwsh(recipe.get(prefix + "FinalizePwsh", recipe.get(prefix + "Finalize")),
+                        prefix + "Finalize[Pwsh]"),
         )
     }
 
 def mergeScripts(fragments, glue):
     """Join all scripts of the recipe and its classes.
 
-    The result is a tuple with (setupScript, mainScript, digestScript)
+    The result is a tuple with (setupScript, mainScript, digestScript). Note
+    that the mainScript contains all "normal" scripts and all "Finalize"
+    scripts in reverse order.
     """
     return (
+        # The "Setup" scripts
         joinScripts((f[0][0] for f in fragments), glue),
-        joinScripts((f[1][0] for f in fragments), glue),
+        # The "normal" scripts and the "Finalize" scripts
+        joinScripts(chain((f[1][0] for f in fragments),
+                          (f[2][0] for f in reversed(fragments))), glue),
+        # All the digest scripts
         joinScripts(
             ( joinScripts((f[0][1] for f in fragments), "\n"),
               joinScripts((f[1][1] for f in fragments), "\n"),
+              joinScripts((f[2][1] for f in fragments), "\n"),
             ), "\n")
     )
 
@@ -2340,13 +2351,14 @@ def resolveClasses(self, rootEnv):
             self.__scriptLanguage = scriptLanguage
         glue = getLanguage(self.__scriptLanguage).glue
 
-        # Consider checkout deterministic by default if no checkoutScript is
-        # involved. A potential checkoutSetup is ignored.
+        # Consider checkout deterministic by default if neither checkoutScript
+        # nor checkoutFinalize is involved. A potential checkoutSetup is
+        # ignored.
         def coDet(r):
             ret = r.__checkoutDeterministic
             if ret is not None:
                 return ret
-            return r.__checkout[self.__scriptLanguage][1][0] is None
+            return all(s[0] is None for s in r.__checkout[self.__scriptLanguage][1:])
 
         checkoutDeterministic = [ coDet(i) for i in inheritAll ]
         self.__checkoutDeterministic = all(checkoutDeterministic)
@@ -4171,19 +4183,28 @@ def __createSchemas(self):
         dependsInnerClause[schema.Optional('depends')] = dependsClause
 
         classSchemaSpec = {
+            schema.Optional('checkoutFinalize') : str,
+            schema.Optional('checkoutFinalizeBash') : str,
+            schema.Optional('checkoutFinalizePwsh') : str,
             schema.Optional('checkoutScript') : str,
             schema.Optional('checkoutScriptBash') : str,
             schema.Optional('checkoutScriptPwsh') : str,
             schema.Optional('checkoutSetup') : str,
             schema.Optional('checkoutSetupBash') : str,
             schema.Optional('checkoutSetupPwsh') : str,
             schema.Optional('checkoutUpdateIf', default=False) : schema.Or(None, str, bool, IfExpression),
+            schema.Optional('buildFinalize') : str,
+            schema.Optional('buildFinalizeBash') : str,
+            schema.Optional('buildFinalizePwsh') : str,
             schema.Optional('buildScript') : str,
             schema.Optional('buildScriptBash') : str,
             schema.Optional('buildScriptPwsh') : str,
             schema.Optional('buildSetup') : str,
             schema.Optional('buildSetupBash') : str,
             schema.Optional('buildSetupPwsh') : str,
+            schema.Optional('packageFinalize') : str,
+            schema.Optional('packageFinalizeBash') : str,
+            schema.Optional('packageFinalizePwsh') : str,
             schema.Optional('packageScript') : str,
             schema.Optional('packageScriptBash') : str,
             schema.Optional('packageScriptPwsh') : str,

test/unit/test_input_recipe.py

@@ -817,3 +817,110 @@ def testIfExpr(self):
         self.assertTrue(p.isShared())
         p = self.parseAndPrepare(recipe, env={"ENABLE" : "no"}).getPackageStep()
         self.assertFalse(p.isShared())
+
+
+# Added in Python 3.10: itertools.pairwise
+def pairwise(seq):
+    prev = seq[0]
+    for i in seq[1:]:
+        yield (prev, i)
+        prev = i
+
+class TestFinalizeScripts(RecipeCommon, TestCase):
+
+    def testOrder(self):
+        """Finalize scripts are evaluated in reverse order"""
+        recipe = {
+            "inherit" : ["a", "b"],
+            "packageScript" : "RECIPE-SCRIPT",
+            "packageFinalize" : "RECIPE-FINALIZE",
+        }
+        classes = {
+            "a" : {
+                "packageScript" : "CLASS-A-SCRIPT",
+                "packageFinalize" : "CLASS-A-FINALIZE",
+            },
+            "b" : {
+                "packageScript" : "CLASS-B-SCRIPT",
+                "packageFinalize" : "CLASS-B-FINALIZE",
+            }
+        }
+
+        mainScript = self.parseAndPrepare(recipe, classes).getPackageStep().getMainScript()
+
+        snippets = (
+            "CLASS-A-SCRIPT",
+            "CLASS-B-SCRIPT",
+            "RECIPE-SCRIPT",
+            "RECIPE-FINALIZE",
+            "CLASS-B-FINALIZE",
+            "CLASS-A-FINALIZE",
+        )
+
+        for s in snippets:
+            self.assertIn(s, mainScript)
+
+        indexes = [ mainScript.index(s) for s in snippets ]
+        diffs = [ b-a for a, b in pairwise(indexes) ]
+        self.assertEqual(len(indexes), len(diffs)+1)
+        self.assertTrue(all(d > 0 for d in diffs))
+
+    def testCheckoutDeterministic(self):
+        """checkoutDeterministic applies equally to checkoutScript and checkoutFinalize"""
+
+        # checkoutFinalize alone -> indeterministic
+        recipe = {
+            "checkoutFinalize" : "asdf",
+            "buildScript" : "asdf",
+            "packageScript" : "asdf",
+        }
+        c = self.parseAndPrepare(recipe).getCheckoutStep()
+        self.assertFalse(c.isDeterministic())
+
+        # checkoutScript and checkoutFinalize -> indeterministic
+        recipe = {
+            "checkoutScript" : "asdf",
+            "checkoutFinalize" : "asdf",
+            "buildScript" : "asdf",
+            "packageScript" : "asdf",
+        }
+        c = self.parseAndPrepare(recipe).getCheckoutStep()
+        self.assertFalse(c.isDeterministic())
+
+        # checkoutFinalize can be overridden by checkoutDeterministic just
+        # like checkoutScript
+
+        recipe = {
+            "checkoutFinalize" : "asdf",
+            "checkoutDeterministic" : True,
+            "buildScript" : "asdf",
+            "packageScript" : "asdf",
+        }
+        c = self.parseAndPrepare(recipe).getCheckoutStep()
+        self.assertTrue(c.isDeterministic())
+
+        recipe = {
+            "checkoutScript" : "asdf",
+            "checkoutFinalize" : "asdf",
+            "checkoutDeterministic" : True,
+            "buildScript" : "asdf",
+            "packageScript" : "asdf",
+        }
+        c = self.parseAndPrepare(recipe).getCheckoutStep()
+        self.assertTrue(c.isDeterministic())
+
+    def testCheckoutDeterministicInherit(self):
+        """checkoutFinalize of inherited classes also affects determinism"""
+
+        recipe = {
+            "inherit" : [ "bar" ],
+            "buildScript" : "asdf",
+            "packageScript" : "asdf",
+        }
+        classes = {
+            "bar" : {
+                "checkoutFinalize" : "asdf",
+            },
+        }
+        c = self.parseAndPrepare(recipe, classes).getCheckoutStep()
+        self.assertFalse(c.isDeterministic())