cabal-install: call Cabal in-library

This architectural change ensures that we build packages using Cabal
library functions (using a Haskell library interface) instead of going
via the command-line interface of Setup.hs, as much as possible.

The main changes are in SetupWrapper: the old InternalMethod becomes
LibraryMethod, which builds the package by calling Cabal library functions.
This is used to build all packages unless build-type: Custom or there is
a Cabal library version mismatch which requires us to fall back to
compiling Setup.hs and running that.
The SelfExec method as well as forceExternalSetupMethod are removed:
they no longer have any purpose, as builds can now be carried out
concurrently thanks to 7b90583 and edb808a.
This change required a bit of GADT trickery to accomodate the fact that
configure returns a LocalBuildInfo which must then be passed to
subsequent phases, while with the old Setup interface everything returns
`IO ()` and communication is done through the filesystem (the local
build info file).

The new Distribution.Client.InLibrary module contains all the necessary
framework for taking information that cabal-install has and invoking
Cabal library functions to perform the appropriate action
(configure, build, test, bench, ...).
Most of these are pretty simple; the main difficulty is with configure
as we need to jump to the part of the Cabal configure code that
continues after figuring out information about the system (e.g. compiler
information, dependency information determined by the solver), to avoid
wasting work with Cabal rediscovering things that cabal-install already
knows. This required a bit of refactoring in Distribution.Simple.Configure.

Packages with 'build-type: Hooks' are now compiled with the in-library
method (instead of going via Setup.hs). This is achieved as follows:

  - The Cabal library provides 'Distribution.Simple.SetupHooks.HooksMain',
    which exposes `hooksMain :: SetupHooks -> IO ()`. This allows us
    to turn any definition of `SetupHooks` into an external hooks
    executable.
  - The new `hooks-exe` library, which `cabal-install` depends on,
    provides the logic for communicating with the external hooks
    executable (using the `CommunicationHandle` functionality from
    haskell/process#308).

This change has been extensively tested by compiling clc-stackage and
diffing the differences in LocalBuildInfo (i.e. the output of the
configure step).

Author: sheaf

.hlint.yaml

@@ -37,6 +37,7 @@
 - ignore: {name: "Functor law", within: [Test.Laws]}
 - ignore: {name: "Monoid law, left identity", within: [Test.Laws, UnitTests.Distribution.Utils.NubList]}
 - ignore: {name: "Monoid law, right identity", within: [Test.Laws, UnitTests.Distribution.Utils.NubList]}
+- ignore: {name: "Replace case with maybe", within: [Distribution.Client.InLibrary]}
 - ignore: {name: "Use fmap", within: [Distribution.Client.HttpUtils, Distribution.Simple.SrcDist]}
 
 - group:

Cabal/Cabal.cabal

@@ -48,8 +48,7 @@ library
     , directory  >= 1.2.7    && < 1.4
     , filepath   >= 1.4.2    && < 1.6
     , pretty     >= 1.1.1    && < 1.2
-    -- segfaults on older macOS versions #11465
-    , process    >= 1.2.1.0  && < 1.6.24 || == 1.6.26.0 || >= 1.6.26.2 && < 1.7
+    , process    >= 1.6.20.0 && < 1.6.24 || == 1.6.26.0 || >= 1.6.26.2 && < 1.7
     , time       >= 1.4.0.1  && < 1.16
 
   if os(windows)
@@ -165,6 +164,7 @@ library
     Distribution.Simple.UHC
     Distribution.Simple.UserHooks
     Distribution.Simple.SetupHooks.Errors
+    Distribution.Simple.SetupHooks.HooksMain
     Distribution.Simple.SetupHooks.Internal
     Distribution.Simple.SetupHooks.Rule
     Distribution.Simple.Utils

Cabal/src/Distribution/Simple.hs

@@ -171,6 +171,15 @@ defaultMainWithSetupHooksArgs setupHooks verbHandles =
       , hscolourHook = setup_hscolourHook
       }
   where
+    preBuildHook =
+      case SetupHooks.preBuildComponentRules (SetupHooks.buildHooks setupHooks) of
+        Nothing -> const $ return []
+        Just pbcRules -> \pbci -> runPreBuildHooks verbHandles pbci pbcRules
+    postBuildHook =
+      case SetupHooks.postBuildComponentHook (SetupHooks.buildHooks setupHooks) of
+        Nothing -> const $ return ()
+        Just hk -> hk
+
     setup_confHook
       :: (GenericPackageDescription, HookedBuildInfo)
       -> ConfigFlags
@@ -188,13 +197,14 @@ defaultMainWithSetupHooksArgs setupHooks verbHandles =
       -> BuildFlags
       -> IO ()
     setup_buildHook pkg_descr lbi hooks flags =
-      build_setupHooks
-        (SetupHooks.buildHooks setupHooks)
-        verbHandles
-        pkg_descr
-        lbi
-        flags
-        (allSuffixHandlers hooks)
+      void $
+        build_setupHooks
+          (preBuildHook, postBuildHook)
+          verbHandles
+          pkg_descr
+          lbi
+          flags
+          (allSuffixHandlers hooks)
 
     setup_copyHook
       :: PackageDescription
@@ -229,14 +239,15 @@ defaultMainWithSetupHooksArgs setupHooks verbHandles =
       -> [String]
       -> IO ()
     setup_replHook pkg_descr lbi hooks flags args =
-      repl_setupHooks
-        (SetupHooks.buildHooks setupHooks)
-        verbHandles
-        pkg_descr
-        lbi
-        flags
-        (allSuffixHandlers hooks)
-        args
+      void $
+        repl_setupHooks
+          preBuildHook
+          verbHandles
+          pkg_descr
+          lbi
+          flags
+          (allSuffixHandlers hooks)
+          args
 
     setup_haddockHook
       :: PackageDescription
@@ -245,13 +256,14 @@ defaultMainWithSetupHooksArgs setupHooks verbHandles =
       -> HaddockFlags
       -> IO ()
     setup_haddockHook pkg_descr lbi hooks flags =
-      haddock_setupHooks
-        (SetupHooks.buildHooks setupHooks)
-        verbHandles
-        pkg_descr
-        lbi
-        (allSuffixHandlers hooks)
-        flags
+      void $
+        haddock_setupHooks
+          preBuildHook
+          verbHandles
+          pkg_descr
+          lbi
+          (allSuffixHandlers hooks)
+          flags
 
     setup_hscolourHook
       :: PackageDescription
@@ -260,13 +272,14 @@ defaultMainWithSetupHooksArgs setupHooks verbHandles =
       -> HscolourFlags
       -> IO ()
     setup_hscolourHook pkg_descr lbi hooks flags =
-      hscolour_setupHooks
-        (SetupHooks.buildHooks setupHooks)
-        verbHandles
-        pkg_descr
-        lbi
-        (allSuffixHandlers hooks)
-        flags
+      void $
+        hscolour_setupHooks
+          preBuildHook
+          verbHandles
+          pkg_descr
+          lbi
+          (allSuffixHandlers hooks)
+          flags
 
 -- | A customizable version of 'defaultMain'.
 defaultMainWithHooks :: UserHooks -> IO ()
@@ -930,12 +943,16 @@ simpleUserHooksWithHandles verbHandles =
     , testHook = defaultTestHook verbHandles
     , benchHook = defaultBenchHook verbHandles
     , cleanHook = \p _ _ f -> clean verbHandles p f
-    , hscolourHook = \p l h f -> hscolour_setupHooks SetupHooks.noBuildHooks verbHandles p l (allSuffixHandlers h) f
-    , haddockHook = \p l h f -> haddock_setupHooks SetupHooks.noBuildHooks verbHandles p l (allSuffixHandlers h) f
+    , hscolourHook = \p l h f -> void $ hscolour_setupHooks noBuildHooks verbHandles p l (allSuffixHandlers h) f
+    , haddockHook = \p l h f -> void $ haddock_setupHooks noBuildHooks verbHandles p l (allSuffixHandlers h) f
     , regHook = defaultRegHook verbHandles
     , unregHook = \p l _ f -> unregisterWithHandles verbHandles p l f
     }
   where
+    noBuildHooks pbci@(SetupHooks.PreBuildComponentInputs{SetupHooks.localBuildInfo = lbi}) =
+      builtinPreBuildHooks
+        (buildType (localPkgDescr lbi))
+        pbci
     finalChecks _args flags pkg_descr lbi =
       checkForeignDeps pkg_descr lbi (modifyVerbosityFlags lessVerbose verbosity)
       where
@@ -1155,13 +1172,14 @@ defaultBuildHook
   -> BuildFlags
   -> IO ()
 defaultBuildHook verbHandles pkg_descr localbuildinfo hooks flags =
-  build_setupHooks
-    SetupHooks.noBuildHooks
-    verbHandles
-    pkg_descr
-    localbuildinfo
-    flags
-    (allSuffixHandlers hooks)
+  void $
+    build_setupHooks
+      (builtinPreBuildHooks (buildType pkg_descr), const $ pure ())
+      verbHandles
+      pkg_descr
+      localbuildinfo
+      flags
+      (allSuffixHandlers hooks)
 
 defaultReplHook
   :: VerbosityHandles
@@ -1172,14 +1190,15 @@ defaultReplHook
   -> [String]
   -> IO ()
 defaultReplHook verbHandles pkg_descr localbuildinfo hooks flags args =
-  repl_setupHooks
-    SetupHooks.noBuildHooks
-    verbHandles
-    pkg_descr
-    localbuildinfo
-    flags
-    (allSuffixHandlers hooks)
-    args
+  void $
+    repl_setupHooks
+      (builtinPreBuildHooks (buildType pkg_descr))
+      verbHandles
+      pkg_descr
+      localbuildinfo
+      flags
+      (allSuffixHandlers hooks)
+      args
 
 defaultRegHook
   :: VerbosityHandles