Migrate to experimental TxOut in createCompatibleTx

Adapt to cardano-api PR #1209, which removes the legacy 'TxOut CtxTx era'
from the Compatible and Experimental APIs:

  * 'createCompatibleTx' now takes '[Exp.TxOut (ShelleyLedgerEra era)]'
    plus a new 'Map L.DataHash (L.Data ...)' argument carrying any
    supplemental datum bodies. The legacy 'TxOut CtxTx era' bundled
    supplemental datums inside outputs; 'Exp.TxOut' only carries the
    datum hash, so callers thread the full datum bodies in explicitly.
  * The bridge helpers 'fromLegacyTxOut', 'legacyDatumToDatum',
    'supplementalDatumFromLegacy', 'toLedgerDatum', and the
    'DatumDecodingError' type are deleted from 'Cardano.Api.Experimental.Tx'.

Migration:

  * 'mkTxOut' and 'toTxOutInAnyEra' now return
    '(Exp.TxOut (ShelleyLedgerEra era), Map DataHash (L.Data ...))'
    directly, building the legacy 'TxOut CtxTx era' internally only as
    a stepping stone for 'toShelleyTxOutAny'.
  * 'createCompatibleTx' call site in 'Compatible/Transaction/Run.hs'
    folds the per-output supplemental datums via 'Map.unions' and
    passes them as the new argument.
  * 'toTxOutInEra' and 'toTxOutInShelleyBasedEra' in
    'EraBased/Transaction/Run.hs' delegate directly to 'mkTxOut' (the
    deleted 'fromLegacyTxOut' is gone).
  * 'TxCmdDatumDecodingError' is removed since the underlying
    'Exp.DatumDecodingError' is gone and the new conversion is total.

Temporary 'cabal.project' additions (to be removed once #1209 is
merged and the next cardano-api is published to CHaP):

  * 'source-repository-package' pointing at the PR branch so CI can
    build against the unpublished API.
  * Per-package '-Wwarn=deprecations -Wwarn=unused-imports' for
    cardano-cli. The PR branch is several commits ahead of
    cardano-api-11.1.0.0 and surfaces unrelated TxBody/TxBodyContent
    deprecations (cardano-api PR #1200) plus a redundant
    'Cardano.Ledger.Core' import in 'Cardano.CLI.Read' that became
    visible after upstream re-exports widened. Both are separate
    cleanups out of scope for this PR.

Author: Jimbo4350

cabal.project

@@ -26,6 +26,29 @@ packages:
 program-options
   ghc-options: -Werror
 
+-- TEMPORARY: build against cardano-api PR #1209
+-- (jordan/remove-legacy-txout-from-compat-and-experimental).
+-- The PR removes legacy TxOut from the Compatible and Experimental APIs,
+-- which this branch migrates cardano-cli to. Remove this stanza and the
+-- per-package warning downgrades below once #1209 is merged and the next
+-- cardano-api release is published to CHaP.
+source-repository-package
+    type: git
+    location: https://github.com/IntersectMBO/cardano-api
+    tag: d848cec1ba4769b9daf4ac5ac9ed61a3f76c8042
+    subdir:
+      cardano-api
+      cardano-api-gen
+
+-- TEMPORARY: the PR branch is several commits ahead of cardano-api-11.1.0.0
+-- and surfaces unrelated TxBody/TxBodyContent deprecations (cardano-api PR
+-- #1200) plus a redundant Cardano.Ledger.Core import in Cardano.CLI.Read
+-- that became visible after upstream re-exports widened. Both are separate
+-- cardano-cli cleanups; downgrade to warnings here so the migration build
+-- is not blocked by them. Remove once the underlying cleanups land.
+package cardano-cli
+  ghc-options: -Wwarn=deprecations -Wwarn=unused-imports
+
 package crypton
   -- Using RDRAND instead of /dev/urandom as an entropy source for key
   -- generation is dubious. Set the flag so we use /dev/urandom by default.

cardano-cli/src/Cardano/CLI/Compatible/Transaction/Run.hs

@@ -38,6 +38,7 @@ import Cardano.CLI.Type.Common
 
 import Control.Monad
 import Data.Map.Ordered.Strict qualified as OMap
+import Data.Map.Strict qualified as Map
 import Lens.Micro
 
 runCompatibleTransactionCmd
@@ -60,7 +61,9 @@ runCompatibleTransactionCmd
     ) = shelleyBasedEraConstraints sbe $ do
     sks <- mapM (fromEitherIOCli . readWitnessSigningData) witnesses
 
-    allOuts <- mapM (toTxOutInAnyEra sbe) outs
+    outsAndDatums <- mapM (toTxOutInAnyEra sbe) outs
+    let allOuts = map fst outsAndDatums
+        extraDatums = Map.unions (map snd outsAndDatums)
 
     certFilesAndMaybeScriptWits <-
       readCertificateScriptWitnesses' sbe certificates
@@ -107,7 +110,7 @@ runCompatibleTransactionCmd
 
     transaction@(ShelleyTx _ ledgerTx) <-
       fromEitherCli $
-        createCompatibleTx sbe ins allOuts fee protocolUpdates votes txCerts
+        createCompatibleTx sbe ins allOuts extraDatums fee protocolUpdates votes txCerts
 
     let txBody = ledgerTx ^. L.bodyTxL
 

cardano-cli/src/Cardano/CLI/Compatible/Transaction/TxOut.hs

@@ -1,3 +1,5 @@
+{-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE GADTs #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE RankNTypes #-}
 
@@ -8,28 +10,44 @@ module Cardano.CLI.Compatible.Transaction.TxOut
 where
 
 import Cardano.Api
+import Cardano.Api.Experimental.Tx qualified as Exp
+import Cardano.Api.Ledger qualified as L
 
 import Cardano.CLI.Compatible.Exception
 import Cardano.CLI.EraBased.Script.Read.Common
 import Cardano.CLI.Orphan ()
 import Cardano.CLI.Read
 import Cardano.CLI.Type.Common
+import Cardano.Ledger.Hashes (DataHash)
+import Cardano.Ledger.Plutus.Data qualified as L
+
+import Data.Map.Strict (Map)
+import Data.Map.Strict qualified as Map
 
 toTxOutInAnyEra
   :: ShelleyBasedEra era
   -> TxOutAnyEra
-  -> CIO e (TxOut CtxTx era)
+  -> CIO e (Exp.TxOut (ShelleyLedgerEra era), Map DataHash (L.Data (ShelleyLedgerEra era)))
 toTxOutInAnyEra era (TxOutAnyEra addr' val' mDatumHash refScriptFp) = do
   let addr = anyAddressInShelleyBasedEra era addr'
   mkTxOut era addr val' mDatumHash refScriptFp
 
+-- | Build an output for a transaction body. Produces the experimental
+-- 'Exp.TxOut' plus any supplemental datum bodies that the caller-supplied
+-- datum carries. The legacy 'TxOut CtxTx era' bundled supplemental datums
+-- inside outputs; 'Exp.TxOut' only carries the datum hash, so callers thread
+-- the full datum bodies in separately (e.g. via 'createCompatibleTx').
+--
+-- The legacy 'TxOut CtxTx era' is used internally as a stepping stone to
+-- reuse the api's 'toShelleyTxOutAny' field-level conversion logic; it is
+-- not exposed.
 mkTxOut
   :: ShelleyBasedEra era
   -> AddressInEra era
   -> Value
   -> TxOutDatumAnyEra
   -> ReferenceScriptAnyEra
-  -> CIO e (TxOut CtxTx era)
+  -> CIO e (Exp.TxOut (ShelleyLedgerEra era), Map DataHash (L.Data (ShelleyLedgerEra era)))
 mkTxOut sbe addr val' mDatumHash refScriptFp = do
   let era = toCardanoEra sbe
   val <- toTxOutValueInShelleyBasedEra sbe val'
@@ -46,7 +64,19 @@ mkTxOut sbe addr val' mDatumHash refScriptFp = do
       (`getReferenceScript` refScriptFp)
       era
 
-  pure $ TxOut addr val datum refScript
+  let legacyTxOut = TxOut addr val datum refScript
+  pure $
+    shelleyBasedEraConstraints sbe $
+      (Exp.TxOut (toShelleyTxOutAny sbe legacyTxOut), supplementalsOf datum)
+ where
+  supplementalsOf
+    :: L.Era (ShelleyLedgerEra era)
+    => TxOutDatum CtxTx era
+    -> Map DataHash (L.Data (ShelleyLedgerEra era))
+  supplementalsOf (TxOutSupplementalDatum _ h) =
+    let ld = toAlonzoData h
+     in Map.singleton (L.hashData ld) ld
+  supplementalsOf _ = mempty
 
 toTxOutValueInShelleyBasedEra
   :: ShelleyBasedEra era

cardano-cli/src/Cardano/CLI/EraBased/Transaction/Run.hs

@@ -366,13 +366,15 @@ runTransactionBuildCmd
           else writeTxFileTextEnvelope eon fpath noWitTx
 
 toTxOutInEra
-  :: Exp.IsEra era
+  :: forall era e
+   . Exp.IsEra era
   => TxOutAnyEra
   -> CIO e (Exp.TxOut (Exp.LedgerEra era), Map.Map DataHash (L.Data (Exp.LedgerEra era)))
 toTxOutInEra (TxOutAnyEra addr' val' mDatumHash refScriptFp) = do
-  let addr = anyAddressInShelleyBasedEra (convert Exp.useEra) addr'
-  o <- mkTxOut (convert Exp.useEra) addr val' mDatumHash refScriptFp
-  fromEitherCli $ Exp.fromLegacyTxOut o
+  let sbe = convert (Exp.useEra @era)
+      addr = anyAddressInShelleyBasedEra sbe addr'
+  obtainCommonConstraints (Exp.useEra @era) $
+    mkTxOut sbe addr val' mDatumHash refScriptFp
 
 runTransactionBuildEstimateCmd
   :: forall era e
@@ -1176,14 +1178,15 @@ getAllReferenceInputs
       ]
 
 toTxOutInShelleyBasedEra
-  :: Exp.IsEra era
+  :: forall era e
+   . Exp.IsEra era
   => TxOutShelleyBasedEra
   -> CIO e (Exp.TxOut (Exp.LedgerEra era), Map.Map DataHash (L.Data (Exp.LedgerEra era)))
 toTxOutInShelleyBasedEra (TxOutShelleyBasedEra addr' val' mDatumHash refScriptFp) = do
-  let sbe = convert Exp.useEra
+  let sbe = convert (Exp.useEra @era)
       addr = shelleyAddressInEra sbe addr'
-  o <- mkTxOut sbe addr val' mDatumHash refScriptFp
-  fromEitherCli $ Exp.fromLegacyTxOut o
+  obtainCommonConstraints (Exp.useEra @era) $
+    mkTxOut sbe addr val' mDatumHash refScriptFp
 
 -- TODO: Currently we specify the policyId with the '--mint' option on the cli
 -- and we added a separate '--policy-id' parser that parses the policy id for the

cardano-cli/src/Cardano/CLI/Type/Error/TxCmdError.hs

@@ -42,7 +42,6 @@ data AnyTxBodyErrorAutoBalance where
 
 data TxCmdError
   = TxCmdCBORDecodeError !CBOR.DecoderError
-  | TxCmdDatumDecodingError Exp.DatumDecodingError
   | TxCmdProtocolParamsError ProtocolParamsError
   | forall era. LostScriptWitnesses
       [Exp.AnyIndexedPlutusScriptWitness (Exp.LedgerEra era)]
@@ -199,8 +198,6 @@ renderTxCmdError = \case
       , pretty (length after)
       , "."
       ]
-  TxCmdDatumDecodingError err ->
-    "Error decoding datum: " <> pshow err
 
 prettyPolicyIdList :: [PolicyId] -> Doc ann
 prettyPolicyIdList =