Migrate to experimental TxOut in createCompatibleTx

Adapt to cardano-api PR #1209, which removes the legacy 'TxOut CtxTx era'
from the Compatible and Experimental APIs:

  * createCompatibleTx now takes '[Exp.TxOut (ShelleyLedgerEra era)]' plus
    a new 'Map L.DataHash (L.Data ...)' of supplemental datum bodies. The
    legacy 'TxOut CtxTx era' bundled supplemental datums inside outputs;
    'Exp.TxOut' only carries the datum hash, so callers thread the full
    datum bodies in explicitly.
  * The bridge helpers in 'Cardano.Api.Experimental.Tx'
    (fromLegacyTxOut, legacyDatumToDatum, supplementalDatumFromLegacy,
    toLedgerDatum, DatumDecodingError) are deleted.

Changes here:

  * Add a 'convertLegacyTxOut' helper in
    'Cardano.CLI.Compatible.Transaction.TxOut' that maps each legacy
    'TxOut CtxTx era' to '(Exp.TxOut, Map DataHash (L.Data ...))' using
    'toShelleyTxOutAny' (which already strips supplemental datums to
    their hash in the ledger TxOut) and pulls the supplemental datum
    bodies into the map.
  * Update the 'createCompatibleTx' call in
    'Cardano.CLI.Compatible.Transaction.Run' to convert outs at the
    boundary and pass the folded 'extraDatums' map.
  * Replace 'Exp.fromLegacyTxOut' in 'toTxOutInEra' and
    'toTxOutInShelleyBasedEra' with 'convertLegacyTxOut', using
    'obtainCommonConstraints' to bridge 'ShelleyLedgerEra ~ LedgerEra'.
  * Drop 'TxCmdDatumDecodingError' from 'TxCmdError' since the
    underlying 'Exp.DatumDecodingError' is gone and the new conversion
    is total.

Temporary cabal.project additions (to be removed once #1209 is merged
and the next cardano-api is published to CHaP):

  * source-repository-package pointing at the PR branch, so CI can
    build against the unpublished API.
  * Per-package '-Wwarn=deprecations -Wwarn=unused-imports' for
    cardano-cli. The PR branch is several commits ahead of
    cardano-api-11.1.0.0 and surfaces unrelated TxBody/TxBodyContent
    deprecations (cardano-api PR #1200) plus a redundant
    'Cardano.Ledger.Core' import in 'Cardano.CLI.Read' that became
    visible after upstream re-exports widened. Both are separate
    cardano-cli cleanups out of scope for this PR.

Author: Jimbo4350

cabal.project

@@ -26,6 +26,29 @@ packages:
 program-options
   ghc-options: -Werror
 
+-- TEMPORARY: build against cardano-api PR #1209
+-- (jordan/remove-legacy-txout-from-compat-and-experimental).
+-- The PR removes legacy TxOut from the Compatible and Experimental APIs,
+-- which this branch migrates cardano-cli to. Remove this stanza and the
+-- per-package warning downgrades below once #1209 is merged and the next
+-- cardano-api release is published to CHaP.
+source-repository-package
+    type: git
+    location: https://github.com/IntersectMBO/cardano-api
+    tag: d848cec1ba4769b9daf4ac5ac9ed61a3f76c8042
+    subdir:
+      cardano-api
+      cardano-api-gen
+
+-- TEMPORARY: the PR branch is several commits ahead of cardano-api-11.1.0.0
+-- and surfaces unrelated TxBody/TxBodyContent deprecations (cardano-api PR
+-- #1200) plus a redundant Cardano.Ledger.Core import in Cardano.CLI.Read
+-- that became visible after upstream re-exports widened. Both are separate
+-- cardano-cli cleanups; downgrade to warnings here so the migration build
+-- is not blocked by them. Remove once the underlying cleanups land.
+package cardano-cli
+  ghc-options: -Wwarn=deprecations -Wwarn=unused-imports
+
 package crypton
   -- Using RDRAND instead of /dev/urandom as an entropy source for key
   -- generation is dubious. Set the flag so we use /dev/urandom by default.

cardano-cli/src/Cardano/CLI/Compatible/Transaction/Run.hs

@@ -38,6 +38,7 @@ import Cardano.CLI.Type.Common
 
 import Control.Monad
 import Data.Map.Ordered.Strict qualified as OMap
+import Data.Map.Strict qualified as Map
 import Lens.Micro
 
 runCompatibleTransactionCmd
@@ -60,7 +61,10 @@ runCompatibleTransactionCmd
     ) = shelleyBasedEraConstraints sbe $ do
     sks <- mapM (fromEitherIOCli . readWitnessSigningData) witnesses
 
-    allOuts <- mapM (toTxOutInAnyEra sbe) outs
+    legacyOuts <- mapM (toTxOutInAnyEra sbe) outs
+    let convertedOuts = map (convertLegacyTxOut sbe) legacyOuts
+        allOuts = map fst convertedOuts
+        extraDatums = Map.unions (map snd convertedOuts)
 
     certFilesAndMaybeScriptWits <-
       readCertificateScriptWitnesses' sbe certificates
@@ -107,7 +111,7 @@ runCompatibleTransactionCmd
 
     transaction@(ShelleyTx _ ledgerTx) <-
       fromEitherCli $
-        createCompatibleTx sbe ins allOuts fee protocolUpdates votes txCerts
+        createCompatibleTx sbe ins allOuts extraDatums fee protocolUpdates votes txCerts
 
     let txBody = ledgerTx ^. L.bodyTxL
 

cardano-cli/src/Cardano/CLI/Compatible/Transaction/TxOut.hs

@@ -1,19 +1,29 @@
+{-# LANGUAGE FlexibleContexts #-}
+{-# LANGUAGE GADTs #-}
 {-# LANGUAGE LambdaCase #-}
 {-# LANGUAGE RankNTypes #-}
 
 module Cardano.CLI.Compatible.Transaction.TxOut
   ( mkTxOut
   , toTxOutInAnyEra
+  , convertLegacyTxOut
   )
 where
 
 import Cardano.Api
+import Cardano.Api.Experimental.Tx qualified as Exp
+import Cardano.Api.Ledger qualified as L
 
 import Cardano.CLI.Compatible.Exception
 import Cardano.CLI.EraBased.Script.Read.Common
 import Cardano.CLI.Orphan ()
 import Cardano.CLI.Read
 import Cardano.CLI.Type.Common
+import Cardano.Ledger.Hashes (DataHash)
+import Cardano.Ledger.Plutus.Data qualified as L
+
+import Data.Map.Strict (Map)
+import Data.Map.Strict qualified as Map
 
 toTxOutInAnyEra
   :: ShelleyBasedEra era
@@ -23,6 +33,28 @@ toTxOutInAnyEra era (TxOutAnyEra addr' val' mDatumHash refScriptFp) = do
   let addr = anyAddressInShelleyBasedEra era addr'
   mkTxOut era addr val' mDatumHash refScriptFp
 
+-- | Convert a legacy 'TxOut CtxTx era' into the experimental 'Exp.TxOut' plus
+-- the supplemental datums it carried. The legacy 'TxOut CtxTx era' bundled
+-- supplemental datum bodies inside 'TxOutSupplementalDatum'; 'Exp.TxOut' only
+-- carries the datum hash, so the body is returned separately in the map for
+-- callers to thread into the witness set (e.g. via 'createCompatibleTx').
+convertLegacyTxOut
+  :: ShelleyBasedEra era
+  -> TxOut CtxTx era
+  -> (Exp.TxOut (ShelleyLedgerEra era), Map DataHash (L.Data (ShelleyLedgerEra era)))
+convertLegacyTxOut sbe tOut@(TxOut _ _ d _) =
+  shelleyBasedEraConstraints sbe $
+    (Exp.TxOut (toShelleyTxOutAny sbe tOut), supplementalsOf d)
+ where
+  supplementalsOf
+    :: L.Era (ShelleyLedgerEra era)
+    => TxOutDatum CtxTx era
+    -> Map DataHash (L.Data (ShelleyLedgerEra era))
+  supplementalsOf (TxOutSupplementalDatum _ h) =
+    let ld = toAlonzoData h
+     in Map.singleton (L.hashData ld) ld
+  supplementalsOf _ = mempty
+
 mkTxOut
   :: ShelleyBasedEra era
   -> AddressInEra era

cardano-cli/src/Cardano/CLI/EraBased/Transaction/Run.hs

@@ -366,13 +366,15 @@ runTransactionBuildCmd
           else writeTxFileTextEnvelope eon fpath noWitTx
 
 toTxOutInEra
-  :: Exp.IsEra era
+  :: forall era e
+   . Exp.IsEra era
   => TxOutAnyEra
   -> CIO e (Exp.TxOut (Exp.LedgerEra era), Map.Map DataHash (L.Data (Exp.LedgerEra era)))
 toTxOutInEra (TxOutAnyEra addr' val' mDatumHash refScriptFp) = do
-  let addr = anyAddressInShelleyBasedEra (convert Exp.useEra) addr'
-  o <- mkTxOut (convert Exp.useEra) addr val' mDatumHash refScriptFp
-  fromEitherCli $ Exp.fromLegacyTxOut o
+  let sbe = convert (Exp.useEra @era)
+      addr = anyAddressInShelleyBasedEra sbe addr'
+  o <- mkTxOut sbe addr val' mDatumHash refScriptFp
+  pure $ obtainCommonConstraints (Exp.useEra @era) $ convertLegacyTxOut sbe o
 
 runTransactionBuildEstimateCmd
   :: forall era e
@@ -1176,14 +1178,15 @@ getAllReferenceInputs
       ]
 
 toTxOutInShelleyBasedEra
-  :: Exp.IsEra era
+  :: forall era e
+   . Exp.IsEra era
   => TxOutShelleyBasedEra
   -> CIO e (Exp.TxOut (Exp.LedgerEra era), Map.Map DataHash (L.Data (Exp.LedgerEra era)))
 toTxOutInShelleyBasedEra (TxOutShelleyBasedEra addr' val' mDatumHash refScriptFp) = do
-  let sbe = convert Exp.useEra
+  let sbe = convert (Exp.useEra @era)
       addr = shelleyAddressInEra sbe addr'
   o <- mkTxOut sbe addr val' mDatumHash refScriptFp
-  fromEitherCli $ Exp.fromLegacyTxOut o
+  pure $ obtainCommonConstraints (Exp.useEra @era) $ convertLegacyTxOut sbe o
 
 -- TODO: Currently we specify the policyId with the '--mint' option on the cli
 -- and we added a separate '--policy-id' parser that parses the policy id for the

cardano-cli/src/Cardano/CLI/Type/Error/TxCmdError.hs

@@ -42,7 +42,6 @@ data AnyTxBodyErrorAutoBalance where
 
 data TxCmdError
   = TxCmdCBORDecodeError !CBOR.DecoderError
-  | TxCmdDatumDecodingError Exp.DatumDecodingError
   | TxCmdProtocolParamsError ProtocolParamsError
   | forall era. LostScriptWitnesses
       [Exp.AnyIndexedPlutusScriptWitness (Exp.LedgerEra era)]
@@ -199,8 +198,6 @@ renderTxCmdError = \case
       , pretty (length after)
       , "."
       ]
-  TxCmdDatumDecodingError err ->
-    "Error decoding datum: " <> pshow err
 
 prettyPolicyIdList :: [PolicyId] -> Doc ann
 prettyPolicyIdList =