@@ -16,10 +16,7 @@ use cudarc::driver::{CudaSlice, CudaStream, LaunchConfig, PushKernelArg};
1616
1717use crate :: Result ;
1818use crate :: device:: { Backend , backend} ;
19- use crate :: merkle:: {
20- keccak_launch_cfg, launch_keccak_base, launch_keccak_base_row_pair, launch_keccak_ext3,
21- launch_keccak_ext3_row_pair,
22- } ;
19+ use crate :: merkle:: { keccak_launch_cfg, launch_keccak_base, launch_keccak_base_row_pair} ;
2320use crate :: ntt:: run_ntt_body;
2421
2522/// Goldilocks `TWO_ADICITY = 32` puts the theoretical domain ceiling at
@@ -969,35 +966,6 @@ pub fn coset_lde_batch_base_into_with_leaf_hash(
969966 . map ( |_| ( ) )
970967}
971968
972- /// Like `coset_lde_batch_base_into_with_leaf_hash`, but also builds the full
973- /// row-pair Merkle tree on device and returns the `2*(lde_size/2) - 1` node
974- /// buffer back to the caller in `merkle_nodes_out` (byte length
975- /// `(2*(lde_size/2) - 1) * 32`).
976- ///
977- /// The leaf hashes are never exposed to the caller — they stay on device and
978- /// feed straight into the pair-hash tree kernel, avoiding the
979- /// pinned→pageable→pinned round-trip that the separate-step GPU tree build
980- /// would pay.
981- pub fn coset_lde_batch_base_into_with_merkle_tree (
982- columns : & [ & [ u64 ] ] ,
983- blowup_factor : usize ,
984- weights : & [ u64 ] ,
985- outputs : & mut [ & mut [ u64 ] ] ,
986- merkle_nodes_out : & mut [ u8 ] ,
987- ) -> Result < ( ) > {
988- coset_lde_batch_base_into_with_merkle_tree_inner (
989- columns,
990- blowup_factor,
991- weights,
992- outputs,
993- merkle_nodes_out,
994- KeccakCommit :: FullTree ,
995- false ,
996- 2 ,
997- )
998- . map ( |_| ( ) )
999- }
1000-
1001969#[ allow( clippy:: too_many_arguments) ]
1002970fn coset_lde_batch_base_into_with_merkle_tree_inner (
1003971 columns : & [ & [ u64 ] ] ,
@@ -1180,229 +1148,6 @@ fn coset_lde_batch_base_into_with_merkle_tree_inner(
11801148 }
11811149}
11821150
1183- /// Ext3 variant of `coset_lde_batch_base_into_with_leaf_hash`: fused LDE +
1184- /// row-pair Keccak-256 leaf hashing over ext3 columns. Thin wrapper over
1185- /// `coset_lde_batch_ext3_into_with_merkle_tree_inner` with `LeavesOnly`.
1186- pub fn coset_lde_batch_ext3_into_with_leaf_hash (
1187- columns : & [ & [ u64 ] ] ,
1188- n : usize ,
1189- blowup_factor : usize ,
1190- weights : & [ u64 ] ,
1191- outputs : & mut [ & mut [ u64 ] ] ,
1192- hashed_leaves_out : & mut [ u8 ] ,
1193- ) -> Result < ( ) > {
1194- coset_lde_batch_ext3_into_with_merkle_tree_inner (
1195- columns,
1196- n,
1197- blowup_factor,
1198- weights,
1199- outputs,
1200- hashed_leaves_out,
1201- KeccakCommit :: LeavesOnly ,
1202- false ,
1203- 2 ,
1204- )
1205- . map ( |_| ( ) )
1206- }
1207-
1208- /// Ext3 variant of the fused `coset_lde_batch_base_into_with_merkle_tree`.
1209- /// LDE + leaf hashing + inner-tree build, all on device; D2Hs only the LDE
1210- /// evaluations and the full `2*(lde_size/2) - 1` row-pair node buffer.
1211- pub fn coset_lde_batch_ext3_into_with_merkle_tree (
1212- columns : & [ & [ u64 ] ] ,
1213- n : usize ,
1214- blowup_factor : usize ,
1215- weights : & [ u64 ] ,
1216- outputs : & mut [ & mut [ u64 ] ] ,
1217- merkle_nodes_out : & mut [ u8 ] ,
1218- ) -> Result < ( ) > {
1219- coset_lde_batch_ext3_into_with_merkle_tree_inner (
1220- columns,
1221- n,
1222- blowup_factor,
1223- weights,
1224- outputs,
1225- merkle_nodes_out,
1226- KeccakCommit :: FullTree ,
1227- false ,
1228- 2 ,
1229- )
1230- . map ( |_| ( ) )
1231- }
1232-
1233- #[ allow( clippy:: too_many_arguments) ]
1234- fn coset_lde_batch_ext3_into_with_merkle_tree_inner (
1235- columns : & [ & [ u64 ] ] ,
1236- n : usize ,
1237- blowup_factor : usize ,
1238- weights : & [ u64 ] ,
1239- outputs : & mut [ & mut [ u64 ] ] ,
1240- nodes_out : & mut [ u8 ] ,
1241- commit : KeccakCommit ,
1242- keep_device_buf : bool ,
1243- // 1 = one leaf per bit-reversed row; 2 = one leaf per row pair (2i, 2i+1),
1244- // matching the CPU `commit_bit_reversed(.., 2)` used for the trace commit.
1245- rows_per_leaf : usize ,
1246- ) -> Result < Option < GpuLdeExt3 > > {
1247- if columns. is_empty ( ) {
1248- assert_eq ! ( outputs. len( ) , 0 ) ;
1249- return Ok ( None ) ;
1250- }
1251- // (is_power_of_two returns false for 0).
1252- if n == 0 {
1253- return Ok ( None ) ;
1254- }
1255- let m = columns. len ( ) ;
1256- assert_eq ! ( outputs. len( ) , m) ;
1257- assert ! ( n. is_power_of_two( ) ) ;
1258- assert_eq ! ( weights. len( ) , n) ;
1259- assert ! ( blowup_factor. is_power_of_two( ) ) ;
1260- for c in columns. iter ( ) {
1261- assert_eq ! ( c. len( ) , 3 * n) ;
1262- }
1263- let lde_size = n * blowup_factor;
1264- assert_u32_domain (
1265- lde_size,
1266- "coset_lde_batch_ext3_into_with_merkle_tree lde_size" ,
1267- ) ;
1268- for o in outputs. iter ( ) {
1269- assert_eq ! ( o. len( ) , 3 * lde_size) ;
1270- }
1271- assert ! (
1272- rows_per_leaf == 1 || rows_per_leaf == 2 ,
1273- "rows_per_leaf must be 1 or 2"
1274- ) ;
1275- assert_eq ! ( lde_size % rows_per_leaf, 0 ) ;
1276- let num_leaves = lde_size / rows_per_leaf;
1277- let nodes_dev_bytes = commit. total_nodes_bytes ( num_leaves) ;
1278- assert_eq ! ( nodes_out. len( ) , nodes_dev_bytes) ;
1279- let log_n = n. trailing_zeros ( ) as u64 ;
1280- let log_lde = lde_size. trailing_zeros ( ) as u64 ;
1281-
1282- let mb = 3 * m;
1283- let be = backend ( ) ?;
1284- let stream = be. next_stream ( ) ;
1285- let staging_slot = be. pinned_staging ( ) ;
1286-
1287- let mut staging = staging_slot. lock ( ) . unwrap ( ) ;
1288- staging. ensure_capacity ( mb * lde_size, & be. ctx ) ?;
1289- let pinned = unsafe { staging. as_mut_slice ( mb * lde_size) } ;
1290-
1291- pack_ext3_to_pinned_slabs ( columns, pinned, n) ;
1292-
1293- let mut buf = stream. alloc_zeros :: < u64 > ( mb * lde_size) ?;
1294- for s in 0 ..mb {
1295- let mut dst = buf. slice_mut ( s * lde_size..s * lde_size + n) ;
1296- stream. memcpy_htod ( & pinned[ s * n..s * n + n] , & mut dst) ?;
1297- }
1298-
1299- let inv_tw = be. inv_twiddles_for ( log_n) ?;
1300- let fwd_tw = be. fwd_twiddles_for ( log_lde) ?;
1301- let weights_dev = stream. clone_htod ( weights) ?;
1302-
1303- let n_u64 = n as u64 ;
1304- let lde_u64 = lde_size as u64 ;
1305- let col_stride_u64 = lde_size as u64 ;
1306- let mb_u32 = mb as u32 ;
1307-
1308- launch_bit_reverse_batched (
1309- stream. as_ref ( ) ,
1310- be,
1311- & mut buf,
1312- n_u64,
1313- log_n,
1314- col_stride_u64,
1315- mb_u32,
1316- ) ?;
1317- run_batched_ntt_body (
1318- stream. as_ref ( ) ,
1319- & mut buf,
1320- inv_tw. as_ref ( ) ,
1321- n_u64,
1322- log_n,
1323- col_stride_u64,
1324- mb_u32,
1325- ) ?;
1326- launch_pointwise_mul_batched (
1327- stream. as_ref ( ) ,
1328- be,
1329- & mut buf,
1330- & weights_dev,
1331- n_u64,
1332- col_stride_u64,
1333- mb_u32,
1334- ) ?;
1335- launch_bit_reverse_batched (
1336- stream. as_ref ( ) ,
1337- be,
1338- & mut buf,
1339- lde_u64,
1340- log_lde,
1341- col_stride_u64,
1342- mb_u32,
1343- ) ?;
1344- run_batched_ntt_body (
1345- stream. as_ref ( ) ,
1346- & mut buf,
1347- fwd_tw. as_ref ( ) ,
1348- lde_u64,
1349- log_lde,
1350- col_stride_u64,
1351- mb_u32,
1352- ) ?;
1353-
1354- // Allocate device output buffer (LeavesOnly -> num_leaves*32; FullTree ->
1355- // (2*num_leaves - 1)*32). Leaf kernel writes to the leaves slab; the
1356- // inner-tree pass (when present) fills the head.
1357- let mut nodes_dev = unsafe { stream. alloc :: < u8 > ( nodes_dev_bytes) } ?;
1358- let leaves_offset_bytes = commit. leaves_offset_bytes ( num_leaves) ;
1359- {
1360- let mut leaves_view =
1361- nodes_dev. slice_mut ( leaves_offset_bytes..leaves_offset_bytes + num_leaves * 32 ) ;
1362- if rows_per_leaf == 2 {
1363- launch_keccak_ext3_row_pair (
1364- stream. as_ref ( ) ,
1365- & buf,
1366- col_stride_u64,
1367- m as u64 ,
1368- lde_u64,
1369- & mut leaves_view,
1370- ) ?;
1371- } else {
1372- launch_keccak_ext3 (
1373- stream. as_ref ( ) ,
1374- & buf,
1375- col_stride_u64,
1376- m as u64 ,
1377- lde_u64,
1378- & mut leaves_view,
1379- ) ?;
1380- }
1381- }
1382-
1383- if commit == KeccakCommit :: FullTree {
1384- crate :: merkle:: build_inner_tree_levels ( stream. as_ref ( ) , be, & mut nodes_dev, num_leaves) ?;
1385- }
1386-
1387- // D2H LDE (mb * lde_size u64) and tree/leaves nodes.
1388- stream. memcpy_dtoh ( & buf, & mut pinned[ ..mb * lde_size] ) ?;
1389- d2h_bytes_via_pinned_hashes ( & stream, be, & nodes_dev, nodes_out) ?;
1390-
1391- unpack_pinned_slabs_to_ext3 ( pinned, outputs, lde_size) ;
1392- drop ( staging) ;
1393-
1394- if keep_device_buf {
1395- Ok ( Some ( GpuLdeExt3 {
1396- buf : Arc :: new ( buf) ,
1397- m,
1398- lde_size,
1399- } ) )
1400- } else {
1401- drop ( buf) ;
1402- Ok ( None )
1403- }
1404- }
1405-
14061151/// Batched ext3 polynomial → coset evaluation.
14071152///
14081153/// Input: M ext3 columns of `n` coefficients each (interleaved, 3n u64).
0 commit comments