diff --git a/aiter/configs/model_configs/glm5_fp4_tuned_fmoe.csv b/aiter/configs/model_configs/glm5_fp4_tuned_fmoe.csv index f2c824633f5..bc74003837b 100644 --- a/aiter/configs/model_configs/glm5_fp4_tuned_fmoe.csv +++ b/aiter/configs/model_configs/glm5_fp4_tuned_fmoe.csv @@ -32,10 +32,10 @@ gfx950,256,8192,6144,1024,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_ gfx950,256,16384,6144,1024,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,1295.9968,flydsl_moe1_afp4_wfp4_bf16_t128x128x256_w4,0.0%,1598.6393,flydsl_moe2_afp4_wfp4_bf16_t128x128x256_atomic_xcd4_persist,4.0%,2894.6361,0,0,0,1922.96,1780.09, gfx950,256,32768,6144,1024,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,2423.733,flydsl_moe1_afp4_wfp4_bf16_t128x128x256_w4,0.0%,3023.9599,flydsl_moe2_afp4_wfp4_bf16_t128x128x256_reduce_bnt2_xcd4_persist,0.6%,5447.6929,0,0,0,2043.54,1001.28, gfx950,256,1,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,13.7924,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w4_kw4,0.0%,9.5013,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce_bnt2,0.2%,23.2937,0,0,0,7.29,104121.49, -gfx950,256,2,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,16.4623,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w4_bnt0_kw2_fp4,1.1%,12.9256,flydsl_moe2_afp4_wfp4_bf16_t32x256x256_reduce,0.1%,29.3879,0,0,0,11.56,82530.33, -gfx950,256,4,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,28.643,flydsl_moe1_afp4_wfp4_bf16_t32x128x256_w4_bnt0,0.0%,17.9639,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce,0.1%,46.6069,0,0,0,14.58,52040.15, -gfx950,256,8,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,42.8562,flydsl_moe1_afp4_wfp4_bf16_t32x64x256_w4_fp4,0.9%,26.1469,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce,0.1%,69.0031,0,0,0,19.69,35150.65, -gfx950,256,16,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,64.4669,flydsl_moe1_afp4_wfp4_bf16_t32x128x256_w2,0.0%,39.6935,flydsl_moe2_afp4_wfp4_bf16_t32x256x256_reduce_bnt2,0.1%,104.1604,0,0,0,26.09,23287.65, +gfx950,256,2,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,16.3611,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_bnt0_kw2_fp4,1.1%,12.3604,flydsl_moe2_afp4_wfp4_bf16_t32x256x256_reduce_bnt2,0.1%,28.7215,0,0,0,11.83,84445.21, +gfx950,256,4,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,28.2911,moe_ck2stages_gemm1_64x32x32x128_1x1_MulABScaleShuffled_v3_Nswizzle0_Quant3_MulRoutedWeight0_silu_FP4X2_FP4X2_B16,0.0%,16.8336,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce_bnt2,0.1%,45.1247,0,0,0,15.06,53749.5, +gfx950,256,8,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,42.8909,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_xcd4_kw4_fp4,0.9%,25.2564,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce_bnt2,0.1%,68.1473,0,0,0,19.94,35592.07, +gfx950,256,16,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,61.2577,flydsl_moe1_afp4_wfp4_bf16_t32x128x256_w2_fp4,0.8%,39.6528,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_atomic_bnt2,1.9%,100.9105,0,0,0,26.93,24037.65, gfx950,256,32,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,89.9858,flydsl_moe1_afp4_wfp4_bf16_t32x128x256_w4_fp4,0.7%,54.6412,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_atomic_persist,2.1%,144.627,0,0,0,37.59,16773.81, gfx950,256,64,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,122.9066,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w4_fp4,0.7%,74.0593,flydsl_moe2_afp4_wfp4_bf16_t32x256x256_atomic_xcd4_persist_sbm64,2.1%,196.9659,0,0,0,55.2,12319.57, gfx950,256,128,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,132.5044,flydsl_moe1_afp4_wfp4_bf16_t32x128x256_w3_fp4,0.8%,77.995,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce_persist,0.1%,210.4994,0,0,0,103.29,11533.12, @@ -43,25 +43,25 @@ gfx950,256,256,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2 gfx950,256,512,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,143.7898,flydsl_moe1_afp4_wfp4_bf16_t32x64x256_w3_fp4,0.8%,89.9336,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_atomic_persist,2.1%,233.7234,0,0,0,372.12,10417.41, gfx950,256,1024,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,159.80810000000002,flydsl_moe1_afp4_wfp4_bf16_t64x64x256_w3,0.0%,114.103,flydsl_moe2_afp4_wfp4_bf16_t64x128x256_atomic_persist,2.1%,273.9111,0,0,0,635.05,8923.45, gfx950,256,2048,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,196.5692,flydsl_moe1_afp4_wfp4_bf16_t128x128x256,0.0%,195.9419,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2_persist_sbm128,0.1%,392.5111,0,0,0,886.32,6275.25, -gfx950,256,4096,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,279.3132,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w4_bnt0_xcd4,0.0%,337.0306,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2_persist,0.1%,616.3438,0,0,0,1128.89,4057.56, -gfx950,256,8192,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,422.7372,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w3_bnt0_xcd4,0.0%,592.4468,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2_xcd4,0.1%,1015.184,0,0,0,1370.76,2537.82, -gfx950,256,16384,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,693.3045999999999,flydsl_moe1_afp4_wfp4_bf16_t128x128x256_bnt0_xcd4,0.0%,1130.7309,flydsl_moe2_afp4_wfp4_bf16_t64x128x256_reduce_xcd4_persist_sbm128,0.1%,1824.0355,0,0,0,1525.81,1495.23, -gfx950,256,32768,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,1249.3197,flydsl_moe1_afp4_wfp4_bf16_t128x128x256_w3_bnt0_xcd4,0.0%,2168.6555,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2_xcd4_persist_sbm128,0.1%,3417.9752,0,0,0,1628.53,886.3, -gfx950,256,1,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,10.4024,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_kw4_fp4,0.4%,8.1043,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce,0.0%,18.5067,0,0,0,4.59,65527.43, -gfx950,256,2,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,13.7996,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_bnt0_kw4,0.0%,9.966,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce,0.0%,23.7656,0,0,0,7.15,51028.17, +gfx950,256,4096,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,280.272,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w2_bnt0_xcd4,0.0%,333.8801,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_xcd4,0.1%,614.1521,0,0,0,1132.92,4072.04, +gfx950,256,8192,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,419.9263,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w2_bnt0_xcd4,0.0%,593.7695,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2_xcd4_persist,0.1%,1013.6958,0,0,0,1372.77,2541.54, +gfx950,256,16384,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,677.1169,moe_ck2stages_gemm1_256x128x128x128_1x4_MulABScaleShuffled_v3_Nswizzle0_Quant3_MulRoutedWeight0_silu_FP4X2_FP4X2_B16,0.0%,1116.5329,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_xcd4_persist_sbm128,0.1%,1793.6498,0,0,0,1551.66,1520.56, +gfx950,256,32768,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,1221.4842,moe_ck2stages_gemm1_256x128x128x128_1x4_MulABScaleShuffled_v3_Nswizzle0_Quant3_MulRoutedWeight0_silu_FP4X2_FP4X2_B16,0.0%,2148.3747,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_xcd4_persist_sbm128,0.1%,3369.8589,0,0,0,1651.78,898.95, +gfx950,256,1,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,10.4658,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_kw4_fp4,0.4%,6.7805,moe_ck2stages_gemm2_256x32x128x128_1x4_MulABScaleExpertWeightShuffled_v3_Nswizzle0_Quant3_MulRoutedWeight1_FP4X2_FP4X2_B16,1.0%,17.2463,0,0,0,4.92,70316.33, +gfx950,256,2,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,14.527,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_bnt0_kw4,0.0%,8.9725,moe_ck2stages_gemm2_256x32x128x128_1x4_MulABScaleExpertWeightShuffled_v3_Nswizzle0_Quant3_MulRoutedWeight1_FP4X2_FP4X2_B16,1.1%,23.4995,0,0,0,7.23,51605.99, gfx950,256,4,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,16.1328,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w4_bnt0_kw2_fp4,0.4%,13.1423,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce_bnt2,0.0%,29.2751,0,0,0,11.61,41426.05, gfx950,256,8,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,24.2983,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w4_kw2_fp4,0.5%,17.294,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce,0.0%,41.5923,0,0,0,16.34,29159.86, -gfx950,256,16,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,32.9941,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w3_fp4,0.9%,24.2111,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce,0.0%,57.2052,0,0,0,23.76,21203.9, +gfx950,256,16,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,35.2725,flydsl_moe1_afp4_wfp4_bf16_t32x64x256_w2,0.0%,21.6687,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce_bnt2,0.0%,56.9412,0,0,0,23.87,21302.2, gfx950,256,32,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,46.9406,flydsl_moe1_afp4_wfp4_bf16_t32x64x256_w4_kw2_fp4,0.8%,31.5012,flydsl_moe2_afp4_wfp4_bf16_t32x256x256_reduce_persist,0.0%,78.4418,0,0,0,34.65,15467.11, gfx950,256,64,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,66.2383,flydsl_moe1_afp4_wfp4_bf16_t32x128x256_w4,0.0%,40.3127,flydsl_moe2_afp4_wfp4_bf16_t32x128x256_reduce_persist,0.0%,106.551,0,0,0,51.02,11392.27, gfx950,256,128,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,67.3417,flydsl_moe1_afp4_wfp4_bf16_t32x64x256_w3_kw2_fp4,0.8%,44.5853,flydsl_moe2_afp4_wfp4_bf16_t16x128x256_atomic_sbm32,0.8%,111.927,0,0,0,97.13,10855.62, gfx950,256,256,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,73.0874,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_kw2_fp4,0.8%,47.7846,flydsl_moe2_afp4_wfp4_bf16_t16x128x256_atomic_bnt2_persist_sbm32,0.8%,120.872,0,0,0,179.89,10071.78, -gfx950,256,512,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,81.41470000000001,flydsl_moe1_afp4_wfp4_bf16_t32x32x256_w3_kw2,0.0%,58.3181,flydsl_moe2_afp4_wfp4_bf16_t32x128x128_atomic_persist,0.8%,139.7328,0,0,0,311.21,8746.09, +gfx950,256,512,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,76.7743,flydsl_moe1_afp4_wfp4_bf16_t32x64x256_w4_kw2_fp4,0.8%,58.686,flydsl_moe2_afp4_wfp4_bf16_t32x128x128_atomic,0.8%,135.4603,0,0,0,321.03,9021.94, gfx950,256,1024,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,91.8469,flydsl_moe1_afp4_wfp4_bf16_t64x64x256_w4,0.0%,84.9999,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_persist,0.0%,176.8468,0,0,0,491.8,6963.95, -gfx950,256,2048,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,114.6878,flydsl_moe1_afp4_wfp4_bf16_t128x128x256_w2,0.0%,140.4587,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2_sbm128,0.0%,255.1465,0,0,0,681.75,4900.82, -gfx950,256,4096,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,149.773,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w2_bnt0_xcd4,0.0%,243.432,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2,0.0%,393.205,0,0,0,884.76,3276.09, +gfx950,256,2048,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,108.2764,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w4_bnt0_xcd4_fp4,0.8%,138.5647,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2_persist,0.0%,246.8411,0,0,0,704.69,5065.72, +gfx950,256,4096,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,149.0838,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w2_bnt0_xcd4,0.0%,239.422,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2,0.0%,388.5058,0,0,0,895.46,3315.72, gfx950,256,8192,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,222.9684,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w2_bnt0_xcd4,0.0%,452.3736,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_persist,0.0%,675.342,0,0,0,1030.27,2019.23, -gfx950,256,16384,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,64,0,369.8823,flydsl_moe1_afp4_wfp4_bf16_t64x128x256_w3_bnt0_xcd4,0.0%,928.2411,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce,0.0%,1298.1234,0,0,0,1071.99,1166.81, +gfx950,256,16384,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,367.4062,flydsl_moe1_afp4_wfp4_bf16_t128x128x256_w2_bnt0_xcd4,0.0%,894.9074,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_bnt2_sbm128,0.0%,1262.3136,0,0,0,1102.4,1199.91, gfx950,256,32768,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,128,0,638.4538,flydsl_moe1_afp4_wfp4_bf16_t128x128x256_w4_bnt0_xcd4,0.0%,1758.0362,flydsl_moe2_afp4_wfp4_bf16_t64x256x256_reduce_persist_sbm128,0.0%,2396.49,0,0,0,1161.34,758.05, gfx950,256,1,6144,2048,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,28.3763,moe_ck2stages_gemm1_64x32x32x128_1x1_MulABScaleShuffled_v3_Nswizzle0_Quant3_MulRoutedWeight0_silu_FP4X2_FP4X2_B16,0.0,17.5043,moe_ck2stages_gemm2_256x32x128x128_1x4_MulABScaleExpertWeightShuffled_v3_Nswizzle0_Quant3_MulRoutedWeight1_FP4X2_FP4X2_B16,0.0575,45.8806,0,0,0,0.0,0.0,flydsl_fallback gfx950,256,2,6144,2048,257,9,ActivationType.Silu,torch.bfloat16,torch.float4_e2m1fn_x2,torch.float4_e2m1fn_x2,QuantType.per_1x32,1,0,32,0,48.718900000000005,moe_ck2stages_gemm1_64x32x32x128_1x1_MulABScaleShuffled_v3_Nswizzle0_Quant3_MulRoutedWeight0_silu_FP4X2_FP4X2_B16,0.0,28.366,moe_ck2stages_gemm2_256x32x128x128_1x4_MulABScaleExpertWeightShuffled_v3_Nswizzle0_Quant3_MulRoutedWeight1_FP4X2_FP4X2_B16,0.0591,77.0849,0,0,0,0.0,0.0,flydsl_fallback diff --git a/aiter/configs/model_configs/mxfp8_tuned_fmoe_glm5_2.csv b/aiter/configs/model_configs/mxfp8_tuned_fmoe_glm5_2.csv new file mode 100644 index 00000000000..978a7e73865 --- /dev/null +++ b/aiter/configs/model_configs/mxfp8_tuned_fmoe_glm5_2.csv @@ -0,0 +1,33 @@ +gfx,cu_num,token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,q_type,use_g1u1,doweight_stage1,block_m,ksplit,us1,kernelName1,err1,us2,kernelName2,err2,us,run_1stage,xbf16,flat,tflops,bw,_tag +gfx950,256,1,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,15.4688,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_gui_xcd4_kw2_fp8,0.0%,11.4841,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_reduce,0.0%,26.9529,0,0,0,6.3,89985.67, +gfx950,256,2,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,22.7963,flydsl_moe1_afp8_wfp8_bf16_t32x128x256_gui_fp8,0.0%,17.2287,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_reduce_bnt2,0.0%,40.025,0,0,0,8.49,60596.96, +gfx950,256,4,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,41.6763,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w4_gui_xcd4_fp8,0.0%,25.5508,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_reduce_bnt2,0.0%,67.2271,0,0,0,10.11,36078.16, +gfx950,256,8,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,73.5866,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w4_gui_kw2_fp8,0.0%,42.2629,flydsl_moe2_afp8_wfp8_bf16_t32x128x128_reduce_bnt2_persist,0.0%,115.8495,0,0,0,11.73,20936.68, +gfx950,256,16,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,113.68,flydsl_moe1_afp8_wfp8_bf16_t32x128x256_w4_gui_fp8,0.0%,67.8507,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_atomic_bnt2,0.0%,181.5307,0,0,0,14.97,13362.21, +gfx950,256,32,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,167.1062,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w3_gui_fp8,0.0%,96.0626,flydsl_moe2_afp8_wfp8_bf16_t32x128x128_atomic_bnt2_persist,0.0%,263.1688,0,0,0,20.66,9218.21, +gfx950,256,64,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,221.2494,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w3_gui_fp8,0.0%,129.9436,flydsl_moe2_afp8_wfp8_bf16_t32x128x128_atomic_bnt2,0.0%,351.193,0,0,0,30.96,6909.41, +gfx950,256,128,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,245.3878,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w2_gui_fp8,0.0%,146.7136,flydsl_moe2_afp8_wfp8_bf16_t32x128x128_atomic,0.0%,392.1014,0,0,0,55.45,6191.55, +gfx950,256,256,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,255.8283,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w2_gui_kw2_fp8,0.0%,153.3288,flydsl_moe2_afp8_wfp8_bf16_t32x128x128_atomic_bnt2_persist,0.0%,409.1571,0,0,0,106.28,5939.22, +gfx950,256,512,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,265.9479,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w2_gui_kw2_fp8,0.0%,161.7576,flydsl_moe2_afp8_wfp8_bf16_t32x128x128_atomic_persist,0.0%,427.7055,0,0,0,203.35,5692.69, +gfx950,256,1024,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,64,0,297.0182,flydsl_moe1_afp8_wfp8_bf16_t64x128x256_w4_gui_fp8,0.0%,177.7574,flydsl_moe2_afp8_wfp8_bf16_t64x128x128_atomic_bnt2,0.0%,474.7756,0,0,0,366.38,5148.18, +gfx950,256,2048,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,128,0,348.9894,flydsl_moe1_afp8_wfp8_bf16_t128x128x256_w4_gui_fp8,0.0%,246.1611,flydsl_moe2_afp8_wfp8_bf16_t64x128x128_atomic_sbm128,0.0%,595.1505,0,0,0,584.55,4138.63, +gfx950,256,4096,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,64,0,529.8860000000001,flydsl_moe1_afp8_wfp8_bf16_t64x128x256_w4_bnt0_gui,0.0%,388.0813,flydsl_moe2_afp8_wfp8_bf16_t64x128x256_atomic_bnt2_persist,0.0%,917.9673,0,0,0,757.96,2724.34, +gfx950,256,8192,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,128,0,775.4362,flydsl_moe1_afp8_wfp8_bf16_t128x128x256_w2_bnt0_gui_fp8,0.0%,740.6954,flydsl_moe2_afp8_wfp8_bf16_t64x128x256_atomic_sbm128,0.0%,1516.1316,0,0,0,917.84,1699.29, +gfx950,256,16384,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,128,0,1335.0552,flydsl_moe1_afp8_wfp8_bf16_t128x256x256_bnt0_gui,0.0%,1454.1213,flydsl_moe2_afp8_wfp8_bf16_t64x128x128_atomic_bnt2_xcd4_sbm128,0.0%,2789.1765,0,0,0,997.84,977.83, +gfx950,256,32768,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,128,0,2372.8591,flydsl_moe1_afp8_wfp8_bf16_t128x256x256_w4_bnt0_gui,0.0%,2740.1934,flydsl_moe2_afp8_wfp8_bf16_t64x128x128_reduce_xcd4_sbm128,0.0%,5113.0525,0,0,0,1088.64,592.47, +gfx950,256,1,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,14.3389,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w4_gui_kw2_fp8,0.0%,9.2202,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_reduce_bnt2_persist,0.0%,23.5591,0,0,0,3.61,51474.66, +gfx950,256,2,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,16.0076,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_bnt0_gui_kw2_fp8,0.0%,12.4048,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_reduce,0.0%,28.4124,0,0,0,5.98,42682.6, +gfx950,256,4,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,23.7735,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w4_bnt0_gui_fp8,0.0%,18.1099,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_reduce_persist,0.0%,41.8834,0,0,0,8.11,28955.43, +gfx950,256,8,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,42.3387,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w3_gui_xcd4_fp8,0.0%,25.891,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_reduce_persist,0.0%,68.2297,0,0,0,9.96,17775.63, +gfx950,256,16,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,56.6658,flydsl_moe1_afp8_wfp8_bf16_t32x128x256_w3_gui_fp8,0.0%,36.8467,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_atomic_persist,0.0%,93.5125,0,0,0,14.53,12971.24, +gfx950,256,32,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,83.1757,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w3_gui_fp8,0.0%,51.3632,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_atomic_persist,0.0%,134.5389,0,0,0,20.2,9017.97, +gfx950,256,64,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,115.5613,flydsl_moe1_afp8_wfp8_bf16_t32x128x256_w3_gui_fp8,0.0%,69.5389,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_atomic_bnt2,0.0%,185.1002,0,0,0,29.37,6557.84, +gfx950,256,128,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,124.4154,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w2_gui_fp8,0.0%,76.9755,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_atomic_persist,0.0%,201.3909,0,0,0,53.98,6033.23, +gfx950,256,256,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,131.4393,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w3_gui_fp8,0.0%,81.255,flydsl_moe2_afp8_wfp8_bf16_t32x128x256_atomic,0.0%,212.6943,0,0,0,102.23,5723.69, +gfx950,256,512,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,32,0,138.504,flydsl_moe1_afp8_wfp8_bf16_t32x64x256_w3_gui_fp8,0.0%,89.0795,flydsl_moe2_afp8_wfp8_bf16_t32x128x128_atomic_bnt2,0.0%,227.5835,0,0,0,191.08,5369.96, +gfx950,256,1024,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,64,0,159.5362,flydsl_moe1_afp8_wfp8_bf16_t64x128x256_gui_fp8,0.0%,113.7872,flydsl_moe2_afp8_wfp8_bf16_t64x128x256_atomic_bnt2_persist,0.0%,273.3234,0,0,0,318.21,4505.84, +gfx950,256,2048,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,128,0,190.241,flydsl_moe1_afp8_wfp8_bf16_t128x128x256_gui_fp8,0.0%,180.7174,flydsl_moe2_afp8_wfp8_bf16_t128x128x128_reduce_persist,0.0%,370.9584,0,0,0,468.91,3370.8, +gfx950,256,4096,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,64,0,291.9083,flydsl_moe1_afp8_wfp8_bf16_t64x128x256_w4_bnt0_gui_xcd4,0.0%,305.7458,flydsl_moe2_afp8_wfp8_bf16_t64x128x256_reduce_persist,0.0%,597.6541,0,0,0,582.1,2155.39, +gfx950,256,8192,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,128,0,400.736,flydsl_moe1_afp8_wfp8_bf16_t128x128x256_w2_bnt0_gui_xcd4_fp8,0.0%,590.5346,flydsl_moe2_afp8_wfp8_bf16_t128x128x128_reduce_persist,0.0%,991.2706,0,0,0,701.91,1375.68, +gfx950,256,16384,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,128,0,653.1725,flydsl_moe1_afp8_wfp8_bf16_t128x128x256_w4_bnt0_gui_xcd4_fp8,0.0%,1137.1143,flydsl_moe2_afp8_wfp8_bf16_t128x128x128_reduce_persist,0.0%,1790.2868,0,0,0,777.29,846.05, +gfx950,256,32768,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0,128,0,1184.7875,flydsl_moe1_afp8_wfp8_bf16_t128x128x256_w2_bnt0_gui_xcd4_fp8,0.0%,2055.1738,flydsl_moe2_afp8_wfp8_bf16_t64x128x256_reduce_bnt2_xcd4_persist_sbm128,0.0%,3239.9613,0,0,0,859.0,560.7, diff --git a/aiter/configs/model_configs/mxfp8_untuned_fmoe_glm5_2.csv b/aiter/configs/model_configs/mxfp8_untuned_fmoe_glm5_2.csv new file mode 100644 index 00000000000..fc3789e0f39 --- /dev/null +++ b/aiter/configs/model_configs/mxfp8_untuned_fmoe_glm5_2.csv @@ -0,0 +1,33 @@ +token,model_dim,inter_dim,expert,topk,act_type,dtype,q_dtype_a,q_dtype_w,q_type,use_g1u1,doweight_stage1 +1,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +2,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +4,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +8,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +16,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +32,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +64,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +128,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +256,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +512,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +1024,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +2048,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +4096,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +8192,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +16384,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +32768,6144,512,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +1,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +2,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +4,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +8,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +16,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +32,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +64,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +128,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +256,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +512,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +1024,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +2048,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +4096,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +8192,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +16384,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0 +32768,6144,256,257,9,ActivationType.Silu,torch.bfloat16,torch.float8_e4m3fn,torch.float8_e4m3fn,QuantType.per_1x32,1,0