Skip to content

Commit aaee513

Browse files
committed
add plumbing from pytorch side, change all internal functions to using the routing map type enum
Signed-off-by: tdophung <tdophung@nvidia.com>
1 parent 889c161 commit aaee513

9 files changed

Lines changed: 297 additions & 55 deletions

File tree

tests/pytorch/test_fused_router.py

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
import torch
55
from typing import Optional
66
from transformer_engine.pytorch.router import (
7+
RoutingMapFormat,
78
fused_topk_with_score_function,
89
fused_compute_score_for_moe_aux_loss,
910
fused_moe_aux_loss,
@@ -458,6 +459,147 @@ def test_fused_moe_aux_loss(dtype, num_tokens, num_experts, topk):
458459
torch.testing.assert_close(probs.grad, probs_clone.grad, atol=atol, rtol=rtol)
459460

460461

462+
# =============================================================================
463+
# Test: routing_map BITMAP_U8 vs BYTEMAP parity (fwd + bwd)
464+
# Mirrors tests/jax/test_fused_router.py::test_topk_bitmap_vs_bytemap.
465+
# =============================================================================
466+
467+
468+
def _bytemap_to_bitmap_u8(bytemap: torch.Tensor) -> torch.Tensor:
469+
"""Reference packer: bool[T, E] -> uint8[T, ceil(E/8)] LSB-first.
470+
471+
Matches numpy.packbits(..., bitorder='little'), which is what the JAX-side
472+
parity test uses.
473+
"""
474+
flat = bytemap.to(torch.uint8).cpu().numpy()
475+
import numpy as np
476+
477+
return torch.from_numpy(np.packbits(flat, axis=-1, bitorder="little")).to(bytemap.device)
478+
479+
480+
@pytest.mark.parametrize("dtype", [torch.float32])
481+
@pytest.mark.parametrize(
482+
"num_tokens,num_experts,topk",
483+
[(128, 32, 4), (256, 128, 8), (256, 130, 8), (128, 1024, 16)],
484+
)
485+
@pytest.mark.parametrize("score_function", ["softmax", "sigmoid", "sqrtsoftplus"])
486+
def test_topk_bitmap_vs_bytemap(dtype, num_tokens, num_experts, topk, score_function):
487+
"""fused_topk_with_score_function should produce identical probs and an
488+
LSB-packed bitmap routing_map when routing_map_format=BITMAP_U8, and
489+
backward gradients should match the bytemap path exactly."""
490+
if topk >= num_experts:
491+
pytest.skip(f"topk ({topk}) >= num_experts ({num_experts})")
492+
if score_function in ("sigmoid", "sqrtsoftplus"):
493+
offset = torch.arange(-num_tokens // 2, num_tokens // 2, dtype=dtype, device="cuda") * 1e-4
494+
logits = (
495+
torch.arange(-num_experts // 2, num_experts // 2, device="cuda", dtype=dtype) * 1e-2
496+
)
497+
logits = logits.unsqueeze(0).repeat(num_tokens, 1) + offset.unsqueeze(1)
498+
else:
499+
logits = (
500+
torch.arange(
501+
-num_tokens * num_experts // 2,
502+
num_tokens * num_experts // 2,
503+
device="cuda",
504+
dtype=dtype,
505+
)
506+
* 1e-4
507+
)
508+
logits = logits.view(num_tokens, num_experts)
509+
510+
logits_byte = logits.detach().clone().requires_grad_(True)
511+
logits_bit = logits.detach().clone().requires_grad_(True)
512+
513+
probs_byte, routing_map_byte = fused_topk_with_score_function(
514+
logits=logits_byte,
515+
topk=topk,
516+
use_pre_softmax=False,
517+
num_groups=None,
518+
group_topk=None,
519+
scaling_factor=None,
520+
score_function=score_function,
521+
expert_bias=None,
522+
routing_map_format=RoutingMapFormat.BYTEMAP,
523+
)
524+
probs_bit, routing_map_bit = fused_topk_with_score_function(
525+
logits=logits_bit,
526+
topk=topk,
527+
use_pre_softmax=False,
528+
num_groups=None,
529+
group_topk=None,
530+
scaling_factor=None,
531+
score_function=score_function,
532+
expert_bias=None,
533+
routing_map_format=RoutingMapFormat.BITMAP_U8,
534+
)
535+
536+
assert probs_byte.dtype == probs_bit.dtype
537+
torch.testing.assert_close(probs_byte, probs_bit, atol=0.0, rtol=0.0)
538+
539+
expected_shape = (num_tokens, (num_experts + 7) // 8)
540+
assert routing_map_bit.shape == expected_shape, (
541+
f"Bitmap shape {tuple(routing_map_bit.shape)} != {expected_shape}"
542+
)
543+
assert routing_map_bit.dtype == torch.uint8
544+
assert routing_map_byte.dtype == torch.bool
545+
546+
packed_expected = _bytemap_to_bitmap_u8(routing_map_byte)
547+
torch.testing.assert_close(routing_map_bit, packed_expected, atol=0, rtol=0)
548+
549+
# Backward parity: grad of probs.sum() must be bit-identical across formats.
550+
probs_byte.sum().backward()
551+
probs_bit.sum().backward()
552+
torch.testing.assert_close(logits_byte.grad, logits_bit.grad, atol=0.0, rtol=0.0)
553+
554+
555+
@pytest.mark.parametrize("dtype", [torch.float32])
556+
@pytest.mark.parametrize(
557+
"num_tokens,num_experts,topk",
558+
[(128, 32, 4), (256, 128, 8), (256, 130, 8)],
559+
)
560+
@pytest.mark.parametrize("score_function", ["softmax", "sigmoid", "sqrtsoftplus"])
561+
def test_score_for_aux_loss_bitmap_vs_bytemap(
562+
dtype, num_tokens, num_experts, topk, score_function
563+
):
564+
"""fused_compute_score_for_moe_aux_loss: bitmap routing_map must equal
565+
LSB-packed bytemap; scores must be bit-identical across formats."""
566+
if topk >= num_experts:
567+
pytest.skip(f"topk ({topk}) >= num_experts ({num_experts})")
568+
offset = torch.arange(-num_tokens // 2, num_tokens // 2, dtype=dtype, device="cuda") * 1e-4
569+
logits = torch.arange(-num_experts // 2, num_experts // 2, device="cuda", dtype=dtype) * 1e-2
570+
logits = logits.unsqueeze(0).repeat(num_tokens, 1) + offset.unsqueeze(1)
571+
572+
logits_byte = logits.detach().clone().requires_grad_(True)
573+
logits_bit = logits.detach().clone().requires_grad_(True)
574+
575+
routing_map_byte, scores_byte = fused_compute_score_for_moe_aux_loss(
576+
logits=logits_byte,
577+
topk=topk,
578+
score_function=score_function,
579+
routing_map_format="bytemap",
580+
)
581+
routing_map_bit, scores_bit = fused_compute_score_for_moe_aux_loss(
582+
logits=logits_bit,
583+
topk=topk,
584+
score_function=score_function,
585+
routing_map_format="bitmap_u8",
586+
)
587+
588+
torch.testing.assert_close(scores_byte, scores_bit, atol=0.0, rtol=0.0)
589+
590+
expected_shape = (num_tokens, (num_experts + 7) // 8)
591+
assert routing_map_bit.shape == expected_shape
592+
assert routing_map_bit.dtype == torch.uint8
593+
assert routing_map_byte.dtype == torch.bool
594+
packed_expected = _bytemap_to_bitmap_u8(routing_map_byte)
595+
torch.testing.assert_close(routing_map_bit, packed_expected, atol=0, rtol=0)
596+
597+
# Backward parity through scores.
598+
scores_byte.sum().backward()
599+
scores_bit.sum().backward()
600+
torch.testing.assert_close(logits_byte.grad, logits_bit.grad, atol=0.0, rtol=0.0)
601+
602+
461603
def profile_topk_softmax(
462604
dtype,
463605
num_tokens,

transformer_engine/common/fused_router/fused_score_for_moe_aux_loss.cu

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@ namespace fused_router {
1919
template <typename DataType, TopkFuncType TopkFunc = TopkFuncType::Naive>
2020
__global__ void fused_score_for_moe_aux_loss_forward_kernel(
2121
const DataType *logits, int num_tokens, int num_experts, int topk, int score_function,
22-
float *scores, uint8_t *routing_map, int routing_map_format, CompType *intermediate_output) {
22+
float *scores, uint8_t *routing_map, NVTERoutingMapFormat routing_map_format,
23+
CompType *intermediate_output) {
2324
/***
2425
* Section: Global Variables/Addresses init
2526
* - Each warp is responsible for one token, and has own shared memory buffer.
@@ -174,7 +175,7 @@ __global__ void fused_score_for_moe_aux_loss_forward_kernel(
174175
template <typename DataType>
175176
void fused_score_for_moe_aux_loss_forward_kernel_launcher(
176177
const DataType *logits, int num_tokens, int num_experts, int topk, int score_function,
177-
float *scores, uint8_t *routing_map, int routing_map_format, CompType *intermediate_output,
178+
float *scores, uint8_t *routing_map, NVTERoutingMapFormat routing_map_format, CompType *intermediate_output,
178179
cudaStream_t stream) {
179180
// Meta data for the kernel
180181
size_t num_token_per_block = kThreadsPerBlock / kThreadsPerWarp;
@@ -212,7 +213,7 @@ void fused_score_for_moe_aux_loss_forward_kernel_launcher(
212213

213214
void fused_score_for_moe_aux_loss_forward(const Tensor &logits, int num_tokens, int num_experts,
214215
int topk, int score_function, Tensor &scores,
215-
Tensor &routing_map, int routing_map_format,
216+
Tensor &routing_map, NVTERoutingMapFormat routing_map_format,
216217
Tensor &intermediate_output, cudaStream_t stream) {
217218
TE_ROUTER_PROBS_TYPE_SWITCH_ALL(
218219
logits.data.dtype, DataType,
@@ -388,7 +389,7 @@ void fused_score_for_moe_aux_loss_backward(const Tensor &intermediate_output,
388389
void nvte_fused_score_for_moe_aux_loss_forward(const NVTETensor logits, int num_tokens,
389390
int num_experts, int topk, int score_function,
390391
NVTETensor scores, NVTETensor routing_map,
391-
int routing_map_format,
392+
NVTERoutingMapFormat routing_map_format,
392393
const NVTETensor intermediate_output,
393394
cudaStream_t stream) {
394395
NVTE_API_CALL(nvte_fused_score_for_moe_aux_loss_forward);

transformer_engine/common/fused_router/fused_topk_with_score_function.cu

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ template <typename DataType, typename BiasType, TopkFuncType TopkFunc = TopkFunc
1919
__global__ void fused_topk_with_score_function_forward_kernel(
2020
const DataType *logits, int num_tokens, int num_experts, int topk, bool use_pre_softmax,
2121
int num_groups, int group_topk, float scaling_factor, int score_function,
22-
const BiasType *expert_bias, DataType *probs, uint8_t *routing_map, int routing_map_format,
22+
const BiasType *expert_bias, DataType *probs, uint8_t *routing_map, NVTERoutingMapFormat routing_map_format,
2323
CompType *intermediate_output) {
2424
/***
2525
* Section: Global Variables/Addresses init
@@ -282,7 +282,7 @@ template <typename DataType, typename BiasType>
282282
void fused_topk_with_score_function_forward_kernel_launcher(
283283
const DataType *logits, int num_tokens, int num_experts, int topk, bool use_pre_softmax,
284284
int num_groups, int group_topk, float scaling_factor, int score_function,
285-
const BiasType *expert_bias, DataType *probs, uint8_t *routing_map, int routing_map_format,
285+
const BiasType *expert_bias, DataType *probs, uint8_t *routing_map, NVTERoutingMapFormat routing_map_format,
286286
CompType *intermediate_output, cudaStream_t stream) {
287287
size_t num_token_per_block = kThreadsPerBlock / kThreadsPerWarp;
288288
size_t grid_size = (num_tokens + num_token_per_block - 1) / num_token_per_block;
@@ -328,7 +328,7 @@ void fused_topk_with_score_function_forward(const Tensor logits, int num_tokens,
328328
int group_topk, float scaling_factor,
329329
int score_function, const Tensor expert_bias,
330330
Tensor probs, Tensor routing_map,
331-
int routing_map_format, Tensor intermediate_output,
331+
NVTERoutingMapFormat routing_map_format, Tensor intermediate_output,
332332
cudaStream_t stream) {
333333
TE_ROUTER_PROBS_TYPE_SWITCH_ALL(
334334
logits.data.dtype, DataType,
@@ -346,7 +346,7 @@ void fused_topk_with_score_function_forward(const Tensor logits, int num_tokens,
346346
template <typename DataType>
347347
__global__ void fused_topk_with_score_function_backward_kernel(
348348
// Inputs tensor
349-
const uint8_t *routing_map, int routing_map_format, const CompType *intermediate_output,
349+
const uint8_t *routing_map, NVTERoutingMapFormat routing_map_format, const CompType *intermediate_output,
350350
const DataType *grad_probs,
351351
// Other parameters
352352
int num_tokens, int num_experts, int topk, bool use_pre_softmax, float scaling_factor,
@@ -528,7 +528,7 @@ __global__ void fused_topk_with_score_function_backward_kernel(
528528

529529
template <typename DataType>
530530
void fused_topk_with_score_function_backward_kernel_launcher(
531-
const uint8_t *routing_map, int routing_map_format, const CompType *intermediate_output,
531+
const uint8_t *routing_map, NVTERoutingMapFormat routing_map_format, const CompType *intermediate_output,
532532
const DataType *grad_probs, int num_tokens, int num_experts, int topk, bool use_pre_softmax,
533533
float scaling_factor, int score_function, DataType *grad_logits, cudaStream_t stream) {
534534
// Meta data for the kernel
@@ -550,7 +550,7 @@ void fused_topk_with_score_function_backward_kernel_launcher(
550550
NVTE_CHECK_CUDA(cudaGetLastError());
551551
}
552552

553-
void fused_topk_with_score_function_backward(const Tensor &routing_map, int routing_map_format,
553+
void fused_topk_with_score_function_backward(const Tensor &routing_map, NVTERoutingMapFormat routing_map_format,
554554
const Tensor &intermediate_output,
555555
const Tensor &grad_probs, int num_tokens,
556556
int num_experts, int topk, bool use_pre_softmax,
@@ -572,8 +572,8 @@ void fused_topk_with_score_function_backward(const Tensor &routing_map, int rout
572572
void nvte_fused_topk_with_score_function_forward(
573573
const NVTETensor logits, int num_tokens, int num_experts, int topk, int use_pre_softmax,
574574
int num_groups, int group_topk, float scaling_factor, int score_function,
575-
const NVTETensor expert_bias, NVTETensor probs, NVTETensor routing_map, int routing_map_format,
576-
NVTETensor intermediate_output, cudaStream_t stream) {
575+
const NVTETensor expert_bias, NVTETensor probs, NVTETensor routing_map,
576+
NVTERoutingMapFormat routing_map_format, NVTETensor intermediate_output, cudaStream_t stream) {
577577
NVTE_API_CALL(nvte_fused_topk_with_score_function_forward);
578578
using namespace transformer_engine;
579579
fused_router::fused_topk_with_score_function_forward(
@@ -585,9 +585,10 @@ void nvte_fused_topk_with_score_function_forward(
585585
}
586586

587587
void nvte_fused_topk_with_score_function_backward(
588-
const NVTETensor routing_map, int routing_map_format, const NVTETensor intermediate_output,
589-
const NVTETensor grad_probs, int num_tokens, int num_experts, int topk, int use_pre_softmax,
590-
float scaling_factor, int score_function, NVTETensor grad_logits, cudaStream_t stream) {
588+
const NVTETensor routing_map, NVTERoutingMapFormat routing_map_format,
589+
const NVTETensor intermediate_output, const NVTETensor grad_probs, int num_tokens,
590+
int num_experts, int topk, int use_pre_softmax, float scaling_factor, int score_function,
591+
NVTETensor grad_logits, cudaStream_t stream) {
591592
NVTE_API_CALL(nvte_fused_topk_with_score_function_backward);
592593
using namespace transformer_engine;
593594
fused_router::fused_topk_with_score_function_backward(

transformer_engine/common/include/transformer_engine/fused_router.h

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,8 @@ typedef enum {
5252
void nvte_fused_topk_with_score_function_forward(
5353
const NVTETensor logits, int num_tokens, int num_experts, int topk, int use_pre_softmax,
5454
int num_groups, int group_topk, float scaling_factor, int score_function,
55-
const NVTETensor expert_bias, NVTETensor probs, NVTETensor routing_map, int routing_map_format,
56-
NVTETensor intermediate_output, cudaStream_t stream);
55+
const NVTETensor expert_bias, NVTETensor probs, NVTETensor routing_map,
56+
NVTERoutingMapFormat routing_map_format, NVTETensor intermediate_output, cudaStream_t stream);
5757

5858
/*! \brief Backward pass for fused topk + softmax/sigmoid.
5959
*
@@ -71,9 +71,10 @@ void nvte_fused_topk_with_score_function_forward(
7171
* \param[in] stream CUDA stream used for the operation.
7272
*/
7373
void nvte_fused_topk_with_score_function_backward(
74-
const NVTETensor routing_map, int routing_map_format, const NVTETensor intermediate_output,
75-
const NVTETensor grad_probs, int num_tokens, int num_experts, int topk, int use_pre_softmax,
76-
float scaling_factor, int score_function, NVTETensor grad_logits, cudaStream_t stream);
74+
const NVTETensor routing_map, NVTERoutingMapFormat routing_map_format,
75+
const NVTETensor intermediate_output, const NVTETensor grad_probs, int num_tokens,
76+
int num_experts, int topk, int use_pre_softmax, float scaling_factor, int score_function,
77+
NVTETensor grad_logits, cudaStream_t stream);
7778

7879
/*! \brief Forward pass for computing scores/routing map for auxiliary loss.
7980
*
@@ -93,7 +94,7 @@ void nvte_fused_topk_with_score_function_backward(
9394
void nvte_fused_score_for_moe_aux_loss_forward(const NVTETensor logits, int num_tokens,
9495
int num_experts, int topk, int score_function,
9596
NVTETensor scores, NVTETensor routing_map,
96-
int routing_map_format,
97+
NVTERoutingMapFormat routing_map_format,
9798
const NVTETensor intermediate_output,
9899
cudaStream_t stream);
99100

transformer_engine/jax/csrc/extensions/router.cpp

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -55,12 +55,12 @@ Error_Type FusedTopkWithScoreFunctionForwardFFI(
5555
". Check FusedTopkWithScoreFunctionFwdPrimitive.abstract in cpp_extensions/router.py.");
5656
auto intermediate_tensor = TensorWrapper(intermediate, flat_shape, DType::kFloat32);
5757

58-
int routing_map_format_int = static_cast<int>(routing_map_format);
58+
auto routing_map_format_nvte = static_cast<NVTERoutingMapFormat>(routing_map_format);
5959
if (compute_aux_scores) {
6060
nvte_fused_score_for_moe_aux_loss_forward(
6161
logits_tensor.data(), num_tokens, num_experts, static_cast<int>(topk),
6262
static_cast<int>(score_function), probs_tensor.data(), routing_map_tensor.data(),
63-
routing_map_format_int, intermediate_tensor.data(), stream);
63+
routing_map_format_nvte, intermediate_tensor.data(), stream);
6464
} else {
6565
auto bias_dims = expert_bias_buf.dimensions();
6666
auto expert_bias_tensor =
@@ -74,7 +74,7 @@ Error_Type FusedTopkWithScoreFunctionForwardFFI(
7474
static_cast<int>(use_pre_softmax), static_cast<int>(num_groups),
7575
static_cast<int>(group_topk), static_cast<float>(scaling_factor),
7676
static_cast<int>(score_function), expert_bias_tensor.data(), probs_tensor.data(),
77-
routing_map_tensor.data(), routing_map_format_int, intermediate_tensor.data(), stream);
77+
routing_map_tensor.data(), routing_map_format_nvte, intermediate_tensor.data(), stream);
7878
}
7979

8080
return ffi_with_cuda_error_check();
@@ -144,10 +144,11 @@ Error_Type FusedTopkWithScoreFunctionBackwardFFI(
144144
TensorWrapper(routing_map_buf.untyped_data(), routing_map_shape, DType::kByte);
145145

146146
nvte_fused_topk_with_score_function_backward(
147-
routing_map_tensor.data(), static_cast<int>(routing_map_format), intermediate_tensor.data(),
148-
grad_probs_tensor.data(), num_tokens, num_experts, static_cast<int>(topk),
149-
static_cast<int>(use_pre_softmax), static_cast<float>(scaling_factor),
150-
static_cast<int>(score_function), grad_logits_tensor.data(), stream);
147+
routing_map_tensor.data(), static_cast<NVTERoutingMapFormat>(routing_map_format),
148+
intermediate_tensor.data(), grad_probs_tensor.data(), num_tokens, num_experts,
149+
static_cast<int>(topk), static_cast<int>(use_pre_softmax),
150+
static_cast<float>(scaling_factor), static_cast<int>(score_function),
151+
grad_logits_tensor.data(), stream);
151152
}
152153

153154
return ffi_with_cuda_error_check();

0 commit comments

Comments
 (0)