We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 8ddcb7d commit e4d97a3Copy full SHA for e4d97a3
1 file changed
atom/model_ops/moe.py
@@ -756,7 +756,7 @@ def create_weights(
756
757
if layer.has_bias:
758
w13_bias = atom_parameter(
759
- torch.empty(
+ torch.zeros(
760
num_experts,
761
2 * intermediate_size_per_partition_after_pad,
762
dtype=torch.bfloat16,
@@ -793,7 +793,7 @@ def create_weights(
793
794
795
w2_bias = atom_parameter(
796
797
798
hidden_size,
799
0 commit comments