diff --git a/mindformers/pynative/transformers/moe/experts.py b/mindformers/pynative/transformers/moe/experts.py
index 5fdb4e7104189d8f7bfd2a75d4976f3aab3d0e82..618e8a7c7438ebdb7fd0bde5cfd167e4b5f2924a 100644
--- a/mindformers/pynative/transformers/moe/experts.py
+++ b/mindformers/pynative/transformers/moe/experts.py
@@ -72,7 +72,7 @@ class GroupedMLP(nn.Cell):
             self.moe_ffn_hidden_size *= 2
             self.mul = mint.mul
         self.moe_token_dispatcher_type = config.moe_token_dispatcher_type
-        self.moe_use_experts_for_loop = True # config.moe_use_experts_for_loop
+        self.moe_use_experts_for_loop = not config.moe_grouped_gemm
         self.init_method = config.init_method
 
         # parameters
@@ -199,9 +199,12 @@ class GroupedMLP(nn.Cell):
                 out_experts_splits.append(h)
                 continue
             h = self.matmul(x_expert, w1[expert_idx])
-            h1, h2 = self.chunk(h, 2, -1)
-            h1 = self.activation_func(h1)
-            h = self.mul(h1, h2)
+            if self.activation_type == 'fusedswiglu':
+                h = self.activation_func(h, -1).reshape((-1, w2.shape[1]))
+            else:
+                x0, x1 = self.chunk(h, 2, -1)
+                act_out = self.activation_func(x0)
+                h = self.mul(act_out, x1)
             h = self.mul(h, permuted_probs_splits[expert_idx].reshape(-1, 1))
             h = self.matmul(h, w2[expert_idx])
             out_experts_splits.append(h)