From 0ceee085f0b29875a538d9892cacaf84fe526a8b Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Thu, 29 Jan 2026 15:08:31 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E9=9B=B6=E8=AE=A1=E7=AE=97?= =?UTF-8?q?=E4=B8=93=E5=AE=B6expert=E4=B8=AD=E5=B0=8F=E5=BC=80=E5=85=B3?= =?UTF-8?q?=EF=BC=8C=E5=B9=B6=E4=B8=94=E4=BF=AE=E6=94=B9=E4=BA=86forloop?= =?UTF-8?q?=E4=B8=AD=E5=AF=B9=E5=BA=94activation=5Ftype?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mindformers/pynative/transformers/moe/experts.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/mindformers/pynative/transformers/moe/experts.py b/mindformers/pynative/transformers/moe/experts.py index 5fdb4e710..618e8a7c7 100644 --- a/mindformers/pynative/transformers/moe/experts.py +++ b/mindformers/pynative/transformers/moe/experts.py @@ -72,7 +72,7 @@ class GroupedMLP(nn.Cell): self.moe_ffn_hidden_size *= 2 self.mul = mint.mul self.moe_token_dispatcher_type = config.moe_token_dispatcher_type - self.moe_use_experts_for_loop = True # config.moe_use_experts_for_loop + self.moe_use_experts_for_loop = not config.moe_grouped_gemm self.init_method = config.init_method # parameters @@ -199,9 +199,12 @@ class GroupedMLP(nn.Cell): out_experts_splits.append(h) continue h = self.matmul(x_expert, w1[expert_idx]) - h1, h2 = self.chunk(h, 2, -1) - h1 = self.activation_func(h1) - h = self.mul(h1, h2) + if self.activation_type == 'fusedswiglu': + h = self.activation_func(h, -1).reshape((-1, w2.shape[1])) + else: + x0, x1 = self.chunk(h, 2, -1) + act_out = self.activation_func(x0) + h = self.mul(act_out, x1) h = self.mul(h, permuted_probs_splits[expert_idx].reshape(-1, 1)) h = self.matmul(h, w2[expert_idx]) out_experts_splits.append(h) -- Gitee