From e4324f2dc52c4a0d52fc72e9cc88f78f3b2d704f Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Sun, 25 Jan 2026 11:19:41 +0800 Subject: [PATCH 01/12] =?UTF-8?q?expert.py=E6=B7=BB=E5=8A=A0=E9=9B=B6?= =?UTF-8?q?=E8=AE=A1=E7=AE=97=E4=B8=93=E5=AE=B62?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pynative/transformers/moe/experts.py | 77 ++++++++++++++----- 1 file changed, 59 insertions(+), 18 deletions(-) diff --git a/mindformers/pynative/transformers/moe/experts.py b/mindformers/pynative/transformers/moe/experts.py index 54c452272..0a7fbe762 100644 --- a/mindformers/pynative/transformers/moe/experts.py +++ b/mindformers/pynative/transformers/moe/experts.py @@ -39,6 +39,14 @@ class GroupedMLP(nn.Cell): self.config = config self.num_local_experts = self.config.num_moe_experts self.top_k = self.config.moe_router_topk + # Fixed number of copy experts (identity experts). + # If config provides num_copy_experts, prefer it; otherwise default to 1. + self.num_copy_experts = getattr(self.config, "num_copy_experts", 1) + if self.num_copy_experts < 0 or self.num_copy_experts > self.num_local_experts: + raise ValueError( + f"num_copy_experts must be in [0, num_moe_experts], but got {self.num_copy_experts}." + ) + self.copy_expert_start = self.num_local_experts - self.num_copy_experts if self.config.moe_apply_probs_on_input: if self.config.moe_router_topk == 1: @@ -183,6 +191,13 @@ class GroupedMLP(nn.Cell): # Process each expert out_experts_splits = [] for expert_idx, x_expert in enumerate(x_splits): + # CopyExpert: identity mapping (no FFN), keep routing weights + if expert_idx >= self.copy_expert_start: + h = x_expert + if not self.config.moe_apply_probs_on_input: + h = self.mul(h, permuted_probs_splits[expert_idx].reshape(-1, 1)) + out_experts_splits.append(h) + continue h = self.matmul(x_expert, w1[expert_idx]) h1, h2 = self.chunk(h, 2, -1) h1 = self.activation_func(h1) @@ -206,23 +221,49 @@ class GroupedMLP(nn.Cell): if self.moe_use_experts_for_loop: return self._run_experts_for_loop(w1, w2, permuted_local_hidden_states, tokens_per_expert, permuted_probs) - # Original grouped_mm implementation - tokens_per_expert = self.cumsum(tokens_per_expert, dim=0, dtype=ms.int64) - fc1_output = GroupedMatmul(split_item=3, group_type=0)( - [permuted_local_hidden_states], [w1], None, None, None, None, None, tokens_per_expert)[0] + # Split tokens into non-copy experts and copy experts to avoid extra compute + counts_list = tokens_per_expert.asnumpy().tolist() + non_copy_experts = self.copy_expert_start + non_copy_tokens = sum(counts_list[:non_copy_experts]) + copy_tokens = sum(counts_list[non_copy_experts:]) if self.num_copy_experts > 0 else 0 - if self.gated_linear_unit: - if self.activation_type == 'fusedswiglu': - intermediate_parallel = self.activation_func(fc1_output, -1).reshape((-1, w2.shape[1])) + outputs = [] + if non_copy_experts > 0 and non_copy_tokens > 0: + # Run GroupedGEMM only for non-copy experts + non_copy_input = permuted_local_hidden_states[:non_copy_tokens] + non_copy_probs = permuted_probs[:non_copy_tokens] + tokens_per_expert_nc = tokens_per_expert[:non_copy_experts] + tokens_per_expert_nc = self.cumsum(tokens_per_expert_nc, dim=0, dtype=ms.int64) + + w1_nc = w1[:non_copy_experts] + w2_nc = w2[:non_copy_experts] + fc1_output = GroupedMatmul(split_item=3, group_type=0)( + [non_copy_input], [w1_nc], None, None, None, None, None, tokens_per_expert_nc)[0] + + if self.gated_linear_unit: + if self.activation_type == 'fusedswiglu': + intermediate_parallel = self.activation_func(fc1_output, -1).reshape((-1, w2_nc.shape[1])) + else: + x0, x1 = self.chunk(fc1_output, 2, -1) + act_out = self.activation_func(x0) + intermediate_parallel = self.mul(act_out, x1) else: - x0, x1 = self.chunk(fc1_output, 2, -1) - act_out = self.activation_func(x0) - intermediate_parallel = self.mul(act_out, x1) - else: - intermediate_parallel = self.activation_func(fc1_output) - - permuted_probs = self.cast(permuted_probs, intermediate_parallel.dtype) - intermediate_parallel = self.mul(intermediate_parallel, self.unsqueeze(permuted_probs, -1)) - fc2_output = GroupedMatmul(split_item=3, group_type=0)( - [intermediate_parallel], [w2], None, None, None, None, None, tokens_per_expert)[0] - return fc2_output + intermediate_parallel = self.activation_func(fc1_output) + + non_copy_probs = self.cast(non_copy_probs, intermediate_parallel.dtype) + intermediate_parallel = self.mul(intermediate_parallel, self.unsqueeze(non_copy_probs, -1)) + fc2_output = GroupedMatmul(split_item=3, group_type=0)( + [intermediate_parallel], [w2_nc], None, None, None, None, None, tokens_per_expert_nc)[0] + outputs.append(fc2_output) + + if self.num_copy_experts > 0 and copy_tokens > 0: + copy_segment = permuted_local_hidden_states[non_copy_tokens: non_copy_tokens + copy_tokens] + if not self.config.moe_apply_probs_on_input: + copy_probs = self.cast(permuted_probs[non_copy_tokens: non_copy_tokens + copy_tokens], + permuted_local_hidden_states.dtype) + copy_segment = self.mul(copy_segment, copy_probs.reshape(-1, 1)) + outputs.append(copy_segment) + + if outputs: + return self.cat(outputs, dim=0) + return permuted_local_hidden_states[:0] -- Gitee From 075d981ee6727934e3b4e0e4eac0d2c45747b25e Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Sun, 25 Jan 2026 11:37:17 +0800 Subject: [PATCH 02/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E6=95=B0=E6=8D=AE?= =?UTF-8?q?=E9=9B=86=E8=B7=AF=E5=BE=84?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .vscode/settings.json | 5 +++++ ds_pynative.yaml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) create mode 100644 .vscode/settings.json diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 000000000..a8c200329 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python-envs.pythonProjects": [] +} \ No newline at end of file diff --git a/ds_pynative.yaml b/ds_pynative.yaml index e76328464..5bc80fef4 100644 --- a/ds_pynative.yaml +++ b/ds_pynative.yaml @@ -108,7 +108,7 @@ train_dataset: &train_dataset pad: -1 # The token id of `pad` in the dataset. data_path: # Megatron dataset sampling ratio and path. - '1' - - "/home/l00913161/data/deepseek-datasets/mmap_deepseekv3_datasets_text_document" + - "/home/w00932055/dsv4/deepseek-datasets/mmap_deepseekv3_datasets_text_document" input_columns: ["input_ids", "labels", "loss_mask", "position_ids"] construct_args_key: ["input_ids", "labels", "loss_mask", "position_ids"] num_parallel_workers: 8 -- Gitee From 7c19f716664abe13ce95f26883427de581efaff2 Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Sun, 25 Jan 2026 17:12:22 +0800 Subject: [PATCH 03/12] =?UTF-8?q?=E5=9C=A8yaml=E5=92=8Ctransformer=5Fconfi?= =?UTF-8?q?g=5Futils=E5=8A=A0=E5=85=A5=E5=A4=8D=E5=88=B6=E4=B8=93=E5=AE=B6?= =?UTF-8?q?=E6=95=B0=E9=87=8F=E8=AE=BE=E7=BD=AE?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ds_pynative.yaml | 1 + mindformers/parallel_core/transformer_config.py | 2 ++ mindformers/parallel_core/transformer_config_utils.py | 3 ++- mindformers/pynative/transformers/moe/experts.py | 8 ++++++-- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ds_pynative.yaml b/ds_pynative.yaml index 5bc80fef4..1180ede9a 100644 --- a/ds_pynative.yaml +++ b/ds_pynative.yaml @@ -234,6 +234,7 @@ model: n_routed_experts: 16 num_experts_per_tok: 8 n_shared_experts: 1 + num_copy_experts: 5 moe_shared_expert_intermediate_size: 2048 moe_grouped_gemm: True moe_router_load_balancing_type: 'seq_aux_loss' diff --git a/mindformers/parallel_core/transformer_config.py b/mindformers/parallel_core/transformer_config.py index cd0b6df0e..3baae2ee1 100644 --- a/mindformers/parallel_core/transformer_config.py +++ b/mindformers/parallel_core/transformer_config.py @@ -354,6 +354,8 @@ class TransformerConfig(ModelParallelConfig, MFModelConfig): moe_apply_probs_on_input: bool = False """Apply probs on input of experts instead of applying after activation and glu.""" + num_copy_experts: int = 5 + # MindFormers New shared_expert_num: int = 0 """Number of shared experts.""" diff --git a/mindformers/parallel_core/transformer_config_utils.py b/mindformers/parallel_core/transformer_config_utils.py index 3f00c66b5..9af2574c2 100644 --- a/mindformers/parallel_core/transformer_config_utils.py +++ b/mindformers/parallel_core/transformer_config_utils.py @@ -382,7 +382,8 @@ COMMON_CONFIG_MAPPING = { "enable_expert_relocation": "enable_expert_relocation", "expert_relocation_initial_iteration": "expert_relocation_initial_iteration", "expert_relocation_freq": "expert_relocation_freq", - + "num_copy_experts": "num_copy_experts", + # Context Parallel # not changes "context_parallel_algo": ("cp_comm_type", get_cp_comm_type), diff --git a/mindformers/pynative/transformers/moe/experts.py b/mindformers/pynative/transformers/moe/experts.py index 0a7fbe762..b61f4a58e 100644 --- a/mindformers/pynative/transformers/moe/experts.py +++ b/mindformers/pynative/transformers/moe/experts.py @@ -40,14 +40,18 @@ class GroupedMLP(nn.Cell): self.num_local_experts = self.config.num_moe_experts self.top_k = self.config.moe_router_topk # Fixed number of copy experts (identity experts). - # If config provides num_copy_experts, prefer it; otherwise default to 1. - self.num_copy_experts = getattr(self.config, "num_copy_experts", 1) + self.num_copy_experts = self.config.num_copy_experts if self.num_copy_experts < 0 or self.num_copy_experts > self.num_local_experts: raise ValueError( f"num_copy_experts must be in [0, num_moe_experts], but got {self.num_copy_experts}." ) self.copy_expert_start = self.num_local_experts - self.num_copy_experts + # 验证yaml中是否起作用 + # 添加打印 + print(f"[GroupedMLP] num_moe_experts: {self.num_local_experts}") + print(f"[GroupedMLP] num_copy_experts: {self.num_copy_experts}") + if self.config.moe_apply_probs_on_input: if self.config.moe_router_topk == 1: raise ValueError("`moe_apply_probs_on_input` only works with `moe_router_topk`=1.") -- Gitee From a92ee66cdc7bcb5a148a140257d872cca0f32920 Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Mon, 26 Jan 2026 09:28:59 +0800 Subject: [PATCH 04/12] =?UTF-8?q?=E4=B8=A4=E7=A7=8D=E6=8E=A7=E5=88=B6?= =?UTF-8?q?=E6=89=8B=E6=AE=B5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- markdown.md | 176 +++++++++++ .../parallel_core/transformer_config.py | 2 +- .../pynative/transformers/moe/experts.py | 4 - .../transformers/moe/test_moe_layer.py | 99 ++++++ .../pynative/transformers/moe/test_router.py | 289 ++++++++++++++++++ 5 files changed, 565 insertions(+), 5 deletions(-) create mode 100644 markdown.md create mode 100644 mindformers/pynative/transformers/moe/test_moe_layer.py create mode 100644 mindformers/pynative/transformers/moe/test_router.py diff --git a/markdown.md b/markdown.md new file mode 100644 index 000000000..f8119370e --- /dev/null +++ b/markdown.md @@ -0,0 +1,176 @@ +基于 LongCat-Flash 技术报告，**计算预算控制 (Computational Budget Control)** 和 **负载均衡控制 (Load Balance Control)** 是为了确保“零计算专家”机制能够稳定、高效运行而设计的两项关键技术。 + +引入零计算专家后，模型需要解决两个核心问题： + +1. **宏观上**：如何确保模型总体的计算量（平均激活参数量）符合预期，既不超支也不偷懒？（计算预算控制） +2. **微观上**：如何防止某些计算型专家过载，同时适配零计算专家的特殊性？（负载均衡控制） + +以下是这两项控制机制的详细说明： + +### 1. 计算预算控制 (Computational Budget Control) + +这套机制的目标是**精细控制零计算专家的平均选择比例**，防止模型在训练过程中倾向于“偷懒”（过度使用零计算专家导致欠拟合）或“超支”（过度使用计算专家导致效率下降。 + + + +- 机制核心：与训练目标（Loss）解耦的**专家偏好（Bias）**项 $b_i$ 来动态调整路由分数。这就好比给每个专家一个“基础分”，根据它最近的忙碌程度动态调整，让它更容易或更难被选中。 + +- 更新公式 + + 专家偏好 $b_i$ 的更新遵循以下规则（类似于 PID 控制器）： + + $$\Delta b_{i} = \mu \left( \frac{K_{e}}{K} \cdot \frac{1}{N} - \frac{T_{i}}{K T_{all}} \right)$$ + + - **适用对象**：仅针对 $N$ 个**标准 FFN 专家**进行更新。 + - **变量含义**： + - $\mu$：调节速率。 + - $K_e$：**期望**激活的 FFN 专家数量（预算目标）。 + - $K$：总共选择的专家数量（Top-K）。 + - $T_i$：分配给专家 $i$ 的 Token 数量。 + - $T_{all}$：总 Token 数量。 + +- 效果 + + 该机制确保了模型在训练过程中，平均激活的 FFN 专家数量能够收敛到预设值（例如 $K_e$ 对应的 27B 参数量），波动幅度小于 1% 。同时，它允许标准差保持在较高水平，这意味着虽然整体预算受控，但模型仍保留了为不同难度的 Token 动态分配不同算力的能力 + +------ + +### 2. 负载均衡控制 (Load Balance Control) + +这套机制的目标是**防止设备间的计算负载不均**。虽然预算控制解决了全局的计算量问题，但如果某个 GPU 上的专家特别热门，会导致该设备成为瓶颈（木桶效应）。 + +- 机制核心：设备级负载均衡 Loss ($\mathcal{L}_{LB}$) + + 为了防止极端的不平衡，模型引入了一个辅助 Loss。LongCat-Flash 对传统的负载均衡 Loss 进行了修改，以适应零计算专家的存在 。 + +- 分组策略 + + 假设 $N$ 个标准 FFN 专家被分布在 $D$ 个设备（或组）上，每个组有 $G = N/D$ 个专家： + + - **标准专家**：归入前 $1$ 到 $D$ 组。 + + **零计算专家**：被强制归入第 **$D+1$ 组**（额外的一个虚拟组）。 + +- 计算公式 + + $$\mathcal{L}_{LB} = \alpha \sum_{j=1}^{D+1} f_{j} P_{j}$$ + + 其中： + + - $f_j$ 是该组专家被选中的频率（对于 $D+1$ 组，即零计算专家的选择频率）。 + - $P_j$ 是该组专家的平均路由概率。 + +- 效果 + + 通过这种设计，Loss 函数不仅平衡了物理设备（前 $D$ 组）之间的负载，还隐式地维护了计算型专家与零计算专家之间的比例平衡（趋近于 $\frac{K_{e}}{K-K_{e}}$）。这确保了在训练大规模模型时，不会因为专家的冷热不均导致计算效率下降。 + +### 总结 + +- **计算预算控制**是“调节阀”，利用 PID 算法动态调整基础分，确保模型整体“不偷懒也不超支”，维持平均 27B 的参数激活量。 +- **负载均衡控制**是“调度员”，通过辅助 Loss 确保任务在不同 GPU 间均匀分配，并将零计算专家作为独立的一类进行管理，防止局部拥堵。 + + + +### 1. 计算预算控制 (Computational Budget Control) + +这一机制的目标是确保模型在训练过程中，**平均**激活的 FFN 专家数量能够收敛到一个预设的期望值（$K_e$），从而控制整体的计算开销（即激活参数量）。 + +- **核心手段：基于 PID 控制器的自适应专家偏置** 龙猫引入了一个专家特定的偏置项 $b_i$，该偏置会根据专家最近的利用率动态更新。这个更新规则借鉴了控制理论中的 **PID 控制器**（Proportional-Integral-Derivative Controller）思想 。 + + + + + +- **偏置更新公式** 对于第 $i$ 个专家，其偏置 $b_i$ 在每一步的增量 $\Delta b_i$ 计算如下 ： + + + + + + $$\Delta b_i = \begin{cases} \mu \left( \frac{K_e}{K} \cdot \frac{1}{N} - \frac{T_i}{K T_{all}} \right), & \text{if } 1 \le i \le N \text{ (FFN Experts)} \\ 0, & \text{if } N < i \le N+Z \text{ (Zero-Computation Experts)} \end{cases}$$ + + - **$\mu$**：偏置适应率（Bias adaptation rate）。 + - **$K_e$**：期望激活的 FFN 专家数量（例如平均 8 个）。 + - **$K$**：每 Token 总共选择的专家数（例如 12 个）。 + - **$T_i$**：路由到第 $i$ 个专家的 Token 数量。 + - **$T_{all}$**：全局 Batch 中的 Token 总数。 + +- **机制亮点：零计算专家的“豁免权”** 该机制的一个关键设计是**不更新零计算专家的偏置** 。 + + + + + + - **逻辑：** 零计算专家本质上是恒等映射（Identity Mapping），不需要像 FFN 专家那样竞争特定的语义特征。 + - **效果：** 通过只调节 $N$ 个 FFN 专家的偏置，强制它们竞争有限的“计算名额”。当所有 FFN 专家都达到其目标利用率时，剩下的概率空间自然会被零计算专家填充，从而自动满足全局约束。 + +- **收敛效果** 实验表明，在大约 20B token 的训练后，各层的平均专家激活数收敛到了期望值，波动小于 1% 。 + + + + + +### 2. 负载均衡控制 (Load Balance Control) + +这一机制的目标是防止计算负载在不同设备（GPU）之间分配不均，同时妥善处理零计算专家带来的特殊分组需求。 + +- **核心手段：分组负载均衡 Loss ($\mathcal{L}_{LB}$)** 为了防止某些专家组（Expert Group）过载，论文引入了设备级的负载均衡损失，并专门为零计算专家分配了一个独立的组 。 + + + + + +- **分组策略** 假设共有 $N$ 个 FFN 专家，被均匀分配到 $D$ 个组中（每组 $G=N/D$ 个专家）。龙猫将所有 **$Z$ 个零计算专家单独归入第 $D+1$ 组** 。 + + + + + +- **Loss 计算公式** 负载均衡损失定义为 ： + + + + + + $$\mathcal{L}_{LB} = \alpha \sum_{j=1}^{D+1} f_j P_j$$ + + 其中： + + - **$f_j$**（第 $j$ 组的选择频率）：表示一个 Batch 中有多少比例的 Token 选择了该组。 + + - 对于 FFN 组 ($1 \le j \le D$)，$f_j$ 归一化时分母包含 $K_e$（期望 FFN 专家数） 。 + + + + + + - 对于零计算专家组 ($j=D+1$)，$f_j$ 归一化时分母包含 $K - K_e$（期望零计算专家数） 。 + + + + + + - + + **$P_j$**（第 $j$ 组的路由概率）：该组内所有专家路由分数的平均和 。 + + + + + + - **$\alpha$**：平衡系数。 + +- **调节目标** 通过这种设计，当损失函数收敛时，模型会倾向于将 FFN 专家与零计算专家的选择比例维持在 $\frac{K_e}{K - K_e}$ 附近 。这意味着模型既实现了设备间的负载均衡，又保证了零计算专家被“按需”选中，而不是被边缘化或过度使用。 + + + + + +\[ \begin{aligned} \text{MoE}(x_t) &= \sum_{i=1}^{N+Z} g_i \, E_i(x_t), \\ g_i &= \begin{cases} R(x_t)_i, & \text{if } R(x_t)_i \in \text{TopK}\bigl(R(x_t)_i + b_i \mid 1 \leq i \leq N+Z, K\bigr), \\ 0, & \text{otherwise}, \end{cases} \\ E_i(x_t) &= \begin{cases} \text{FFN}_i(x_t), & \text{if } 1 \leq i \leq N, \\ x_t, & \text{if } N < i \leq N+Z, \end{cases} \end{aligned} \tag{1} + + +$$ +\[ \begin{aligned} \text{MoE}(x_t) &= \sum_{i=1}^{N+Z} g_i \, E_i(x_t), \\ g_i &= \begin{cases} R(x_t)_i, & \text{if } R(x_t)_i \in \text{TopK}\bigl(R(x_t)_i + b_i \mid 1 \leq i \leq N+Z, K\bigr), \\ 0, & \text{otherwise}, \end{cases} +\\ E_i(x_t) &= \begin{cases} \text{FFN}_i(x_t), & \text{if } 1 \leq i \leq N, \\ x_t, & \text{if } N < i \leq N+Z, \end{cases} \end{aligned} \tag{1} \] +$$ +\noindent where \(R\) denotes the softmax router, \(b_i\) is the expert bias corresponding to the \(i\)-th expert, and \(K\) denotes the number of experts selected per token. \ No newline at end of file diff --git a/mindformers/parallel_core/transformer_config.py b/mindformers/parallel_core/transformer_config.py index 3baae2ee1..329f5a386 100644 --- a/mindformers/parallel_core/transformer_config.py +++ b/mindformers/parallel_core/transformer_config.py @@ -354,7 +354,7 @@ class TransformerConfig(ModelParallelConfig, MFModelConfig): moe_apply_probs_on_input: bool = False """Apply probs on input of experts instead of applying after activation and glu.""" - num_copy_experts: int = 5 + num_copy_experts: int = 0 # MindFormers New shared_expert_num: int = 0 diff --git a/mindformers/pynative/transformers/moe/experts.py b/mindformers/pynative/transformers/moe/experts.py index b61f4a58e..fdc7ff27d 100644 --- a/mindformers/pynative/transformers/moe/experts.py +++ b/mindformers/pynative/transformers/moe/experts.py @@ -47,10 +47,6 @@ class GroupedMLP(nn.Cell): ) self.copy_expert_start = self.num_local_experts - self.num_copy_experts - # 验证yaml中是否起作用 - # 添加打印 - print(f"[GroupedMLP] num_moe_experts: {self.num_local_experts}") - print(f"[GroupedMLP] num_copy_experts: {self.num_copy_experts}") if self.config.moe_apply_probs_on_input: if self.config.moe_router_topk == 1: diff --git a/mindformers/pynative/transformers/moe/test_moe_layer.py b/mindformers/pynative/transformers/moe/test_moe_layer.py new file mode 100644 index 000000000..24c7d6062 --- /dev/null +++ b/mindformers/pynative/transformers/moe/test_moe_layer.py @@ -0,0 +1,99 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# ... (License header) ... +"""MoE Layer implementation.""" +from mindspore import nn, Tensor, mint, ops +import mindspore as ms +from mindspore.common.parameter import Parameter + +from mindformers.parallel_core.transformer_config import TransformerConfig +from mindformers.pynative.layers.linear import Linear +from mindformers.pynative.transformers.mlp import MLPSubmodules + +# 【注意】这里必须导入新的 LongCatRouter，因为它返回 4 个值 +from .router import LongCatRouter +from .experts import GroupedMLP +from .shared_experts import SharedExpertMLP + + +class MoELayer(nn.Cell): + """ + MoE Layer that combines LongCatRouter, Grouped Experts, and Shared Experts. + """ + + def __init__(self, config: TransformerConfig): + super().__init__() + self.config = config + self.num_experts = config.num_moe_experts + self.top_k = config.moe_router_topk + + # 【改动】使用新的 LongCatRouter + # 它内部管理了 Computational Budget Control (PID) 和 Load Balance Loss + self.router = LongCatRouter(config) + + # Experts (保持不变，支持 copy experts) + self.experts = GroupedMLP(config) + + # Shared Experts (保持不变) + self.shared_experts = None + if config.shared_expert_num > 0: + submodules = MLPSubmodules( + linear_fc1=Linear, + linear_fc2=Linear + ) + self.shared_experts = SharedExpertMLP(config, submodules) + + # Buffers for logging only (Optional) + # 实际的 bias 和 tokens count 现在主要由 Router 内部维护和使用 + self.tokens_per_expert = Parameter( + mint.zeros(self.num_experts, dtype=ms.float32), + name="tokens_per_expert", + requires_grad=False + ) + + # Mint operators + self.reshape = mint.reshape + self.add = mint.add + + def construct(self, hidden_states: Tensor): + """ + Forward pass for MoELayer. + Args: + hidden_states (Tensor): Input tensor of shape (bs, slen, dim) + """ + bs, slen, dim = hidden_states.shape + x_flat = self.reshape(hidden_states, (-1, dim)) + + # 【定义 aux_loss】 + # 调用 self.router (LongCatRouter)，它返回 4 个值： + # 1. top_scores: 路由权重 + # 2. selected_experts_indices: 选中的专家索引 + # 3. num_tokens_per_expert: 每个专家的 token 数量统计（用于 logging 或其他用途） + # 4. aux_loss: 计算好的负载均衡 Loss (Scalar) + top_scores, selected_experts_indices, num_tokens_per_expert, aux_loss = self.router( + x_flat, + training=self.training + ) + + # 统计 Expert 使用情况 (可选，仅用于兼容旧逻辑或日志) + if self.tokens_per_expert is not None: + self.tokens_per_expert.add_(num_tokens_per_expert) + + # 执行专家计算 + routed_output = self.experts(hidden_states, top_scores, selected_experts_indices) + + # 执行共享专家计算 + shared_output = None + if self.shared_experts is not None: + shared_output, _ = self.shared_experts(hidden_states) + + out_experts = self.reshape(routed_output, (bs, slen, dim)) + + # 结果融合 + if shared_output is not None: + final_out = self.add(shared_output, out_experts) + else: + final_out = out_experts + + # 返回最终输出和 aux_loss + # 这里的 aux_loss 就是上面从 router 返回的那个变量 + return final_out, aux_loss \ No newline at end of file diff --git a/mindformers/pynative/transformers/moe/test_router.py b/mindformers/pynative/transformers/moe/test_router.py new file mode 100644 index 000000000..4bfc04a63 --- /dev/null +++ b/mindformers/pynative/transformers/moe/test_router.py @@ -0,0 +1,289 @@ +# Copyright 2026 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""LongCat Router Mechanism for MoE""" +from typing import Tuple, Optional + +from mindspore import nn, Tensor, mint, ops +from mindspore.common.parameter import Parameter +from mindspore.common import dtype as mstype + +from mindformers.parallel_core.transformer_config import TransformerConfig +from mindformers.parallel_core.utils.init_method import init_method_normal + + +class LongCatRouter(nn.Cell): + """ + Implements the LongCat-Flash routing mechanism with Computational Budget Control + and Load Balance Control. + + Mechanism 1: Computational Budget Control (PID-controlled Bias) + - Adjusts expert bias dynamically to maintain expected activation rates. + - Updates are EXCLUDED for Zero-Computation (Copy) experts. + + Mechanism 2: Load Balance Control + - Computes auxiliary loss based on groups. + - FFN experts are divided into D groups. + - Zero-Computation experts form the (D+1)-th group. + + Args: + config (TransformerConfig): Configuration object. + """ + + def __init__(self, config: TransformerConfig): + super().__init__() + self.config = config + + # Dimensions + self.hidden_size = config.hidden_size + self.num_total_experts = config.num_moe_experts + self.top_k = config.moe_router_topk + + # LongCat specific configurations + # N: FFN experts, Z: Zero-Computation experts + self.num_copy_experts = config.num_copy_experts + self.num_ffn_experts = self.num_total_experts - self.num_copy_experts + + if self.num_ffn_experts <= 0: + raise ValueError(f"num_ffn_experts must be positive, got {self.num_ffn_experts}") + + # Ke: Expected number of activated FFN experts + # Usually defined in config, fallback to top_k * ratio if not present + self.expected_ffn_k = getattr(config, 'moe_expected_ffn_experts', float(self.top_k * self.num_ffn_experts / self.num_total_experts)) + + # PID Controller parameters + self.bias_update_rate = getattr(config, 'moe_router_bias_update_rate', 0.001) + + # Load Balancing parameters + # D: Number of FFN groups + self.num_ffn_groups = config.moe_router_num_groups if config.moe_router_num_groups else 1 + self.aux_loss_coeff = config.moe_aux_loss_coeff + + # Routing parameters + self.score_func = config.moe_router_score_function + self.route_norm = config.norm_topk_prob + self.route_scale = ( + config.moe_router_topk_scaling_factor + if config.moe_router_topk_scaling_factor is not None + else 1.0 + ) + + # Learnable Router Weight + self.weight = Parameter( + init_method_normal(0.02)((self.num_total_experts, self.hidden_size)), + name='weight' + ) + + # Expert Bias (Controlled by PID, detached from Gradient Descent) + self.expert_bias = Parameter( + mint.zeros(self.num_total_experts, dtype=mstype.float32), + name="expert_bias", + requires_grad=False + ) + + # Ops + self.linear = mint.nn.functional.linear + self.sigmoid = mint.nn.functional.sigmoid + self.softmax = mint.nn.functional.softmax + self.cast = ops.cast + self.topk = mint.topk + self.gather = mint.gather + self.mul = mint.mul + self.div = mint.div + self.sum = mint.sum + self.histc = mint.histc + self.ones_like = mint.ones_like + self.reshape = mint.reshape + self.mean = mint.mean + self.cat = mint.cat + + def update_expert_bias(self, num_tokens_per_expert: Tensor, total_tokens: int): + """ + PID Controller for Computational Budget Control. + Formula: delta_b_i = mu * (Target_Rate - Current_Rate) + Only updates FFN experts (indices 0 to N-1). Zero-Computation experts are skipped. + """ + if self.training and self.bias_update_rate > 0: + # Current selection rate: Ti / (K * T_all) + # Note: We use K * T_all as denominator to normalize to [0, 1] per expert slot + current_rate = num_tokens_per_expert / (self.top_k * total_tokens) + + # Target selection rate for FFN experts: Ke / (K * N) + target_rate_ffn = self.expected_ffn_k / (self.top_k * self.num_ffn_experts) + + # Calculate error: Target - Current + # We construct a target tensor that only has values for FFN experts + # For Copy experts, we set error to 0 so bias doesn't change + target_rates = self.ones_like(current_rate) * target_rate_ffn + + # Mask out Copy Experts (Last Z experts) + # Assuming experts are ordered [FFN_0 ... FFN_N-1, Copy_0 ... Copy_Z-1] + if self.num_copy_experts > 0: + ffn_mask = mint.zeros(self.num_total_experts, dtype=mstype.bool_) + ffn_mask[:self.num_ffn_experts] = True + + # Zero out targets for copy experts (logic: no update) + # We achieve "no update" by making error term 0 effectively or masking update + pass # Implementation below uses mask on update directly + + error = target_rates - current_rate + update_step = self.bias_update_rate * error + + # Apply mask: Set update to 0 for Copy Experts + if self.num_copy_experts > 0: + mask = mint.zeros(self.num_total_experts, dtype=mstype.float32) + mask[:self.num_ffn_experts] = 1.0 + update_step = self.mul(update_step, mask) + + # Update bias in-place + ops.assign_add(self.expert_bias, update_step) + + def compute_load_balancing_loss(self, router_probs: Tensor, expert_indices: Tensor) -> Tensor: + """ + Grouped Load Balancing Loss. + Groups 1..D: FFN Experts. + Group D+1: Zero-Computation Experts. + + L_LB = alpha * sum_{j=1}^{D+1} (f_j * P_j) + """ + if self.aux_loss_coeff <= 0: + return mint.zeros((), dtype=router_probs.dtype) + + bs_slen = router_probs.shape[0] + + # 1. Calculate f_j (Frequency of selection per group) + # Flatten selected indices + selected_flat = self.reshape(expert_indices, (-1,)) + + # Count hits per expert + expert_counts = self.histc( + selected_flat, + bins=self.num_total_experts, + min=0, + max=self.num_total_experts + ) # Shape [Total_Experts] + + # Aggregation for FFN Groups + # Assuming FFN experts are 0..N-1, split into D groups + experts_per_ffn_group = self.num_ffn_experts // self.num_ffn_groups + + ffn_counts = expert_counts[:self.num_ffn_experts] + ffn_counts_reshaped = self.reshape(ffn_counts, (self.num_ffn_groups, experts_per_ffn_group)) + group_counts_ffn = self.sum(ffn_counts_reshaped, dim=1) # Shape [D] + + # Aggregation for Copy Group (Group D+1) + if self.num_copy_experts > 0: + copy_counts = expert_counts[self.num_ffn_experts:] + group_count_copy = self.sum(copy_counts).unsqueeze(0) # Shape [1] + all_group_counts = self.cat((group_counts_ffn, group_count_copy)) # Shape [D+1] + else: + all_group_counts = group_counts_ffn + + # Normalize counts to frequencies f_j + # FFN groups normalized by (Ke * T) ? Paper implies slightly different normalization, + # usually it is fraction of total selections. Let's use standard fraction within TopK. + # f_j = (tokens in group) / (Total Tokens * TopK) + # Note: LongCat paper Eq 3/4/5 suggests specific normalization denominators. + # Here we implement generic LB loss: sum(f_j * P_j) * N_groups + f_j = all_group_counts / (bs_slen * self.top_k) + + # 2. Calculate P_j (Sum of probabilities per group) + # Sum probs across batch for each expert + expert_prob_sum = self.sum(router_probs, dim=0) # Shape [Total_Experts] + + ffn_probs = expert_prob_sum[:self.num_ffn_experts] + ffn_probs_reshaped = self.reshape(ffn_probs, (self.num_ffn_groups, experts_per_ffn_group)) + group_probs_ffn = self.sum(ffn_probs_reshaped, dim=1) + + if self.num_copy_experts > 0: + copy_probs = expert_prob_sum[self.num_ffn_experts:] + group_prob_copy = self.sum(copy_probs).unsqueeze(0) + all_group_probs = self.cat((group_probs_ffn, group_prob_copy)) + else: + all_group_probs = group_probs_ffn + + # Normalize P_j (average probability per token) + P_j = all_group_probs / bs_slen + + # 3. Compute Loss + # Multiply and Sum + # We multiply by number of groups to keep magnitude similar to standard LB loss + num_groups = self.num_ffn_groups + (1 if self.num_copy_experts > 0 else 0) + loss = self.aux_loss_coeff * num_groups * self.sum(f_j * P_j) + + return loss + + def construct( + self, x: Tensor, training: bool = True + ) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """ + Returns: + top_scores: [BS*Seq, K] + selected_indices: [BS*Seq, K] + num_tokens_per_expert: [Num_Experts] + aux_loss: Scalar tensor + """ + bs_slen, _ = x.shape + router_dtype = self.config.moe_router_dtype + + # 1. Compute Router Scores + x_cast = self.cast(x, router_dtype) + weight = self.cast(self.weight, router_dtype) + logits = self.linear(x_cast, weight) # [BS*Seq, Total_Experts] + + # 2. Apply Softmax/Sigmoid + if self.score_func == "sigmoid": + probs = self.sigmoid(self.cast(logits, mstype.float32)) + elif self.score_func == "softmax": + probs = self.softmax(self.cast(logits, mstype.float32), dim=1) + else: + raise NotImplementedError(f"Unknown score function {self.score_func}") + + # 3. Add Bias (Computational Budget Control) + # bias is updated by PID but applied here for routing + probs_for_routing = probs + self.expert_bias + + # 4. TopK Selection + # Note: No node-limited routing here, standard TopK over all experts (FFN + Copy) + _, selected_indices = self.topk( + probs_for_routing, k=self.top_k, dim=-1, sorted=False + ) + selected_indices = self.cast(selected_indices, mstype.int64) + + # Gather real probabilities (without bias) for gating + top_scores = self.gather(probs, dim=1, index=selected_indices) + + # 5. Normalize and Scale + if self.route_norm: + denominator = self.sum(top_scores, dim=-1, keepdim=True) + 1e-20 + top_scores = self.div(top_scores, denominator) + + top_scores = self.mul(top_scores, self.route_scale) + + # 6. Statistics + num_tokens_per_expert = self.histc( + selected_indices, + bins=self.num_total_experts, + min=0, + max=self.num_total_experts, + ) + + # 7. Update Bias (PID Control) + if self.training and training: + self.update_expert_bias(num_tokens_per_expert, bs_slen) + + # 8. Compute Aux Loss (Load Balance Control) + aux_loss = self.compute_load_balancing_loss(probs, selected_indices) + + return top_scores, selected_indices, num_tokens_per_expert, aux_loss \ No newline at end of file -- Gitee From 30aeea51c55f4579ed1e3418788cd6899e7e1ee4 Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Mon, 26 Jan 2026 17:21:22 +0800 Subject: [PATCH 05/12] =?UTF-8?q?=E4=BC=98=E5=8C=96=E9=9B=B6=E8=AE=A1?= =?UTF-8?q?=E7=AE=97=E4=B8=93=E5=AE=B6w1w2=E5=86=85=E5=AD=98?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../base_models/gpt/moe_module_specs.py | 3 --- .../pynative/transformers/moe/experts.py | 19 +++++++++---------- 2 files changed, 9 insertions(+), 13 deletions(-) diff --git a/mindformers/pynative/base_models/gpt/moe_module_specs.py b/mindformers/pynative/base_models/gpt/moe_module_specs.py index 9d540addb..09e19ff8c 100644 --- a/mindformers/pynative/base_models/gpt/moe_module_specs.py +++ b/mindformers/pynative/base_models/gpt/moe_module_specs.py @@ -31,9 +31,6 @@ def get_moe_module_spec( if num_experts is None: raise ValueError("num_experts cannot be None.") - # experts spec - if not moe_grouped_gemm: - raise NotImplementedError("moe_grouped_gemm = 'False' is not supported now.") shared_experts = SharedExpertMLP diff --git a/mindformers/pynative/transformers/moe/experts.py b/mindformers/pynative/transformers/moe/experts.py index fdc7ff27d..1da72d734 100644 --- a/mindformers/pynative/transformers/moe/experts.py +++ b/mindformers/pynative/transformers/moe/experts.py @@ -40,13 +40,14 @@ class GroupedMLP(nn.Cell): self.num_local_experts = self.config.num_moe_experts self.top_k = self.config.moe_router_topk # Fixed number of copy experts (identity experts). - self.num_copy_experts = self.config.num_copy_experts + # If config provides num_copy_experts, prefer it; otherwise default to 1. + self.num_copy_experts = getattr(self.config, "num_copy_experts", 1) if self.num_copy_experts < 0 or self.num_copy_experts > self.num_local_experts: raise ValueError( f"num_copy_experts must be in [0, num_moe_experts], but got {self.num_copy_experts}." ) self.copy_expert_start = self.num_local_experts - self.num_copy_experts - + self.num_ffn_experts = self.copy_expert_start if self.config.moe_apply_probs_on_input: if self.config.moe_router_topk == 1: @@ -77,10 +78,10 @@ class GroupedMLP(nn.Cell): # parameters self.weight1 = Parameter( - self.init_method([self.num_local_experts * self.hidden_size, self.moe_ffn_hidden_size]), + self.init_method([self.num_ffn_experts * self.hidden_size, self.moe_ffn_hidden_size]), name='w1') self.weight2 = Parameter( - self.init_method([self.num_local_experts * self.config.moe_ffn_hidden_size, self.hidden_size]), + self.init_method([self.num_ffn_experts * self.config.moe_ffn_hidden_size, self.hidden_size]), name='w2') self.cast = ops.cast @@ -223,7 +224,7 @@ class GroupedMLP(nn.Cell): # Split tokens into non-copy experts and copy experts to avoid extra compute counts_list = tokens_per_expert.asnumpy().tolist() - non_copy_experts = self.copy_expert_start + non_copy_experts = self.num_ffn_experts non_copy_tokens = sum(counts_list[:non_copy_experts]) copy_tokens = sum(counts_list[non_copy_experts:]) if self.num_copy_experts > 0 else 0 @@ -235,14 +236,12 @@ class GroupedMLP(nn.Cell): tokens_per_expert_nc = tokens_per_expert[:non_copy_experts] tokens_per_expert_nc = self.cumsum(tokens_per_expert_nc, dim=0, dtype=ms.int64) - w1_nc = w1[:non_copy_experts] - w2_nc = w2[:non_copy_experts] fc1_output = GroupedMatmul(split_item=3, group_type=0)( - [non_copy_input], [w1_nc], None, None, None, None, None, tokens_per_expert_nc)[0] + [non_copy_input], [w1], None, None, None, None, None, tokens_per_expert_nc)[0] if self.gated_linear_unit: if self.activation_type == 'fusedswiglu': - intermediate_parallel = self.activation_func(fc1_output, -1).reshape((-1, w2_nc.shape[1])) + intermediate_parallel = self.activation_func(fc1_output, -1).reshape((-1, w2.shape[1])) else: x0, x1 = self.chunk(fc1_output, 2, -1) act_out = self.activation_func(x0) @@ -253,7 +252,7 @@ class GroupedMLP(nn.Cell): non_copy_probs = self.cast(non_copy_probs, intermediate_parallel.dtype) intermediate_parallel = self.mul(intermediate_parallel, self.unsqueeze(non_copy_probs, -1)) fc2_output = GroupedMatmul(split_item=3, group_type=0)( - [intermediate_parallel], [w2_nc], None, None, None, None, None, tokens_per_expert_nc)[0] + [intermediate_parallel], [w2], None, None, None, None, None, tokens_per_expert_nc)[0] outputs.append(fc2_output) if self.num_copy_experts > 0 and copy_tokens > 0: -- Gitee From ca3a6428b592c47329045e2c3778673606eeea22 Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Mon, 26 Jan 2026 21:57:31 +0800 Subject: [PATCH 06/12] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=AE=A1=E7=AE=97?= =?UTF-8?q?=E9=A2=84=E7=AE=97=E6=8E=A7=E5=88=B6=E5=92=8C=E8=B4=9F=E8=BD=BD?= =?UTF-8?q?=E5=9D=87=E8=A1=A1=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ds_pynative.yaml | 3 + .../parallel_core/transformer_config.py | 6 + .../pynative/transformers/moe/experts.py | 5 +- .../pynative/transformers/moe/moe_layer.py | 21 +- .../pynative/transformers/moe/router.py | 242 ++++++++++++++++++ 5 files changed, 270 insertions(+), 7 deletions(-) diff --git a/ds_pynative.yaml b/ds_pynative.yaml index 1180ede9a..2bd7ef681 100644 --- a/ds_pynative.yaml +++ b/ds_pynative.yaml @@ -235,6 +235,9 @@ model: num_experts_per_tok: 8 n_shared_experts: 1 num_copy_experts: 5 + use_longcat_router: bool = True + moe_expected_ffn_experts: float = 1.375 # Default best value: top-k * FFN/(FFN + COPY) + moe_router_bias_update_rate: float = 0.001 moe_shared_expert_intermediate_size: 2048 moe_grouped_gemm: True moe_router_load_balancing_type: 'seq_aux_loss' diff --git a/mindformers/parallel_core/transformer_config.py b/mindformers/parallel_core/transformer_config.py index 329f5a386..4257157e5 100644 --- a/mindformers/parallel_core/transformer_config.py +++ b/mindformers/parallel_core/transformer_config.py @@ -356,6 +356,12 @@ class TransformerConfig(ModelParallelConfig, MFModelConfig): num_copy_experts: int = 0 + use_longcat_router: bool = True + + moe_expected_ffn_experts: float = 1.375 # Default best value: top-k * FFN/(FFN + COPY) + + moe_router_bias_update_rate: float = 0.001 + # MindFormers New shared_expert_num: int = 0 """Number of shared experts.""" diff --git a/mindformers/pynative/transformers/moe/experts.py b/mindformers/pynative/transformers/moe/experts.py index 1da72d734..4c68208a3 100644 --- a/mindformers/pynative/transformers/moe/experts.py +++ b/mindformers/pynative/transformers/moe/experts.py @@ -39,9 +39,8 @@ class GroupedMLP(nn.Cell): self.config = config self.num_local_experts = self.config.num_moe_experts self.top_k = self.config.moe_router_topk - # Fixed number of copy experts (identity experts). - # If config provides num_copy_experts, prefer it; otherwise default to 1. - self.num_copy_experts = getattr(self.config, "num_copy_experts", 1) + self.num_copy_experts = self.config.num_copy_experts + if self.num_copy_experts < 0 or self.num_copy_experts > self.num_local_experts: raise ValueError( f"num_copy_experts must be in [0, num_moe_experts], but got {self.num_copy_experts}." diff --git a/mindformers/pynative/transformers/moe/moe_layer.py b/mindformers/pynative/transformers/moe/moe_layer.py index 2290d92f5..ed39080c5 100644 --- a/mindformers/pynative/transformers/moe/moe_layer.py +++ b/mindformers/pynative/transformers/moe/moe_layer.py @@ -20,7 +20,7 @@ from mindspore.common.parameter import Parameter from mindformers.parallel_core.transformer_config import TransformerConfig from mindformers.pynative.layers.linear import Linear from mindformers.pynative.transformers.mlp import MLPSubmodules -from .router import TopKRouter +from .router import LongCatRouter, TopKRouter from .experts import GroupedMLP from .shared_experts import SharedExpertMLP @@ -37,7 +37,12 @@ class MoELayer(nn.Cell): self.top_k = config.moe_router_topk # Router - self.router = TopKRouter(config) + # Prefer LongCatRouter when copy experts or aux loss is enabled + use_longcat_router = self.config.use_longcat_router + if use_longcat_router: + self.router = LongCatRouter(config) + else: + self.router = TopKRouter(config) # Experts self.experts = GroupedMLP(config) @@ -94,7 +99,15 @@ class MoELayer(nn.Cell): bs, slen, dim = hidden_states.shape x_flat = self.reshape(hidden_states, (-1, dim)) - top_scores, selected_experts_indices, num_tokens_per_expert = self.router(x_flat, self.expert_bias) + if isinstance(self.router, TopKRouter): + top_scores, selected_experts_indices, num_tokens_per_expert = self.router( + x_flat, self.expert_bias + ) + aux_loss = None + else: + top_scores, selected_experts_indices, num_tokens_per_expert, aux_loss = self.router( + x_flat, training=self.training + ) self.tokens_per_expert.add_(num_tokens_per_expert) @@ -111,4 +124,4 @@ class MoELayer(nn.Cell): else: final_out = out_experts - return final_out, None + return final_out, aux_loss diff --git a/mindformers/pynative/transformers/moe/router.py b/mindformers/pynative/transformers/moe/router.py index 8c8d469c3..bce13bb9b 100644 --- a/mindformers/pynative/transformers/moe/router.py +++ b/mindformers/pynative/transformers/moe/router.py @@ -220,3 +220,245 @@ class TopKRouter(nn.Cell): ) return top_scores, selected_experts_indices, num_tokens_per_expert + + +class LongCatRouter(nn.Cell): + """ + LongCat-Flash routing mechanism with Computational Budget Control and + Load Balance Control. + + Computational Budget Control: + - PID-like bias update to maintain expected FFN activation rate. + - Bias updates are excluded for zero-computation (copy) experts. + + Load Balance Control: + - Auxiliary loss computed over expert groups. + - FFN experts are divided into D groups; copy experts form group D+1. + """ + + def __init__(self, config: TransformerConfig): + super().__init__() + self.config = config + + # Dimensions + self.hidden_size = config.hidden_size + self.num_total_experts = config.num_moe_experts + self.top_k = config.moe_router_topk + + # LongCat configuration + self.num_copy_experts = self.config.num_copy_experts + if self.num_copy_experts < 0 or self.num_copy_experts > self.num_total_experts: + raise ValueError( + f"num_copy_experts must be in [0, num_moe_experts], but got {self.num_copy_experts}." + ) + self.num_ffn_experts = self.num_total_experts - self.num_copy_experts + if self.num_ffn_experts <= 0: + raise ValueError( + f"num_ffn_experts must be positive, got {self.num_ffn_experts}" + ) + + # Expected number of activated FFN experts + self.expected_ffn_k = getattr( + config, + "moe_expected_ffn_experts", + float(self.top_k * self.num_ffn_experts / self.num_total_experts), + ) + + # Bias update rate for computational budget control + self.bias_update_rate = self.config.moe_router_bias_update_rate + + # Load balance control (grouped) + self.num_ffn_groups = config.moe_router_num_groups or 1 + if self.num_ffn_groups <= 0: + raise ValueError(f"moe_router_num_groups must be > 0, got {self.num_ffn_groups}") + if self.num_ffn_experts % self.num_ffn_groups != 0: + raise ValueError( + f"num_ffn_experts ({self.num_ffn_experts}) must be divisible by " + f"moe_router_num_groups ({self.num_ffn_groups})" + ) + self.aux_loss_coeff = config.moe_aux_loss_coeff + + # Routing parameters + self.score_func = config.moe_router_score_function + self.route_norm = config.norm_topk_prob + self.route_scale = ( + config.moe_router_topk_scaling_factor + if config.moe_router_topk_scaling_factor is not None + else 1.0 + ) + + # Learnable router weight + self.weight = Parameter( + init_method_normal(0.02)((self.num_total_experts, self.hidden_size)), + name="weight", + ) + + # Expert bias (updated by controller; not trained by gradients) + self.expert_bias = Parameter( + mint.zeros(self.num_total_experts, dtype=mstype.float32), + name="expert_bias", + requires_grad=False, + ) + + # Ops + self.linear = mint.nn.functional.linear + self.sigmoid = mint.nn.functional.sigmoid + self.softmax = mint.nn.functional.softmax + self.cast = ops.cast + self.topk = mint.topk + self.gather = mint.gather + self.mul = mint.mul + self.div = mint.div + self.sum = mint.sum + self.histc = mint.histc + self.ones_like = mint.ones_like + self.reshape = mint.reshape + self.cat = mint.cat + self.zeros = mint.zeros + + def update_expert_bias(self, num_tokens_per_expert: Tensor, total_tokens: int): + """ + Bias update for Computational Budget Control. + delta_b_i = mu * (target_rate - current_rate) + Only updates FFN experts; copy experts are excluded. + """ + if not self.training or self.bias_update_rate <= 0 or total_tokens <= 0: + return + + # Current selection rate: Ti / (K * T_all) + current_rate = num_tokens_per_expert / (self.top_k * total_tokens) + + # Target selection rate for FFN experts: Ke / (K * N) + target_rate_ffn = self.expected_ffn_k / (self.top_k * self.num_ffn_experts) + target_rates = self.ones_like(current_rate) * target_rate_ffn + + error = target_rates - current_rate + update_step = self.bias_update_rate * error + + # Mask out copy experts (no bias update) + if self.num_copy_experts > 0: + mask = self.zeros((self.num_total_experts,), dtype=mstype.float32) + mask[: self.num_ffn_experts] = 1.0 + update_step = self.mul(update_step, mask) + + ops.assign_add(self.expert_bias, update_step) + + def compute_load_balancing_loss(self, router_probs: Tensor, expert_indices: Tensor) -> Tensor: + """ + Grouped load balancing loss: + L_LB = alpha * sum_{j=1}^{D+1} (f_j * P_j) + """ + if self.aux_loss_coeff <= 0: + return self.zeros((), dtype=router_probs.dtype) + + bs_slen = router_probs.shape[0] + if bs_slen == 0: + return self.zeros((), dtype=router_probs.dtype) + + # 1) f_j: frequency of selection per group + selected_flat = self.reshape(expert_indices, (-1,)) + expert_counts = self.histc( + selected_flat, + bins=self.num_total_experts, + min=0, + max=self.num_total_experts, + ) + expert_counts = self.cast(expert_counts, router_probs.dtype) + + experts_per_ffn_group = self.num_ffn_experts // self.num_ffn_groups + ffn_counts = expert_counts[: self.num_ffn_experts] + ffn_counts_reshaped = self.reshape( + ffn_counts, (self.num_ffn_groups, experts_per_ffn_group) + ) + group_counts_ffn = self.sum(ffn_counts_reshaped, dim=1) + + if self.num_copy_experts > 0: + copy_counts = expert_counts[self.num_ffn_experts :] + group_count_copy = self.sum(copy_counts).unsqueeze(0) + all_group_counts = self.cat((group_counts_ffn, group_count_copy)) + else: + all_group_counts = group_counts_ffn + + f_j = all_group_counts / (bs_slen * self.top_k) + + # 2) P_j: probability mass per group + expert_prob_sum = self.sum(router_probs, dim=0) + ffn_probs = expert_prob_sum[: self.num_ffn_experts] + ffn_probs_reshaped = self.reshape( + ffn_probs, (self.num_ffn_groups, experts_per_ffn_group) + ) + group_probs_ffn = self.sum(ffn_probs_reshaped, dim=1) + + if self.num_copy_experts > 0: + copy_probs = expert_prob_sum[self.num_ffn_experts :] + group_prob_copy = self.sum(copy_probs).unsqueeze(0) + all_group_probs = self.cat((group_probs_ffn, group_prob_copy)) + else: + all_group_probs = group_probs_ffn + + P_j = all_group_probs / bs_slen + + num_groups = self.num_ffn_groups + (1 if self.num_copy_experts > 0 else 0) + loss = self.aux_loss_coeff * num_groups * self.sum(f_j * P_j) + return loss + + def construct( + self, x: Tensor, training: bool = True + ) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """ + Returns: + top_scores: [bs*slen, K] + selected_indices: [bs*slen, K] + num_tokens_per_expert: [num_experts] + aux_loss: scalar + """ + bs_slen, _ = x.shape + router_dtype = self.config.moe_router_dtype + + # 1) Router logits + x_cast = self.cast(x, router_dtype) + weight = self.cast(self.weight, router_dtype) + logits = self.linear(x_cast, weight) + + # 2) Probabilities + if self.score_func == "sigmoid": + probs = self.sigmoid(self.cast(logits, mstype.float32)) + elif self.score_func == "softmax": + probs = self.softmax(self.cast(logits, mstype.float32), dim=1) + else: + raise NotImplementedError(f"Unknown score function {self.score_func}") + + # 3) Add bias for routing only + probs_for_routing = probs + self.expert_bias + + # 4) TopK selection + _, selected_indices = self.topk( + probs_for_routing, k=self.top_k, dim=-1, sorted=False + ) + selected_indices = self.cast(selected_indices, mstype.int64) + + # 5) Gather routing scores + top_scores = self.gather(probs, dim=1, index=selected_indices) + + # 6) Normalize and scale + if self.route_norm: + denominator = self.sum(top_scores, dim=-1, keepdim=True) + 1e-20 + top_scores = self.div(top_scores, denominator) + top_scores = self.mul(top_scores, self.route_scale) + + # 7) Statistics + num_tokens_per_expert = self.histc( + selected_indices, + bins=self.num_total_experts, + min=0, + max=self.num_total_experts, + ) + + # 8) Update bias (PID-like control) + if self.training and training: + self.update_expert_bias(num_tokens_per_expert, bs_slen) + + # 9) Load balance loss + aux_loss = self.compute_load_balancing_loss(probs, selected_indices) + + return top_scores, selected_indices, num_tokens_per_expert, aux_loss \ No newline at end of file -- Gitee From 9d81d1581e4d66ea29f7fb3bf9a46dbd395d2451 Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Mon, 26 Jan 2026 22:05:27 +0800 Subject: [PATCH 07/12] =?UTF-8?q?=E4=BF=AE=E5=A4=8Dconvertbug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mindformers/parallel_core/transformer_config_utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mindformers/parallel_core/transformer_config_utils.py b/mindformers/parallel_core/transformer_config_utils.py index 9af2574c2..7d48113f2 100644 --- a/mindformers/parallel_core/transformer_config_utils.py +++ b/mindformers/parallel_core/transformer_config_utils.py @@ -383,6 +383,9 @@ COMMON_CONFIG_MAPPING = { "expert_relocation_initial_iteration": "expert_relocation_initial_iteration", "expert_relocation_freq": "expert_relocation_freq", "num_copy_experts": "num_copy_experts", + "use_longcat_router": "use_longcat_router", + "moe_expected_ffn_experts": "moe_expected_ffn_experts", + "moe_router_bias_update_rate": "moe_router_bias_update_rate", # Context Parallel # not changes -- Gitee From 2426abc7285b7f943cda032defaa201779735e33 Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Mon, 26 Jan 2026 23:43:01 +0800 Subject: [PATCH 08/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9moe=5Fexpected=5Fffn=5F?= =?UTF-8?q?experts:=202.0=E5=8A=A0=E4=BA=86float=E5=87=BA=E7=8E=B0?= =?UTF-8?q?=E7=9A=84bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ds_pynative.yaml | 12 ++++++------ mindformers/parallel_core/transformer_config.py | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ds_pynative.yaml b/ds_pynative.yaml index 2bd7ef681..1a9b81642 100644 --- a/ds_pynative.yaml +++ b/ds_pynative.yaml @@ -234,19 +234,19 @@ model: n_routed_experts: 16 num_experts_per_tok: 8 n_shared_experts: 1 - num_copy_experts: 5 - use_longcat_router: bool = True - moe_expected_ffn_experts: float = 1.375 # Default best value: top-k * FFN/(FFN + COPY) - moe_router_bias_update_rate: float = 0.001 + num_copy_experts: 8 + use_longcat_router: True + moe_expected_ffn_experts: 2.0 # Default best value: top-k * FFN/(FFN + COPY) + moe_router_bias_update_rate: 0.001 moe_shared_expert_intermediate_size: 2048 moe_grouped_gemm: True moe_router_load_balancing_type: 'seq_aux_loss' - moe_aux_loss_coeff: 0. # 0.001 + moe_aux_loss_coeff: 0.001 # 0.001 scoring_func: 'sigmoid' norm_topk_prob: True moe_token_drop_policy: probs moe_router_enable_expert_bias: True - moe_router_bias_update_rate: 0. # 0.001 + moe_router_bias_update_rate: 0.001 # 0.001 # callbacks callbacks: diff --git a/mindformers/parallel_core/transformer_config.py b/mindformers/parallel_core/transformer_config.py index 4257157e5..3030952dc 100644 --- a/mindformers/parallel_core/transformer_config.py +++ b/mindformers/parallel_core/transformer_config.py @@ -254,7 +254,7 @@ class TransformerConfig(ModelParallelConfig, MFModelConfig): The default is "sub_seq_aux_loss". """ - moe_router_topk: int = 2 + moe_router_topk: int = 4 """Number of experts to route to for each token.""" moe_router_num_groups: Optional[int] = None @@ -358,7 +358,7 @@ class TransformerConfig(ModelParallelConfig, MFModelConfig): use_longcat_router: bool = True - moe_expected_ffn_experts: float = 1.375 # Default best value: top-k * FFN/(FFN + COPY) + moe_expected_ffn_experts: float = 2.0 # Default best value: top-k * FFN/(FFN + COPY) moe_router_bias_update_rate: float = 0.001 -- Gitee From 24c7a3f763651d1b0ecf5d06a86dd4fe31c96fb7 Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Wed, 28 Jan 2026 10:49:09 +0800 Subject: [PATCH 09/12] =?UTF-8?q?=E9=9B=B6=E8=AE=A1=E7=AE=97=E4=B8=93?= =?UTF-8?q?=E5=AE=B6=E8=B7=AF=E7=94=B1=E5=8A=A0=E4=B8=8A=E4=BA=86P?= =?UTF-8?q?=E9=A1=B9I=E9=A1=B9D=E9=A1=B9?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 773 bytes ..._model_embedding.py.baiduyun.uploading.cfg | Bin 0 -> 791 bytes .../rope_utils.py.baiduyun.uploading.cfg | Bin 0 -> 776 bytes ...ry_pos_embedding.py.baiduyun.uploading.cfg | Bin 0 -> 789 bytes ...ry_pos_embedding.py.baiduyun.uploading.cfg | Bin 0 -> 790 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 765 bytes ..._model_embedding.py.baiduyun.uploading.cfg | Bin 0 -> 796 bytes .../rope_utils.py.baiduyun.uploading.cfg | Bin 0 -> 783 bytes ...ry_pos_embedding.py.baiduyun.uploading.cfg | Bin 0 -> 792 bytes ...ry_pos_embedding.py.baiduyun.uploading.cfg | Bin 0 -> 797 bytes .../pynative/transformers/moe/router.py | 43 ++++++++++++++++-- .../base_model.py.baiduyun.uploading.cfg | Bin 0 -> 777 bytes .../run_parallel.py.baiduyun.uploading.cfg | Bin 0 -> 779 bytes .../test_llama.py.baiduyun.uploading.cfg | Bin 0 -> 777 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 786 bytes ...deepseek3_671b.yaml.baiduyun.uploading.cfg | Bin 0 -> 804 bytes ...n_deepseek_infer.py.baiduyun.uploading.cfg | Bin 0 -> 798 bytes ...t_deepseek_infer.py.baiduyun.uploading.cfg | Bin 0 -> 803 bytes ...t_deepseek_infer.py.baiduyun.uploading.cfg | Bin 0 -> 799 bytes ...t_deepseek_infer.py.baiduyun.uploading.cfg | Bin 0 -> 805 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 779 bytes .../test_moe_eplb.py.baiduyun.uploading.cfg | Bin 0 -> 786 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 788 bytes ...oe_token_permute.py.baiduyun.uploading.cfg | Bin 0 -> 804 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 784 bytes ...glm4_moe_infer.yaml.baiduyun.uploading.cfg | Bin 0 -> 792 bytes .../run_glm4_moe.py.baiduyun.uploading.cfg | Bin 0 -> 789 bytes ...t_glm4_moe_infer.py.baiduyun.uploading.cfg | Bin 0 -> 797 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 778 bytes ...en3_0_6b_infer.yaml.baiduyun.uploading.cfg | Bin 0 -> 790 bytes .../run_qwen3.py.baiduyun.uploading.cfg | Bin 0 -> 781 bytes ...test_qwen3_infer.py.baiduyun.uploading.cfg | Bin 0 -> 787 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 786 bytes ..._30b_a3b_infer.yaml.baiduyun.uploading.cfg | Bin 0 -> 803 bytes .../run_qwen3_moe.py.baiduyun.uploading.cfg | Bin 0 -> 793 bytes ..._qwen3_moe_infer.py.baiduyun.uploading.cfg | Bin 0 -> 800 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 786 bytes ..._telechat2_infer.py.baiduyun.uploading.cfg | Bin 0 -> 799 bytes ...elechat2_infer.yaml.baiduyun.uploading.cfg | Bin 0 -> 795 bytes ..._telechat2_infer.py.baiduyun.uploading.cfg | Bin 0 -> 800 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 777 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 777 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 779 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 783 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 804 bytes ...entropy_parallel.py.baiduyun.uploading.cfg | Bin 0 -> 853 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 772 bytes ...st_hf_dataloader.py.baiduyun.uploading.cfg | Bin 0 -> 784 bytes ...loader_broadcast.py.baiduyun.uploading.cfg | Bin 0 -> 794 bytes ...loader_streaming.py.baiduyun.uploading.cfg | Bin 0 -> 794 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 762 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 770 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 775 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 756 bytes .../test_utils.py.baiduyun.uploading.cfg | Bin 0 -> 775 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 779 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 789 bytes .../test_activation.py.baiduyun.uploading.cfg | Bin 0 -> 788 bytes ...est_fused_swiglu.py.baiduyun.uploading.cfg | Bin 0 -> 790 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 767 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 783 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 804 bytes ...el_cross_entropy.py.baiduyun.uploading.cfg | Bin 0 -> 822 bytes ...el_cross_entropy.py.baiduyun.uploading.cfg | Bin 0 -> 823 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 769 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 765 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 773 bytes .../test_activation.py.baiduyun.uploading.cfg | Bin 0 -> 774 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 766 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 774 bytes .../test_activation.py.baiduyun.uploading.cfg | Bin 0 -> 775 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 763 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 784 bytes ...al_mask_generate.py.baiduyun.uploading.cfg | Bin 0 -> 793 bytes ...al_mask_generate.py.baiduyun.uploading.cfg | Bin 0 -> 795 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 770 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 778 bytes ..._flash_attention.py.baiduyun.uploading.cfg | Bin 0 -> 785 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 749 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 772 bytes .../run_linear.py.baiduyun.uploading.cfg | Bin 0 -> 763 bytes .../test_linear.py.baiduyun.uploading.cfg | Bin 0 -> 767 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 760 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 766 bytes .../test_fused_norm.py.baiduyun.uploading.cfg | Bin 0 -> 768 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 765 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 775 bytes .../run_mlp.py.baiduyun.uploading.cfg | Bin 0 -> 764 bytes .../test_mlp.py.baiduyun.uploading.cfg | Bin 0 -> 766 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 765 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 778 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 787 bytes ...ransformer_block.py.baiduyun.uploading.cfg | Bin 0 -> 794 bytes ...ransformer_block.py.baiduyun.uploading.cfg | Bin 0 -> 796 bytes .../__init__.py.baiduyun.uploading.cfg | Bin 0 -> 779 bytes .../data_gen_utils.py.baiduyun.uploading.cfg | Bin 0 -> 787 bytes ...ransformer_layer.py.baiduyun.uploading.cfg | Bin 0 -> 794 bytes ...ransformer_layer.py.baiduyun.uploading.cfg | Bin 0 -> 797 bytes 98 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 mindformers/parallel_core/inference/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg create mode 100644 mindformers/parallel_core/inference/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg create mode 100644 mindformers/parallel_core/inference/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg create mode 100644 mindformers/parallel_core/inference/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg create mode 100644 mindformers/parallel_core/inference/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg create mode 100644 mindformers/parallel_core/training_graph/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg create mode 100644 mindformers/parallel_core/training_graph/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg create mode 100644 mindformers/parallel_core/training_graph/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg create mode 100644 mindformers/parallel_core/training_graph/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg create mode 100644 mindformers/parallel_core/training_graph/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/base_model.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/run_parallel.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/test_llama.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/predict_deepseek3_671b.yaml.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_deepseek_infer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_quant_deepseek_infer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_deepseek_infer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_quant_deepseek_infer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_eplb/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_eplb/test_moe_eplb.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/test_moe_token_permute.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/glm4_moe_infer.yaml.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/run_glm4_moe.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/test_glm4_moe_infer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/qwen3_0_6b_infer.yaml.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/run_qwen3.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/test_qwen3_infer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/qwen3_moe_30b_a3b_infer.yaml.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/run_qwen3_moe.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/test_qwen3_moe_infer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/run_telechat2_infer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/telechat2_infer.yaml.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/test_telechat2_infer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_parallel_core/test_inference/test_base_models/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_base_models/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_tensor_parallel/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/test_vocab_parallel_cross_entropy_parallel.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader_broadcast.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader_streaming.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_inference/test_base_models/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_inference/test_common/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_inference/test_utils/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_inference/test_utils/test_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_activation.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_fused_swiglu.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_training_graph/test_transformer/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/run_vocab_parallel_cross_entropy.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/test_vocab_parallel_cross_entropy.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_base_models/test_embeddings/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layer/test_activation/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layer/test_activation/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layer/test_activation/test_activation.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_activation/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_activation/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_activation/test_activation.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/run_causal_mask_generate.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/test_causal_mask_generate.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_flash_attention/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_flash_attention/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_flash_attention/test_flash_attention.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_linear/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_linear/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_linear/run_linear.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_linear/test_linear.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_norm/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_norm/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_layers/test_norm/test_fused_norm.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_mlp/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_mlp/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_mlp/run_mlp.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_mlp/test_mlp.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_moe/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/run_transformer_block.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/test_transformer_block.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/__init__.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/data_gen_utils.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/run_transformer_layer.py.baiduyun.uploading.cfg create mode 100644 tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/test_transformer_layer.py.baiduyun.uploading.cfg diff --git a/mindformers/parallel_core/inference/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg b/mindformers/parallel_core/inference/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9caae8fd2cb675dfdf8a44ccba74ba6d91e3a500 GIT binary patch literal 773 zcmV+g1N!`iF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!kB>NhH5;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN zAxjFUQ>hc^Xxka=d}C%*RF09c!v|>qW!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)>Ro3`@;VHK{d%D`w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmQ$>gK&?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhcw6 zKo9uv2ulB>mc0?9A@d=-?K9fO?je)NqYXHA%J>Zv(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyDE3 z0%e3A|9Kg=r~O;IRTvc|O8Q#S_N4Yw-zn&jsjYFWM*c>xqYq0Ky42xF_KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!X{>oYkD&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGKAWDyjjgkdmKN|de+LW)snS@c)b3& z5^zV;Z2m*I_Y;7*91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVq9g&Rh5g>W5A zE*V{;Q~DO~ZqpvUAZ%g)R@IuZ!diN1O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 z6At?FQy=`I%;OdFa`17uyJz0W?Rw72q7FZIoUQ_0ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJgtO<&_Fqua!%QHJ$gI*w7%;Y{LXoJ8!iaU!d6&#}w^g-5Axa{;kjm;N8|a(qcr z0B46@^gABEq5M{{TLoPSI{#JH<&MZ3$`85Fo~>_%M&Wd;vjZGN@~`1tt1&wofMNMq VG~J_(TKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!qG zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS=95>kgkve;+*2Ez-`d)tt7~dGD?3 z9U()3VfsO{_Y;7*91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS;IhD;G5g>)WI zFHTvfQ2H3`KhjO^b7M0E1lXRl$y$18O55&~%aVeM6L3g)`A|ziBHJhv*+btpe=W(LrT>215NA>@Jg?^IdLv$nsDCyxiVUXu>!k)Z}ICNAe6H4OaD$ z^Fkh7Y;5MhSX)`T^5ZoMx1C(EN6Qqf?yrSVC_e%XI+v#E@*Dr=XohFhrK#!7IB_Oj zO!<$Wx#2z_ZGWFWD1J*oIO`>}d0hwM7D)z01}AZe>nVnX$W}Mecrm|LCelp++kGU5 zX8)rWD(?nUk^~#BvWy5y9Nop$T<49*)}D?ZhQu8-g!);^xD{j-LaB^&L>g8lZGuG= zNCAJWN>$+?zQoM1P}-_ZDVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e7=shtRr`7N|%RTB}&HTpH;z&4ZLFcjy z%uB~1A@t$zCUVO@l#p+Vm7dK?j>HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{us-!55 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0yAV4rQc) zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip5NHzX_U1eIGc}f76+&hLX2~dAR$y z7j_raH~bRs@EViQGZm+Ip8m<6HFn|$OKtl|PNr%XhQfo^4WN-o)_0GMgcueihjt!I zEgV?zSE&~5Y0?_FAZcV!QI444m0fdsBlPQ?oRxw13r|gQ|9D~$DcMpy+)C6U6xl0I zEwrMq=~rh<{XPSO@_nx?%(BiV{al|kK=}c6LOY;yyf~Cn-_CvKIBHd1UizuPK6vkY zmDz3a+ze#GW?|-l&i1#ht23~vXEfJ?-y!SITJ{HZd#S#|ApO3IvHU~2w{D?Jc1fVB zQ~~$%21xs(p5#NOaH1i_y^VlXU&LEH3JP#HGv}gG}XUJrK2UpUxKhdTI(n zKxu_s_j?unru0y-Ru5AYa^M@v?T^R~)(@o6iI--#LFsa;z6lZp<@l`&#CsYEmu1XL T5Y*|}O@jhoF;TVnl#*6ShnKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!kA?=v|H&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSkrIEd`mjkebmGK+?TM_cDeiM z6nGkkGW8L&qZ5F+91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS;OgiH`}iFZm# zC>~#`0P_>-KHVF*acXK^0*jI3$qssHO55&~%aVeM6L3g)`A|ziBHJhv*+bu9%m-^5(t8(>Xl3ALY+ZCU79 zl7|51fL3yrJ~QZ%$K<2$?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ)a zLtOfz2uu9;pW{ZVb@_4X=r`V$=_!)Lq77nsmj4YC(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyDE3 z0%e3A|9Kg=r~O;IRTvcmbm3a(@RZ#h-6@ODmAhQOPVRH6vkn_gyYc84xJ?ZWgmUhB U1<1eKUil|OGGDOTyO&=Q@KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-7w?aGOc6pSdYw~9 zOHW{=NL}J1lgh@hM%u1kI+b43<4)0CjY{smaVex^&z!^of=B9pJrTQZm;M#IdtwAs v3vPp7{by8-obX({Ml@wqW$Fl*(Vp81$|8i`j-Wl}4B2L)xCRky{O$A_cg2EA literal 0 HcmV?d00001 diff --git a/mindformers/parallel_core/training_graph/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg b/mindformers/parallel_core/training_graph/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9b4391cde0863c32794e1c5e99b4be8dba48c0e7 GIT binary patch literal 796 zcmV+%1LORLF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!kF?>IRM&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS=RDzYm#5E*)!uB8SiahQs95b-VoT zL39+3X8sYr@)Ll$91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS;CgcuMfh$S2t ze?bhU0;xy7Ki(O?c0XoQ3*VaK#9w-8O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zK??iv1xxYw&9_9WBlRHb>NbzX>Ufx$rd?uvocsh>ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJgtO<&_Fqua!%QHJ$gI*w7%;Y{LXoJ8!iaX+_q!>zyofqmhBa{|09nD7YkL1GD2 z3TBxe`Fj?>toI$TUkP6xar#={@zli~!8f7aovmEAOz3l>hyoZ$@a6SQ`FTbUiz~Bv a0nVX-5QY;+HV*BPsE%J4tqt&I@fkEV4vmxm literal 0 HcmV?d00001 diff --git a/mindformers/parallel_core/training_graph/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg b/mindformers/parallel_core/training_graph/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..3ee15affdf17506693b557d7c83586bebfc62f39 GIT binary patch literal 783 zcmV+q1MvKYF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e6?K(CYr`7N|%RTB}&HTpH;z&4ZLFcjy z%uB~1A@t$zCUVO@l#p+Vm7dK?j>HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{us-!55 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0yAV4rQc) zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip2+W?E}t6co=HdBGj0u)Rp7YcJ8dX z9CJg~Z2ca*pcj+SGZm+Ip8m<6HFn|$OKtl|PNr%XhQfo^4WN-o)_0H6he{l9iE|kl zC_!KGRr?XSIo(X}A!1EKXT}yUtiEjhJ+vk)BVX8_ z_iY?vaUrg_4RKY2=h`XHwb3#OVLQ$-C3`_cKj%K%I%f#(CPgnvLVg_3$~uvO{Y4YlGInR=Wv6dKvujb^ zV71yNf0QBt?locsK+GOj8p@g39r}jO-JggBo`PR1hxkX2@&{@!5!u-?M;Bf$d4N(1 zUR+`7Nm%JCs>a)|Mu({bE1g2z;YI6Kph)qgJ7lqC%c{o*h(+0IU=yTk)$qsU~=lAqh$A26b%qz2h N6wtVY29O(HI}WOuhED(h literal 0 HcmV?d00001 diff --git a/mindformers/parallel_core/training_graph/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg b/mindformers/parallel_core/training_graph/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..37d86bce7d01e3fa82c49b6439c1b8d0bbcf23e2 GIT binary patch literal 792 zcmV+z1LypPF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!n9=sh_K&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGTGjGx(Sg_bQ^5bd5OxXfs*8ib-eq! z6mvs{H?Ks$@DqT!91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS{FgdG+yfPWev zEgAx&RjwAkV%{FOB5Gj-1CGnK#a()7O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zP73?<14H?(%di&kE%0~jyEf9r?SIV7p$cj&&8Y=iud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJgtO<&_Fqua!%QHJ$gI*w7%;Y{LXoJ8!iaX+_q!>zyofqmhBa{|09nD7YkL1GD2 z3TBxe`Fj?>toI$TUkP6xar#={@zmQB+cBobrMPUVeCd0vvI7xA`{ut??jlDlmu2ct W7VNf>3g}K83qZZmoSJnWpbUuM(2KYL literal 0 HcmV?d00001 diff --git a/mindformers/parallel_core/training_graph/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg b/mindformers/parallel_core/training_graph/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c8c1a0b8ea66cbe37ea0d1b9c822180f8270a4e8 GIT binary patch literal 797 zcmV+&1LFLKF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8>oYkD&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGTDuGu?fu`ei&ibeuA3#*OKajaO|k+ z6(vE0XQ&Y9qZ5F+91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS|Bh!z%e(sdaU zDIE#(1^5=QXxmQfa%(pR0p85wk`8)lO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zPFeT=10SoW$h{KxCZH#}yExduy)2x~_Xucw%KcIaud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJgtO<&_Fqua!%QHJ$gI*w7%;Y{LXoJ8!iaX+_q!>zyofqmhBa{|09nD7YkL1GD2 z3TBxe`Fj?>toI$TUkP6xar#={@zl^A-9C%WmAqo8TGM8sq&N^q_vH0J{%{Xbf^MJ% bH`%3=2c#lWDp>D=(yK!oqX^PT^bl`X@9T~V literal 0 HcmV?d00001 diff --git a/mindformers/pynative/transformers/moe/router.py b/mindformers/pynative/transformers/moe/router.py index bce13bb9b..0044afc97 100644 --- a/mindformers/pynative/transformers/moe/router.py +++ b/mindformers/pynative/transformers/moe/router.py @@ -258,14 +258,25 @@ class LongCatRouter(nn.Cell): ) # Expected number of activated FFN experts - self.expected_ffn_k = getattr( + expected_ffn_k = getattr( config, "moe_expected_ffn_experts", float(self.top_k * self.num_ffn_experts / self.num_total_experts), ) + try: + self.expected_ffn_k = float(expected_ffn_k) + except (TypeError, ValueError) as exc: + raise ValueError( + f"moe_expected_ffn_experts must be a number, but got {expected_ffn_k}" + ) from exc # Bias update rate for computational budget control self.bias_update_rate = self.config.moe_router_bias_update_rate + # PID gains and integral decay + self.pid_p = float(getattr(config, "moe_router_pid_p", 1.0)) + self.pid_i = float(getattr(config, "moe_router_pid_i", 0.01)) + self.pid_d = float(getattr(config, "moe_router_pid_d", 0.05)) + self.pid_i_decay = float(getattr(config, "moe_router_pid_i_decay", 0.9)) # Load balance control (grouped) self.num_ffn_groups = config.moe_router_num_groups or 1 @@ -299,6 +310,17 @@ class LongCatRouter(nn.Cell): name="expert_bias", requires_grad=False, ) + # PID controller states + self.integral = Parameter( + mint.zeros(self.num_total_experts, dtype=mstype.float32), + name="expert_bias_integral", + requires_grad=False, + ) + self.previous_error = Parameter( + mint.zeros(self.num_total_experts, dtype=mstype.float32), + name="expert_bias_prev_error", + requires_grad=False, + ) # Ops self.linear = mint.nn.functional.linear @@ -326,22 +348,35 @@ class LongCatRouter(nn.Cell): return # Current selection rate: Ti / (K * T_all) - current_rate = num_tokens_per_expert / (self.top_k * total_tokens) + current_rate = self.cast(num_tokens_per_expert, mstype.float32) / ( + self.top_k * total_tokens + ) # Target selection rate for FFN experts: Ke / (K * N) target_rate_ffn = self.expected_ffn_k / (self.top_k * self.num_ffn_experts) target_rates = self.ones_like(current_rate) * target_rate_ffn error = target_rates - current_rate - update_step = self.bias_update_rate * error # Mask out copy experts (no bias update) if self.num_copy_experts > 0: mask = self.zeros((self.num_total_experts,), dtype=mstype.float32) mask[: self.num_ffn_experts] = 1.0 - update_step = self.mul(update_step, mask) + error = self.mul(error, mask) + + # PID terms + p_term = self.mul(error, self.pid_p) + integral = self.mul(self.integral, self.pid_i_decay) + self.mul( + error, 1.0 - self.pid_i_decay + ) + i_term = self.mul(integral, self.pid_i) + d_term = self.mul(error - self.previous_error, self.pid_d) + + update_step = self.bias_update_rate * (p_term + i_term + d_term) ops.assign_add(self.expert_bias, update_step) + ops.assign(self.integral, integral) + ops.assign(self.previous_error, error) def compute_load_balancing_loss(self, router_probs: Tensor, expert_indices: Tensor) -> Tensor: """ diff --git a/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/base_model.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/base_model.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..66f597b6599f74b6deec78db46777952c5252dcb GIT binary patch literal 777 zcmV+k1NQueF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!eC zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSJd_>jNAYbG``WzI2d24UhkSsIhf5W5gCs^D zdq@bR0{lSiY2QohAUyFj2bFJ8a4 zM+W$!03Q1Gm#{#hC-)$}?PS@+xGc_?qzG&+pYsI`ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQeiyT1ndnsDdwddZ z0%eX=_B#)^uim^ZSaOq^YvK1vu;pwj!xJwulop$(a H4bZaN^U;JS literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/run_parallel.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/run_parallel.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..f3ea0c014aba1445dde72a4665a87498951908fc GIT binary patch literal 779 zcmV+m1N8icF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!G5 zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSJ(7>H^LkbV_W}e$vdV(81?`b-by* zKzI?3WcU`c^%H=(91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSs9gH0W9(tk@I zEFD>;R{ur4YTh2|CO&Fb3*DUL$XI%5O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zM-Tj@2pp@eny?r5cJ_3==r!8K?JdZds#s!wn)?V}ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQeiyT1ndnsDdwddZ z0%eX=_B#)^uKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!d} zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSkv5>It73bsId=DAt~&(8RHbChYm^ z9CaFkZLUVU^Amu%91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS#L(oGg2)qWaJ zEK6PYS^q%mVvSC^b7E&*2h_^6!Crc4O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zL^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQeiyT1ndnsDdwddZ z0%eX=_B#)^uKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k9kaU>0R|dKNxXH~JL5ssQPh85y5_zVOkqJr1}IGdR&i5cY0q!r z4fXS)=w5A0^EnNK`h1}_w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmS#^j^#?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhc+7 z5()V62Op@VoZ}n!aPoG(y=U0Ky(*H-rVluIo&OCJ(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxye{lVkK0 QSd)U44!ILo7+JmB!>|8`82|tP literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/predict_deepseek3_671b.yaml.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/predict_deepseek3_671b.yaml.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c54b469ed0441300758f5d49f6db513a4f5596a5 GIT binary patch literal 804 zcmV+<1Ka$DF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e6>^M0J&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGK7hJunC?_D<5IfEz`{YhmqxoaqIiJ z5p+X@XRbuO@)Ll$91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVr5gBKPcgCHIf zEKXRZ1E>|hV%kZ#A#Y{`0p7~C$6tDBO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 z6J7lN15Enz&%GC^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OY z3U8W1|8WrE?Cn>kMl@a@aN|ea=a8Bs#y+yzi05;tM(BU7qc{;y@bTh7@^xDv#2~Fm iO2xK_4(eTCJ3^=1kFHV!q*sa{$P8u%6kWUO%BLWtW|AZT literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_deepseek_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_deepseek_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..ac0bc73d4e6ee92700de02c730ef879235b1de41 GIT binary patch literal 798 zcmV+(1L6FJF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!X{>OVOO&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGK7kFunwI~e@<=RDbvdMfRnX{c(|>< z6m>v_VgD1juM>c|91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVr4h8Gbdf^<(F zAxc=I1+N$HZPOa=Cu(O00N9)1!UB3}O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 z6JGiL1W)^>p0OIBckn0ex@X^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OY z3U8W1|8WrE?Cn>kMl@a@aN|ea=a8Bs#y+yzi05&yP1AC#vKIzV@aoP@`FITt+-Kz$ cOYgFm6te(hKSa5J!O><1pa$1#+ZAREON6hEIsgCw literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_quant_deepseek_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_quant_deepseek_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..1565e24f9bb9e429f74ae4c739af3a7723bec351 GIT binary patch literal 803 zcmV+;1Kj+EF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!Y9>o_?I&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS}TKz6p>SFC1;veuK)T(463iCbgyP z6m>_2Huyp4@DqT!91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS>E(is*m)+8Ms zD;)@tpe=W(LrT>2!k)Z}ICNAe6H4OaD$ z^Fkh7Y;5MhSX)`T^5ZoMx1C(EN6Qqf?yrSVC_e%XI+v#E@*Dr=XohFhrK#!7IB_Oj zO!<$Wx#2z_ZGWFWD1J*oIO`>}d0hwM7D)z01}AZe>nVnX$W}Mecrm|LCelp++kGU5 zX8)rWD(?nUk^~#BvWy5y9Nop$T<49*)}D?ZhQu8-g!);^xD{j-LaB^&L>g8lZGuG= zNC9S`S4H$-zuktmMux6)Hi=xxB#mK!8WnY)w*`HeEnmuxf3c*@Z{lF@_Pdyn|bQ8y?GR-M5r35L!~7{alav4e literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_deepseek_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_deepseek_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9b3147e4cc44f7e925e74bf412f2c9430727cc90 GIT binary patch literal 799 zcmV+)1K|9IF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!b8?L9dP&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGJuJH?hKwtE=z6KE{4jefx)nadF=b| zMkWxCW&B36^Amu%91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVf1fk_h}hb2lL zDNPNdRjx(qZ_-YyFj2bFJ8a4 z7hV3K22B3dW^jTwmo~sC1ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OY z3U8W1|8WrE?Cn>kMl@a@aN|ea=a8Bs#y+yzi05~|Q20rzvH}HJ@Z|b?{%{T&fH~|_ dSd)Z+5TF-94pF7n!@?XQyam~A`8s0-5C^0Rk2wGU literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_quant_deepseek_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_quant_deepseek_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..b7103922c5119ab69931b62f2884a424820351da GIT binary patch literal 805 zcmV+=1KRwCF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!b7?=v|H&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSS|D>jNAYbG``WzI2d24UhkSv7flC!~(sxcy zD;iz!SN|02JJ}h#a${v(P}G~}mkxSqO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zMhgC-03NHQpWzVja-(&U^BVrBgX}o%mM?ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OY z3U8W1|8WrE?Cn>kMl@a@aN|ea=a8Bs#y+yzi05~|Q20r|q6QmY%H{My=WPudnMn0S j0@KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k9?l&r9;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN+_Z`&X2B5Y$(QH-18!3SvoW!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)>0M_@`Y{WF`g@@^w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmS&E%u+?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhc(8 zLR$Uy0v@lYpRq&qC-EWZyENLs?th%krU*KGnEDM9(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!Y7>M}VB&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS`oM>jjNAYbG``WzI2d24UhkS?7f=m}7hkh9v zcuiUHSpPz?Kif^~c4KN?2h`5vz+QT3O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zKwJH#0!{v`$hAYKcA|Q{?>UdazATu^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OY z3U8W1|8WrE?Cn>kMl@FyaLiH2?wN)N$|IoQm#1chM*MoL`4e{lVkK0 QSd)U44!ILo7+JmB!=X@%h5!Hn literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..00d9084d452cf0548488546dd5340d45c320b1f6 GIT binary patch literal 788 zcmV+v1MB>TF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8?l&r9;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNXAv!r z4fXS)=v`t<{y7ea_w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmT&E%u+?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ-o zM_K>>1RMJBnB)`iarYp(>M+~DxqQpYt5|7!o&60H(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyMt3KUA03k(&+B7>V%! literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/test_moe_token_permute.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/test_moe_token_permute.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..0a2eccbd9ea3b0657f8db4a8a0bb3ba038adbb3b GIT binary patch literal 804 zcmV+<1Ka$DF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!bC>^(UO&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGT7dIx>%k_C>?LoD~HOe)sp9jaPO+W zLUcobVD%rr^Amu%91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS^EgB?d8ha(&u zEJzHaSNaycZQD)hCp=~Z1=*49#a?=8O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zPYV3>QcV8w&9fJxBk*;&yEWawzAnj{rV4C(m+}Q!ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OY z3U8W1|8WrE?Cn>kMl@FyaLhp0>7K_U-!-PvkGy8}SLtQChzc4{%Ju$F@^Ci~mu2x^ i5ZU>a8|qzPJ3^=1kFHV!q*sa{$P8u%6kWUO%BLXi!II$s literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..3ac51db3b7c4ac2922ba5ce1b814358839ee5f03 GIT binary patch literal 784 zcmV+r1MmEXF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8={G84;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN8)fho=)gwn8 zDj5T$QTar;XpKv~B4S}tR@!r z4fXS)>RDk;_&*AO`*EQ*w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmT%H*T(?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ%l zLRdpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxy&c>Gc4>y^(L-G7VVlcs8}eA76-u3K7p@bl+EKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!G4>o+Q6;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNCkgf7QsZfRwd?cj*4U z9d}36HLf4+ssQPh85y5_zVOkqJr1}IGdR&i5cY0qi zd>me(Qm+v2Yurk`CSzp)QH;v%mj`J8W!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)=~-Y*`a2GY_;{fM-4vzk8F*_Xs*ImaGjE(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxy&c>Gc4>y^(L-G8&wm;79}OzBCdxd9MZyZG=)`yxjymu2ct W7VNf>3g}K83qZZmoSJnWpbUtrd5VPq literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/run_glm4_moe.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/run_glm4_moe.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..be632016d96edc0616468ffa3d63ee206d876ea0 GIT binary patch literal 789 zcmV+w1M2*SF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!tC=s7tG&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGT7dD>kghCbWLrBDu&Oi)sVJjNAYbG``WzI2d24UhkS^EhD;G4(SJ-H zcu5VWR{9j|YmOVdB{*bMQ`(W^#|nCBO55&~%aVeM6L3g)`A|ziBHJhv*+btpe=W(LrT>2!k)Z}ICNAe6H4OaD$ z^Fkh7Y;5MhSX)`T^5ZoMx1C(EN6Qqf?yrSVC_e%XI+v#E@*Dr=XohFhrK#!7IB_Oj zO!<$Wx#2z_ZGWFWD1J*oIO`>}d0hwM7D)z01}AZe>nVnX$W}Mecrm|LCelp++kGU5 zX8)rWD(?nUk^~#BvWy5y9Nop$T<49*)}D?ZhQu8-g!);^xD{j-LaB^&L>g8lZGuG= zNC9S`S4H$-zuktmMux6)Hi=xxI|Drs_tDRR>p2O8#5j-IB)|$|C#CkEUF`PvIhwx(E;i<@l`&#CsYEmu1XL T5Y*|}O@jhoF;TVnl#*6SphAi0 literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/test_glm4_moe_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/test_glm4_moe_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c751417db70d47da43948c4a553e63d5b18bd5d9 GIT binary patch literal 797 zcmV+&1LFLKF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!b5 zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS$+1x(v=8c^W%`dDG3L(Z=i4Chw)W z79~W1G5Qheq!WO-91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS*Ng&h-c(sCO} ze;Qb%S^h$?K8;Q0A!s)P1c;vFkO_KeO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zLI(Qu1{|xP%I8GzclUd|?>UZ^z9G-f_YG|-nEX%^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OZ z183fD_j?}wxBNk&Z46cyAa@ZR`J@p~Fkf^MJ% bH`%3=2c#lWDp>D=(yK!oqX^PT^bl`XiK&hq literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..7a9b70fe3f2f0554875166e5e610526ca271c4f1 GIT binary patch literal 778 zcmV+l1NHodF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8?l&r9;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN_CTnY4QH+r6lLu)4W!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)>0Dw${4ouQ_;H~&w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmT&E%u+?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhc(9 z5DKsGS4jKvn6?nAaiu5gy*Jpw=zE*Rs|Y%Gm#7UB(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e7=s7tG&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSS~DyjaW}EK6X6D2SM;fR^BgaK8KN zK_n5?G5#O#q!WO-91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSv6*BeK2(7oCl?i%jO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zMhmF&QXZ=Ho4iJ$ETVSpzcSsx?|+?{_E^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OD z7;l6EwO}5hx5h)FR|N?reEm}0@YL7~%0B1d%9k~`eC2qkvK3oX^z8i@xJ?ZWgmUhB U1<1eKUil|OGGDOTyO&=Q@WaF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!tF>^(UO&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGKGjMz6i=1cuio0DTJM*fs^BgCho27 z7I+qhHmVfw@DqT!91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVu7)fpFb)pJY} zdP)!US*{nkY>iCrc4%r*1CP$;m0fyiO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 z5?reA0vN2JoxDMyAg6omx-s0AzbTl`s$Xg?$oW-Tud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OD z7;l6EwO}5hx5h)FR|N?reEm}0@YLH0!4IX#pQbSMQ}rt0gaQX#@bke%<91d<-9qd} L3*ED$3%VB#=h%ex literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/test_qwen3_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/test_qwen3_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..69936bc8f64e6500d267a10f4c668add369b2339 GIT binary patch literal 787 zcmV+u1MK{UF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!b5 zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSu2Cz7LriD;Q|deSn?%)5NudbG-iV zMj=6iY_Af(rW1g<91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS&E)EN*b)gT!g zE*=Z?2mCtpe=W(LrT>2N45EzbeR`@n2&soBja+yxiVUXu>!k)Z}ICNAe6H4OaD$ z^Fkh7Y;5MhSX)`T^5ZoMx1C(EN6Qqf?yrSVC_e%XI+v#E@*Dr=XohFhrK#!7IB_Oj zO!<$Wx#2z_ZGWFWD1J*oIO`>}d0hwM7D)z01}AZe>nVnX$W}Mecrm|LCelp++kGU5 zX8)rWD(?nUk^~#BvWy5y9Nop$T<49*)}D?ZhQu8-g!);^xD{j-LaB^&L>g8lZGuG= zNC9S`S4H$-zuktmMux6)Hi=xxKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8>NhH5;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN z5h4+QV)zlcssQPh85y5_zVOkqJr1}IGdR&i5cY0qbb8j$K1>2nC!v|>qW!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)=}}=z_&*JX@^GOxw)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmT$>gK&?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhcz9 zMh2*$R~h@DoZ%OucA#|ax@Fvyzb>DipdpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyI|Drs_tDLmXceCd^sa=cK?A$u*{srI%{7O87~U`4e{lVkK0 QSd)U44!ILo7+JmB!)j58WdHyG literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/qwen3_moe_30b_a3b_infer.yaml.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/qwen3_moe_30b_a3b_infer.yaml.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..48fbe44827fbf521d64ebe8483e6b8c5bcb8c943 GIT binary patch literal 803 zcmV+;1Kj+EF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e7=sr0L&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSb_4u?fjeC`vzuEYZWH*2A!dbnmX} z6(SndX81(Bq!WO-91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSyBh87lgfg?*y zcu!iYS@}omY~M}oa%(VMP~V)k#tC|9O55&~%aVeM6L3g)`A|ziBHJhv*+bu9%m-^5(t8(>Xl3ALY+ZCU79 zl7|51fL3yrJ~iu>%H*T(?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ`l zK?eM%03ZGG&$2_REb}Gmx;2Qy>L|#>@L6vxnfMJ8(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyI|Drs_tDLmXceCd^sa=cK?A$u*{si@#>H5Yv01vN%Q>_ulXq^+5pQ8y?GR-M5r35L!~B4>ki7r^ literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/run_qwen3_moe.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/run_qwen3_moe.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..de7a3114f8d1a86aaeb5d4707810086033694610 GIT binary patch literal 793 zcmV+!1LpjOF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!tH zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSb|AxB|`|e;Yf4dDh9S-JG(6b-Av+ z6L~~|HmDKl^%H=(91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSyAgC8Am(R3O} zeHvY$QSun*Y2F*{C2ea11dq()lURCbO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zMF{-%2pp)U%jZV$a`SVrwP4xB=qk(0s|{@}%KTGbud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OD z7;l6E#d#FB?Cn>kMl?$eaQz48@083V#y+yzi05&yP19$&vI$0Y`0n%y=Vw$

pnA X0nUen7QRdmab=;>lAl*Gr3KS5;6RQG literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/test_qwen3_moe_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/test_qwen3_moe_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..29530b796c4055c38e1a136feed261c221ff9824 GIT binary patch literal 800 zcmV+*1K<3HF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!b8>^(UO&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS|~A=~>Didq{18eT1F;hL*C1A?&Z| z5^xZQW%)z8qZ5F+91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS>O(oRNn(Qru~ ze;yC?0I3q_Y>H0oacp8&0^FUkzz%w8O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zKnJa-Qy8eCn6X6kAg3bhzhK;vy(^!}qFHNxo2~!}ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OD z7;l6E#d#FB?Cn>kMl?$eaQz48@083V#y+yzi05~|Q20r|paL5i%Iy9@$ae-Amu0eA e9f0_@2c!g6EMTz8mB0^JnHkz({1{9C8C=6y9*uhd literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..d2cae9af0066104e04c2102cd203becf450a1d53 GIT binary patch literal 786 zcmV+t1MU2VF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8>^CZ7;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNXQ~MR@Jc&)ec0X!WP}-lgk_Tx3W!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)=}}-v`aBDU`+lJ|w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmT&*Y=;?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhcq2 z6bSyJ0U7@FowFLFA@g&+?PrLRy(`beq6utxU)(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxye{lVkK0 QSd)U44!ILo7+JmB!-y@0ng9R* literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/run_telechat2_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/run_telechat2_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..a564e1f93e72c9ef18f8d9380dce5cdd485aa1ae GIT binary patch literal 799 zcmV+)1K|9IF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!a|>N7bC&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGKAfFx&xUO_ z5g{6YHm@JO^Amu%91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVq6gdH3wh<+Lx zE=dod0Q(v2Zrn@lac?tS1&_<^$6k7AO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 z6AAnD2}|*;%e57wAoC)*v}E4LxGJB?^HXds&GH9Xud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OI z3TBT|@^ug3?Cn>kMl?KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!G0>o+Q6;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN!r z4fXS)>QG=q{5%SV{d%D`w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvIH$mFB%?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ=m zLI(TtR73Ibm*GLAa;GM~zhsP*>wV74t5RZpmiY}6(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyU=052jgnjzs4w4R_(H;ai(&u( literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/test_telechat2_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/test_telechat2_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..07111a5776ccdbd4821b5de796c3e6de9ba293e1 GIT binary patch literal 800 zcmV+*1K<3HF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!b8?LIjQ&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS=F8un3tMd`n=1DAdWRfW+W|b?&Y0 z5G56bH~AE~^Amu%91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS;GhZh$o)^|!1 zeM?`URIfm~Yu-%mb#6CO0FIsJ!(Mu6O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zK@Rx-0!y!@o4iJ$aPxNU?K0cNzbTu@_grZ#oB0D-ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQe;u%9mc9@1ZF@OI z3TBT|@^ug3?Cn>kMl?KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP>+NJZ`tugKe`P^!62D49c&`Aq#zpj-c>KPRbj#+=Fllt-y~cLAVWlm8v@Y-$Kn z3MHOY>0l(VqUu+QUj$wsdEgu8-IB)@+YgJ8im^ZSaOq^YvK1vu;pwj!xJwulop$(a H4bZaNbyKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP>+NJZ`tugKe`P^!62D49c&`Aq#zpj-c>KPRbj#+=Fllt-y~cLAVWlm8v@Y-$Kn z3MHOY>0l(LvG`o3R0K#DaN!&0-IB)@+YgJ8im^ZSaOq^YvK1vu;pwj!xJwulop$(a H4bZaNcAAG| literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_base_models/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_base_models/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..97086c49a55d985a801dc18d91e4f1d22340c14e GIT binary patch literal 779 zcmV+m1N8icF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP>+NJZ`tugKe`P^!62D49c&`Aq#zpj-c>KPRbj#+=Fllt-y~cLAVWj^i5ra8Ln5 z4tjzf?r{jFocmLzS2R}@arRitt)%u9!8xGdrRH;^AEQO9r4bNV#o*;ksUk-zf_w8z J0Q$a(2k3~;h&2EJ literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_tensor_parallel/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_tensor_parallel/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..88970d419ad03cbab9409752d2ce97cf6c925453 GIT binary patch literal 783 zcmV+q1MvKYF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP>+NJZ`tugKe`P^!62D49c&`Aq#zpj-c>KPRbj#+=Fllt-y~cLAVWj^i5ra8Ln5 z4tjzf?r{jFob6Y;K?F!8W%5SZ?w8CEfDen-l%aQpe4=Kz>qsU~=lAqh$A26b%qz2h N6wtVY29O(HI}V`lh|2%~ literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..ebcb347bf5e81df6ba98df506211ce05839944cd GIT binary patch literal 804 zcmV+<1Ka$DF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP>+NJZ`tugKe`P^!62D49c&`Aq#zpj-c>KPRbj#+=Fllt-y~cLAVWj^i5ra8Ln5 z4tjzf?r{jFobO$>Qwe1Xbm3Lh?w-XX%p$wYiLGs}QR8N}h!+%C^zHpY`!P2ln|bwK iJLCD48|qzPJ3^=1kFHV!q*sa{$P8u%6kWUO%BLWZsg!X5 literal 0 HcmV?d00001 diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/test_vocab_parallel_cross_entropy_parallel.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/test_vocab_parallel_cross_entropy_parallel.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..0c163c6c5e8edec73a1579bbeed93bf4c39a6ffc GIT binary patch literal 853 zcmV-b1FHOnF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!b7?=m?G&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS`oOx(driFBoprDb&NQhQs25BfY8Q z9dJW{W3CqNq!WO-91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS?7fE^Wei6j{v zDoS6fP^m|_Xow%`d}Cq+1=`BwlnHuiO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zKwJOx02%Q1$g&ZjaQSlU>M+=o>@AYTq6%zy%==Uhud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9}^(0Vt$o&#;CXgHiHQViTrfmi-?1L2L;_ z3MHOY>0l(LvG`lNZ45{oW&0b<=ZM`S**~}5n3r?AQSEx7vI{Cn=kDTH@^BmtooD5B z6y2qS4!Rd#aYVGuiJ?UpuL#so?Hzas8C96?`JrrfuHuLGjxQoYgCFUKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8?Kdi8;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN!r z4fXS)=ul)r{67te{dl1^w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmT&g7%-?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ}e zL=X1yR7tF*&9xSxaP%RuxHgQ)dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyJwfKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!F?>^nIM&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGJw-}>|LB0cu#JFFVdO*(!=1jNAYbG``WzI2d24UhkVeCiAzEv)^koB zeN0*K1N}$3JC7RYAUI%D1lXPI#SeOEO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 z7YFvDS5B{?pRz`#cB3J`>N1Oz=^)OYpjvHuoAU(-ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k9 zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSb?1u?x*lC`)gGD2c-VfRy9bA-S%- z6m}YpWcm}hr4xX;91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSyCh!zxcfFnvA zd`kkW0yFj2bFJ8a4 zMGp7>0U!DCo3lruE%+eqy)lfD=qk(1q*G#lnEnC}ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!nC>pM9L&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGT4cKzYLm8EKWbsDAB|CftTuqce<{) z6LCj|HvB@pqZ5F+91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS_Effy4cgC|c+ zFH2gg1+EjYZ`>H|AZ=p+1CN-n#}0aFO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zPhI};21xq#p5qjub@?akzBu2<>nhI6qYi2^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP3l3q|5Qxc1qjQIxhoK9FzT;#=z#i%#vL0V1n$&#}w^g-5A*cLAVWnD7YkL1GD2 s3TBxe#Y+;rs_kux92-MMI>1rN^q$)+&Lq5n#^+Y5Pvdv0kq#M8tCS#qrT_o{ literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_parallel_core/test_inference/test_common/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_inference/test_common/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..78ad5a468ee5a2d3c221c2f09de832953fb58056 GIT binary patch literal 770 zcmV+d1O5DlF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!kB={G84;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNPYKl$gb7N*yRNI)g$OmZvW!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)=}=}*|2z(d{(GS|w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmQ%H*T(?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ@r z6I%K6R80G#p0X69a-e#>zA=lEdpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxy~98GA^k|$*kK8$yVcjW-YyKob(CtaBKq} z8*iCK#dj3HuJ&B|Z8TF9eCKQDtc%bqsD4`R?#Q(RTzI%x=+x A5dZ)H literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..52aaaa4ac22b4ecc76cc9d52d1d7cd83ea68f441 GIT binary patch literal 775 zcmV+i1Ni)gF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!kB?l&r9;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNg_5y{bh@dz z6m%PfW35KAssQPh85y5_zVOkqJr1}IGdR&i5cY0q!r z4fXS)>04n*{yhVP`FNo=w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmQ&E%u+?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhcmf zP7eA0P)x6^nX?q7cc3D^?l#`X=zN;Ut66L=&+`ov(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxy~98GA^k|$*kK8$yVcjW-YyKob(CtaBKq} z8*iCK#b61ut?E>|Kn7n3I?QX)?2_Fh@P4Gm#^+GKQTlVdhXw&ctG&K)=W+)gmQnRa F0fJ*Zf+_$2 literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_parallel_core/test_inference/test_utils/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_inference/test_utils/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..20d423b9deddda353af6dfa4a11372111f024a39 GIT binary patch literal 756 zcmVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP3l3q|5Qxc1qjQIxhoK9FzT;#=z#i%#vL0V1n$&#}w^g-5A*cLAVWkL42hZ6G;r m24sUp#Ycy2)`XIQ_o2*5%Pw6PAB77|X literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_parallel_core/test_inference/test_utils/test_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_inference/test_utils/test_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8ff9709aecb4b4448c7282d0a5847c2c31af9492 GIT binary patch literal 775 zcmV+i1Ni)gF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e8>N_(Tr`7N|%RTB}&HTpH;z&4ZLFcjy z%uB~1A@t$zCUVO@l#p+Vm7dK?j>HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{us-!55 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0yAV4rQc) zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip5uTzF3?|dP_Lae}kE%)s?V@BfR*% z6?qhfWcwAm@Jf@=GZm+Ip8m<6HFn|$OKtl|PNr%XhQfo^4WN-o)_0GBg&9X8hH*+u zE*uE(S^5#`Y=|DaC2MO_1KpqJz+EGHBlPQ?oRxw13r|gQ|9D~$DcMpy+)C6U6xl0I zEwrMq=~re*`7sZI`F^P>%(BiV{al|kK=}c6LOY;yyf~Cn-_CvKIBHd1UizuPK6vkY zmDz3a+ze#GW?|-l&h@pcsWY&tXEfJ?-y!SITJ{HZd#S#|ApO3IvHU~2w{D?Jc1fVB zQ~~(%2O0gMnXp5paPlFsy)lW#zbMb0qf%{tp8W$2_4|$xNyQo`ikCh1Kc`OvCRYEj z^kEc5R6D8i4-0yR?&=R4zq?ubFq3)yy^exxA}%yqal)+Q_Zac-aPl{lgs#=cDMxV| zdaK&6u-i00TPV?LFMdf$IOIChDH}KIEJGP{QDH~Yr3{9J?ou)7Cu%YEtlU=*+(s0R zM#-QSHth~pofJQ5RdP~W0L_@sSG z0cnpO?r{jFoa{!tUKkY)aNt1a_m993+KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k9=r<~3;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNdmTUCB7@EO)t2Fcb?cTZAwM<-R*cQ!k_Tx3W!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)>0E6{|2P7J{CJ@@w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmS%jBc)?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZoU zLk{-!QyBmE$h;e&BBXMdpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxy~98GA^k|$*kK8$yVcjW-Xv`oA?#|aA-MJ z9c!9dwO}5hx5iVpMgmM5VftIvt)%u9!8xGdrRH;^AEQO9r4bNV#o*;ksUk-zf_w8z J0Q$a(2k5)xgfIXA literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..6810be0e287ec1d0e09e806b72a12ecbb20bd4d0 GIT binary patch literal 789 zcmV+w1M2*SF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e8?L06Ur`7N|%RTB}&HTpH;z&4ZLFcjy z%uB~1A@t$zCUVO@l#p+Vm7dK?j>HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{us-!55 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0yAV4rQc) zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip2jd zd`S!QSNlTkKaWfACq7{YSJ|Dk##wiIBlPQ?oRxw13r|gQ|9D~$DcMpy+)C6U6xl0I zEwrMq=~re=`#KDS^LVK*%(BiV{al|kK=}c6LOY;yyf~Cn-_CvKIBHd1UizuPK6vkY zmDz3a+ze#GW?|-l&h@gasxz>uXEfJ?-y!SITJ{HZd#S#|ApO3IvHU~2w{D?Jc1fVB zQ~|5;21BUvnCB7qCGjG^ykd@(xP6@vn{$NyQo`ikCh1Kc`OvCRYEj z^kEc5R6D8i4-0yR?&=R4zq?ubFq3)yy^exxA}%yqal)+Q_Zac-aPl{lgs#=cDMxV| zdaK&6u-i00TPV?LFMdf$IOIChDH}KIEJGP{QDH~Yr3{9J?ou)7Cu%YEtlU=*+(s0R zM#-QSHth~pofJQ5RdP~W0L_@sSGkMl@R)VfsSZ_L9aDfHbwso0nw2P19q#z6%Bf<@l`&#CsYEmu1XL T5Y*|}O@jhoF;TVnl#*6SjSz>Y literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_activation.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_activation.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..4b08320731166e3ff199c45f6db298f71dab93a4 GIT binary patch literal 788 zcmV+v1MB>TF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!Y5=rTD9&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGJuYEy$qQfbx%6iD}kP@fWha0cfJ1P z7I{I3HTXoc^b>%&91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVe{i5e3p)P7D$ zDM$+UQLjb4YTO^ad}wP10*sut$ys`7O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 z7h3l721~81pXL&xC8Trh=r!8M?0S;Us|af@nyCq1ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9KPRbj#+=Fllt-y~cLAVWj^i5raApWr zFm0Gn>U>nDrs_tDQwUBIWAZ@J?UjZI$|IoQo2z%TRqbM@xCvWY?7hBs@@yU&fJgma S41(**H>Mt3KUA03k(&*k!ijzW literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_fused_swiglu.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_fused_swiglu.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..44893c4bbf18ffc7820c696c1dac442efe4092a8 GIT binary patch literal 790 zcmV+x1L^#RF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!d}>pVFM&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGJ@ZEuwI@jNAYbG``WzI2d24UhkVk2hZjL0i6coL zElv-r0Q?oWJB}IaaBDLG3XaL=!&-W2O55&~%aVeM6L3g)`A|ziBHJhv*+b-`n9M^lhXsRrD@IQ){7rzCycK*6CB^G-7glME&E!KvwK` zo&Q9j$zOiac{%x$#__bdzh&aQdnW3c_ARW4SN0b)DXIUGF72U+z_59>yFj2bFJ8a4 z6$|+OR7~*kn6MV4Ao(J`xG;;A>MWkk^ABSz&-epfud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9KPRbj#+=Fllt-y~cLAVWj^i5raApWr zFm0Gn>U>nDrs_tDQwUBIWAZ@J?UjZI$|IoQoV{?rN7H4wz5p3tyYc84xJ?ZWgmUhB U1<1eKUil|OGGDOTyO&=Q^39ToU;qFB literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_transformer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_transformer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..81c7a6dc03a6e9fae879d82182d53a9c282c0b8f GIT binary patch literal 767 zcmVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP3l3q|5Qxc1qjQIxhoK9FzT;#=z#i%#vL0YA5P!>zyofqmhBa{|09j`RrNL3KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP3l3q|5Qxc1qjQIxhoK9FzT;#=z#i%#vL0YA5P!>zyofqmhBa{|09j`RrNL3s&M z4rsMO`(_%xul!z$Q5jVUX7*al@RZpFfDen-l%aQpe4=Kz>qsU~=lAqh$A26b%qz2h N6wtVY29O(HI}W_*iB|vs literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8bf4ba5f0c30b598caaed27eea4a351045093b18 GIT binary patch literal 804 zcmV+<1Ka$DF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!eC?mamQ&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGKb%OxL=(ed>=iAeucxW)s?V`A@8i~ z9d#DeHT)3o@DqT!91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVz7fFD33(sv$C zcpVO^Q2G(-V~S1hb7N;wQH+@4m0fyiO55&~%aVeM6L3g)`A|ziBHJhv*+b?jzh&aQdnW3c_ARW4SN0b)DXIUGF72U+z_59>yFj2bFJ8a4 z5DWkIRZpv>&*4M&A^CE+ykgUlz9^l=r(ZZOp8Zu?ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9KPRbj#+=Fllt-y~cLAVWj^i5raApWr zFm0Gn>U>nDrs_tDLKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!qA?lL(F&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSG=EzFEskbw+C6eu&7g(U$6mCFrE( z6?I3BF!~qo^Amu%91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSt6(iui|g?>jF zAsqvuQvMRKZH`LkeQRd`3yGfPm0o&jO55&~%aVeM6L3g)`A|ziBHJhv*+bv-`n9M^lhXsRrD@IQ){7rzCycK*6CB^G-7glME&E!KvwK` zo&Q9j$zOiac{%x)&h)yuzh&aQdnW3c_ARW4SN0b)DXIUGF72U+z_59>yFj2bFJ8a4 zM_sM(QyKiMmbM!AEb?%?zc}5Mxge9xrw(E(nEF*&ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9KPRbj#+=Fllt-y~cLAVWj^i5raApWr zFm0Gn>U>nDrs_tDLKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!J4?lL(F&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSQ7KumZ^$dK@^^BY~Ot(2%l)biMw+ z7H}5UXZc0#r4xX;91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSw3(-$3g)^Qz6 zDnJVG1^*Q2Z{HuiCum|(Q{J7omk4@kO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zMp>->2pg}X%;XWOC8H<3xnSSK?0U)0qF*|Eo2XY0ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9KPRbj#+=Fllt-y~cLAVWj^i5raApWr zFm0Gn>U>nDrs_tDLKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k9=r<~3;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN!r z4fXS)?^tX?{5T7S_h;L1ocP=zUzb-?H)g@nXZYZ!m12WZ4WVEK$~* z>v~BNUOTqqWJgedkop1@Cj^R+qeH2(VfXe#@Zb#!mSh5f=D1I6X=C|MGRdpZJZAi zUR`Fk3qtc?xXjwNe1NV4FPTHh(pmRZo?rK%J7lqP*|+u#hf(8Fe;u%9mf}?4dwddZ z3TKE^{%{h!r0N}tZ30viVa#im@`9C9{57V@pSn}KSL7)2_%a1X`r!3q{&EdL?!|(= literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layer/test_activation/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layer/test_activation/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c02035522a93edb079e667380065c42abcfd8898 GIT binary patch literal 765 zcmVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k9=r<~3;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNdmTUCB7@EO)t2Fcb?cTZAwM<-R*cQ!k_Tx3W!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)>0E6{|2P7J{CJ@@w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmS%jBc)?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZoU zLk{-!QyBmE$h;e&BBXMdpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e8?m96Ur`7N|%RTB}&HTpH;z&4ZLFcjy z%uB~1A@t$zCUVO@l#p+Vm7dK?j>HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{utDq~0 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0o@R3uUB% zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip2;f>0g*md`URgDTto_+`+eiCAInN z6C)ajF{wrBuT7KCGZm+Ip8m<6HFn|$OKtl|PNr%XhQfo^4WN-o)_0H5(oR4ofpyFj2bFJ8a4 z6I`#N0ZaI!&$JM#aie?f=rNDTxG9~Vs#`uP%&Q1qud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&nda|^L?)#V=NYETDK z8)KYM?tK)z?EO=rQ#4&4d(1)1>zCS6-zn&jsjYFWM*c>xqYq0Ky42xF_KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!Y4={z|K&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwnpr$K; z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{sN0sp19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSG^9y9~`ndq`~GEr6b?)R(r_dB6PV z5OoxQWc5M2_Y;7*91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSt5g&Y%eiEv3D zAx;bTSNjv}JklM%aBgQ)1>KRe!CHE0O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zM_&5%1|RUEn6Mh8B&2n@zGRHa?s%Tar(JA+m#hI@ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&nda|^L?)#V=NYETDK z8)KYM?tK)z?Cn>kMl@R)VfsSZ_L9aDfgr7e)thFhRq14&wi6u`sjkFS`*8^yw0`hU ET4J7r`v3p{ literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layers/test_activation/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_activation/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..1dc05b9b4328db8094a07b02ccf1d94b3124a3b6 GIT binary patch literal 766 zcmVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k9=r<~3;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNdmTUCB7@EO)t2Fcb?cTZAwM<-R*cQ!k_Tx3W!~q*%#r_+EOkPD(Rmt5AfX2q&~n}a8o>!r z4fXS)>0E6{|2P7J{CJ@@w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmS%jBc)?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZoU zLk{-!QyBmE$h;e&BBXMdpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyj`RrNL2d;_ w27HS}^LrKjoW@(bTOVa5JL64+t*Fcn*)^u`o4jk}AEQIQy9g9c)9C#{sq-OyP5=M^ literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layers/test_activation/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_activation/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..189db90766e15b60a588012dbf146eb3af1af925 GIT binary patch literal 774 zcmV+h1Nr=hF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e8?m96Ur`7N|%RTB}&HTpH;z&4ZLFcjy z%uB~1A@t$zCUVO@l#p+Vm7dK?j>HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{us-!55 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0yAV4rQc) zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip2;f>0g*md`URgDTto_+`+eiCAInN z6C)ajF{wrBuT7KCGZm+Ip8m<6HFn|$OKtl|PNr%XhQfo^4WN-o)_0H5(oR4ofpyFj2bFJ8a4 z6I`#N0ZaI!&$JM#aie?f=rNDTxG9~Vs#`uP%&Q1qud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&nda|^L?iM|iKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!Y4={z|K&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSG^9y9~`ndq`~GEr6b?)R(r_dB6PV z5OoxQWc5M2_Y;7*91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSt5g&Y%eiEv3D zAx;bTSNjv}JklM%aBgQ)1>KRe!CHE0O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zM_&5%1|RUEn6Mh8B&2n@zGRHa?s%Tar(JA+m#hI@ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&nda|^L?iM|iKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP3l3q|5Qxc1q*O_Z)hEtwz5=vw+^lS=2aEI8_8&#>Jwg+udxa~_Lso8l15d1?q$ t9c!LcwM!(st?)*QZAD50DZFRh@Qut4|1qG~#PlqWQ{Z{8vmqQt@V{|8fujHb literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..45bde5f6ffdbf6b62de5d9f1fdedee6cfbcd9ea7 GIT binary patch literal 784 zcmV+r1MmEXF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!X{ zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGT7KIy9&xic}F{hBY>EyhnKO1dAs|) z5qA-WW33XnqZ5F+91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS^K&>R(XiE>6r zdO!>DSEv)~Z_*jKC2wU`1KP>p!VY?AO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zPYkd102uI~m*+vMB%*cdzGL6X>VM10rC4fs&;J4mud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&nda|^L?iM|ii->;! literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/run_causal_mask_generate.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/run_causal_mask_generate.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9a3972dd8691ec37d8d826743f7d66ede49867b6 GIT binary patch literal 793 zcmV+!1LpjOF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!t3>peLN&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGKbkIxDJp@C`@66E7F*&hr{8BcI*DQ z5GE6aHuOWhp%Z|)91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVzD*B=osgdtpe=W(LrT>2!k)Z}ICNAe6H4OaD$ z^Fkh7Y;5MhSX)`T^5ZoMx1C(EN6Qqf?yrSVC_e%XI+v#E@*Dr=XohFhrK#!7IB_Oj zO!<$Wx#2z_ZGWFWD1J*oIO`>}d0hwM7D)z01}AZe>nVnX$W}Mecrm|LCelp++kGU5 zX8)rWD(?nUk^~#BvWy5y9Nop$T<49*)}D?ZhQu8-g!);^xD{j-LaB^&L>g8lZGuG= zNC9S`S4H$-zuktmMux6)E{Gq==vw+^k5cfrDk8ITg}2Ndl6~rIYZs<))#V=NYET7G z8hMLd#d#W`sK!;WTnpnA X0nUen7QRdmab=;>lAl*Gr3KS5N;->Q literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/test_causal_mask_generate.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/test_causal_mask_generate.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..2d9585b1d02aed617f64861431b760388cff0dcd GIT binary patch literal 795 zcmV+$1LXXMF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8 zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSl04v0s@ReoAhJEP|P()0g0aA-Su$ zM|B#}WT-*7^%H=(91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS#Ci55U7)OAZ8 zcuxuNSo{;ZXo?%?AvtGVRNc(8!dQA~O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zL=X1=RZFd>nzBQsCZl@3zB1U6=r7Nor&B$D%B%ukud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&nda|^L?iM|iU=052jgnjzs4w4R_(HaFjwb*B literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c8460f8ebf3a870e4b7d20dc0db3bf1d1f97c10a GIT binary patch literal 770 zcmV+d1O5DlF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k9kaU>0R|dKN!r z4fXS)>|S6=`8o-Q^K`Obnepu#{!OYkcj^uW0%orV>u9%m-^5(t8(>Xl3ALY+ZCU79 zl7|51fL3yrJ~Qi^-RAe{{(s|+d@tP1u6VPG1g2DEY4`4eGo0_i@VQ;G>oDddEFG=B zPhR<@2pO#M&9*?NB=RQfx-*Q#xhT%eqX|BKpY|Sp|MAgOM$JHa-Sr=;ZT=!DAX)Lg z@qGgjBowpiRANkn?e0DzxBCX7JDWM9>Ys;sBzsl^KcA=N@et;iE30VGm8-zPd?-r| zcJ_m>wa7ALJaCdbB6>_rBZOv-c2PI(KSe?uYkp9g{4KMV>|cGBpo9D)1dt^bt|ubR zAJ6t}4!wF}>#ynVq>&C?K+%NASK)@q(1ey*#M>S=n!gj+wl6nj7w?aGOc6pSdYw~9 zPfKC3S03~`JwibMZsU;+DgpW+Y8cw+}b z2UUwj?r{~Ss`gy^Z8TF9eCKQDtc%bqsD4`R?#Q(RTzI%t6KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!nF?mjsR&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSJg`?OvEmbVoSec!|oW)W_x1CAg~U z9C1d2HmDZ5@)Ll$91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkSsHhe|;vgnvd$ zAR7XsRr^D~IgT3Xa$;iv29M0-z+ZZ4O55&~%aVeM6L3g)`A|ziBHJhv*+btpe=W(LrT>2o1bc_Fp+GpZiq+yxiVUXu>!k)Z}ICNAe6H4OaD$ z^Fkh7Y;5MhSX)`T^5ZoMx1C(EN6Qqf?yrSVC_e%XI+v#E@*Dr=XohFhrK#!7IB_Oj zO!<$Wx#2z_ZGWFWD1J*oIO`>}d0hwM7D)z01}AZe>nVnX$W}Mecrm|LCelp++kGU5 zX8)rWD(?nUk^~#BvWy5y9Nop$T<49*)}D?ZhQu8-g!);^xD{j-LaB^&L>g8lZGuG= zNC9S`S4H$-zuktmMux6)E{Gq==vw+^k5cfrDk8ITg}2Ndl6~rIYZs<))#V=NYES@O z4tauZ`(O{Zt?gU7Ttr?QVe)I==au#e+c~(*)VO-)4BvCSp#leb^!51~t1&$fo_PIE IBGtQqTKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!Y4 zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS!H7>RHTAbV@kUDuU1d-ITS{dFiOX z6L=cbV*WzyrW1g<91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS+Ch)qUvfqzR& zFG&fe0`d^=Inqn(Cv9q7QQMlZmkWAnO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zLR|NxQy%!On&cFtCiNh&y=RHZxGc%bs#7?5oUd05ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&nda|^L?iM|iKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!h4^G$3??ZEIa%TT{#!LFai`9(8LVDF|K zp%Q^>S*@|9G-<{)v$1EMgY%enf|gL8@+og3-P2FJGqt+|$DATf+9d(pb(R|iuDw~+ zniQ~vSR$SZXad+Ho`TCFWI$=01<&+CMg??!VG()a*mtK7Hp_f#^ZjI1pw*4ARyFQ~ zO{Gn1Eeat^%#5#9Qd-H==*N4+(_HCw*|5-UJ%!UO>;jM(eNH>oD}+3L0T0QSmZhI7@7(n}pK)^<-w zD@YIbSo0CNW6~G9Cp>CV0NTyz-hD4BX@IxKvh>)fBYA9psUs6Ib&Crw+EUOOINvD| zT=?;z^e;UYu3c2cswCcPldigPt$(67QQ84nQ#0Qv@o?-1@yAf;JVaV27y7h>X9uAq z)RI)^h(cueb$-5|-S(lo{A2U(Cup0}tQ-Bq1hINEZujnjEvvVg_vCZ3=QW}>A|9`} zLR$C!0ZQ<#mf%73EcJ1|={?1=uy)Cp-w$qalI3?*|NHP$WXc{0h^a08J>3Z@DsSqZ z_iqzte0}7)b6Zo4>D)6Bw(1G4P0wTZxA4J1Eh+?9SkCCT`4F16E~jqjuejfub|oQI zETz)G^Wi%)Yb5w)YcEDjU$S_Sa7HxdF-SrW-iC(+9`y`M0@2?Uqk`!y7mp$?rS9Dh zk0CYhwPE~{AM)h#xrz-JU5ML`9psJQ)3V-Kn}R`L)29T`xB`4-4&#z(Sy4kiFq=gb zPjP3l3q|5Qxc1q*O_Z)hEtwz5=vw+^lS=2aEI8_8&#>Jwic9Eda{&8XrT7*7YEU^J fA9L0o$#Ds!rt4y~Mgs&!I>toQ?V8Ox#yqismG61m literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layers/test_linear/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_linear/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..f47b70ea32ea2ab9d496716876241c9c0409f87a GIT binary patch literal 772 zcmV+f1N;1jF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!d~?l>|Tr`7N|%RTB}&HTpH;z&4ZLFcjy z%uB~1A@t$zCUVO@l#p+Vm7dK?j>HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{us-!55 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0yAV4rQc) zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip5eR><-QzbRRp?Du>Vg(vh*!a_*$M z6L?34V);hu^%#@TGZm+Ip8m<6HFn|$OKtl|PNr%XhQfo^4WN-o)_0GG*Gmy_(SIID zEJ+Ne0R0u~IN40+B5i5_1KXXnkXUhgBlPQ?oRxw13r|gQ|9D~$DcMpy+)C6U6xl0I zEwrMq=~rb#{XY(Z{CTe|%(BiV{al|kK=}c6LOY;yyf~Cn-_CvKIBHd1UizuPK6vkY zmDz3a+ze#GW?|-l&i=8es57vsXEfJ?-y!SITJ{HZd#S#|ApO3IvHU~2w{D?Jc1fVB zQ~~{KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!qA?J_wE&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSS*9?gGnBEKF>IDbvlbftIjjNAYbG``WzI2d24UhkSvB)JYw6fhSEL zd`eiPQmzu}YL6Z0a&9(P1>4ED!&-W2O55&~%aVeM6L3g)`A|ziBHJhv*+boDddEKIEJ z5C*OHP)PjooaPjvaq=L!=ri2L=zW;WqzXBG&-NaE|MAgOM$JHa-Sr=;ZT=!DAX)Lg z@qGgjBowpiRANkn?e0DzxBCX7JDWM9>Ys;sBzsl^KcA=N@et;iE30VGm8-zPd?-r| zcJ_m>wa7ALJaCdbB6>_rBZOv-c2PI(KSe?uYkp9g{4KMV>|cGBpo9D)1dt^bt|ubR zAJ6t}4!wF}>#ynVq>&C?K+%NASK)@q(1ey*#M>S=n!gj+wl6nj7w?aGOc6pSdYw~9 zPfKC3S03~`JwibMZsU;+DgpW+Y8a%Kor t4tT{w?|LM@s`pp4Kt)OeDZFRh@Qut4|1qG~#PlqWQ{Z{8vmqQt@V{p1g8={l literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layers/test_linear/test_linear.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_linear/test_linear.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..6525b8c12de0f1dadd04f1967ba0f6e462e3339b GIT binary patch literal 767 zcmVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!J1>^(UO&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGTDuJx>?IkcpPfbBY~f<)R4E+cI^D> z7IG1XX7oe9r4xX;91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS|BhaEyFj2bFJ8a4 zPFeW$QyKW7%(fM(C8cn=xn+sQy?C6?pjc~tmaYX4ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&nda|^L?iM|iKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!n1kaU>0R|dKNkF6~eM)VFc+}6Rhrzapa=ZBG z9C8|hV*W(EssQPh85y5_zVOkqJr1}IGdR&i5cY0q!r z4fXS)=v!k>{y7VY|9PP{w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvpc$K<2$?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ@n z5)Q5Q0U!CGn&v_FAoU>a={eZOxh;~;s$Ok<&HN1$(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyj`RrNL176) q0wuL>^Li4doW>lbOIH;saN$DT@TKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!G5?l&r9;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNtD@BdPZ-AD2Jb|fRf^fbnB?- zL3kU|VD%KTssQPh85y5_zVOkqJr1}IGdR&i5cY0q!r z4fXS)>0V+>{XGqb`Ea2&w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvIK&E%u+?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZrd zL<_B=R6_o&nC20saPTL-y*b{N=`GL1s}65=o%0P7(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyj`RrNL176) w0ws=6?sX)!ruS{2Mgmp|JL64+t*Fcn*)^u`o4jk}AEQIQy9g9c)9C#{sZ!m400000 literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_layers/test_norm/test_fused_norm.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_norm/test_fused_norm.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8a44c6e87c5249eb76335785562831ad1d52a9db GIT binary patch literal 768 zcmV+b1ONPnF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!tF zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS-eP>H?V@DI98oErFk^hmf$*cf0uL z79~c2V*Enyr4xX;91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS<7(oG#F(;^)i zd`=4XSEv-~JKPz&aA-FH1>2agl?ZxiO55&~%aVeM6L3g)`A|ziBHJhv*+btpe=W(LrT>2!k)Z}ICNAe6H4OaD$ z^Fkh7Y;5MhSX)`T^5ZoMx1C(EN6Qqf?yrSVC_e%XI+v#E@*Dr=XohFhrK#!7IB_Oj zO!<$Wx#2z_ZGWFWD1J*oIO`>}d0hwM7D)z01}AZe>nVnX$W}Mecrm|LCelp++kGU5 zX8)rWD(?nUk^~#BvWy5y9Nop$T<49*)}D?ZhQu8-g!);^xD{j-LaB^&L>g8lZGuG= zNC9S`S4H$-zuktmMux6)E{Gq==vw+^k5cfrDk8ITg}2Ndl6~rIYZs<))#V=NYETGX y9cRTw|7Q=4rR+hlRy0%KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!n1>o+Q6;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN!r z4fXS)=vZb*{y74P_;8^%w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvpc$mFB%?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZoi zP7U{^S3~{qm*f)hET?kqxo6Uq?thz@_FHKwp7RY8(em9~VB`^T-{y9OX!ivNBUSmC z)Mo@*CT^VXT~cYE?%+EQny?q3MYTGw;-|f8J0MvCJCmyC@EY~?1*l}$t&!8-Cn_W| zbNq#wkJ~LjHg%ddC3#3hF!))Ed{+U~ELs^J1V~4o?>dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!YA?>;gar`7N|%RTB}&HTpH;z&4ZLFcjy z%uB~1A@t$zCUVO@l#p+Vm7dK?j>HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{us-!55 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0yAV4rQc) zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip5NI=>p13C`LTdDu>Pa(UszYaOte% z5qA>RW2i*9^c<7WGZm+Ip8m<6HFn|$OKtl|PNr%XhQfo^4WN-o)_0GMg-#uGgCj>z zFG&of1pX1YXxmHgd~an{P}0l(kujw7JQy*I;a`sy1_m993+KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!J5=`%SB&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS%HIu?@~0c^hxmeukK<-NChic)P6b z5+O&0W3CYC_Y;7*91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS*C(iReM(sCLc zC>;&)R`NyfYttF;c5E{M0NTpplUjOdO55&~%aVeM6L3g)`A|ziBHJhv*+bu9%m-^5(t8(>Xl3ALY+ZCU79 zl7|51fL3yrK0NG~$K<2$?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ%m z6$bdAQ%$?k~%kpjdpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!n9>OVOO&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS`WD?OK=`d`dsiBG$tAhmz)ja=ELx zKy*gZXY?Psq!WO-91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS?Dhe|~z)ptuw zEtpe=W(LrT>2V3}0_6}%wmiq$$yxiVUXu>!k)Z}ICNAe6H4OaD$ z^Fkh7Y;5MhSX)`T^5ZoMx1C(EN6Qqf?yrSVC_e%XI+v#E@*Dr=XohFhrK#!7IB_Oj zO!<$Wx#2z_ZGWFWD1J*oIO`>}d0hwM7D)z01}AZe>nVnX$W}Mecrm|LCelp++kGU5 zX8)rWD(?nUk^~#BvWy5y9Nop$T<49*)}D?ZhQu8-g!);^xD{j-LaB^&L>g8lZGuG= zNC9S`S4H$-zuktmMux6)E{Gq==vw+^k5cfrDk8ITg}2Ndl6~@ja~!8-mE#uiYKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!n1kaU>0R|dKNJ7_HeMvutEYg_y(vr7^bhP>F zLUcsdVf+=cssQPh85y5_zVOkqJr1}IGdR&i5cY0q!r z4fXS)>Rn(>{V)xJ_dpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyI{KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8={G84;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKN<*nsd`558FNn&k)RN!r z4fXS);#+M``aceY`*gBjnepu#{!OYkcj^uW0%orV>u9%m-^5(t8(>Xl3ALY+ZCU79 zl7|51fL3yrJ~Qf<-RAe{{(s|+d@tP1u6VPG1g2DEY4`4eGo0_i@VQ;G>oDddEE=!v zL|gmw221gxoUt0CBk^>(yl05V>M75j^I2Ys;sBzsl^KcA=N@et;iE30VGm8-zPd?-r| zcJ_m>wa7ALJaCdbB6>_rBZOv-c2PI(KSe?uYkp9g{4KMV>|cGBpo9D)1dt^bt|ubR zAJ6t}4!wF}>#ynVq>&C?K+%NASK)@q(1ey*#M>S=n!gj+wl6nj7w?aGOc6pSdYw~9 zPfKC3S03~`Jwl2GSkVHCS@m-GqeF?$0- z8&#e|`+5kptm|K}Kr~nudhlPD-K4-0#w3g2)VO-)4BvCSp#leb^!51~t1&$fo_PIE IBGtQqTu>2+6aWAK literal 0 HcmV?d00001 diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..68c295097ef03a724b6b1409be485bd5d6e2615e GIT binary patch literal 787 zcmV+u1MK{UF-iDYjVh`4HCJ0Df~^-ztXiAJX}YXmS<2Q7pVKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!b9=`uMA&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGSky|=~~TAc}+agErFh`g~PChce?rO zM|l_4WU3Ro@DqT!91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS#KhE7Fuf^tkx zFHQ>gP^w4jY}^>Sb2)2JQ{B$tz+HN2O55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zLSEl->wTS{t6n;P$o>FYud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&n#U=zA|Kn7n3O7U3N;gE(H&Lp+qovm$!R``FSq)7!+sjk97KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!J6={q?J&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwmq@^x^ z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{so#~y19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGTDeO><`IFcuPCfD}b4=)ReM;cjNAYbG``WzI2d24UhkS|G&`S_^hI2|t zDo6|PQu`3-Ytu{VcWPl&Q;Es5!d`l5O55&~%aVeM6L3g)`A|ziBHJhv*+b-`n9M^lhXsRrD@IQ){7rzCycK*6CB^G-7glME&E!KvwK` zo&Q9j$zOiac{%x-$@8yFj2bFJ8a4 zPF$~|1sngNo97XtB%^S-y^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&n#U=zA|Kn7n3O7U3N;gE(7+CGcRh^cLKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e6={_+Tr`7N|%RTB}&HTpH;z&4ZLFcjy z%uB~1A@t$zCUVO@l#p+Vm7dK?j>HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{us-!55 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0yAV4rQc) zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip5uU?gO7lbQ)lWeud5b(w4S?BkQW} z9CQ?eHT*`n@=lY`GZm+Ip8m<6HFn|$OKtl|PNr%XhQfo^4WN-o)_0GBf=L`9hkqLr zcpVO;0R0l|Ylundc4B5z28@`r!e1wPBlPQ?oRxw13r|gQ|9D~$DcMpy+)C6U6xl0I zEwrMq=~rz_{x}Ya{&}b+-`n9M^lhXsRrD@IQ){7rzCycK*6CB^G-7glME&E!KvwK` zo&Q9j$zOiac{%x$&hxgfy)opkbUVtUHwb3#OVLQ$-C3`_cKj%K%I%f#(CPgnvLVg_3$~uvO{Y4YlGInR=Wv6dKvujb^ zV71yNf0QBt?locsK+GOj8p@g39r}jO-JggBo`PR1hxkX2@&{@!5!u-?M;Bf$d4N(1 zUR+`7P*n6gzlPej41vF2FQY}tqFw7ri*)XKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!k8?Kdi8;s5*w#}wupl!TN1?r<+(UGkaU>0R|dKNI zMIjM_HS`hbssQPh85y5_zVOkqJr1}IGdR&i5cY0q!r z4fXS)>QQ4t{4oQ9{C=S}w)EsG#a6#MSlb^}18S{(#BIJ_@yAf;JVjSwL$~3)ZdKra zyVe!8nni8SAtvmT&g7%-?Qs6CMm)onv1*}+N4G^ZX!!AhG4IgC`oDFXx;XhHFhZ`q z5C^RG22A|*o#qtwB=vH->0*qOxhcz?pdpE`9mv*Y52r$YrSAah+A0w zR<_zYeViu_{t>0(zLpP53YUY@M)sQ2%Y}~y+1Vd8hvxyKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!bC?>IRM&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwnpr$K; z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{sN0sp19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGS}KJ>Ilk4drNEABY?^IhQ{KECAt2% zMRgR0W%?iOr4xX;91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkS>H&`lR~iEv67 zDL@OL1gS!}VvZZVAUkJM1KghNmk4@kO55&~%aVeM6L3g)`A|ziBHJhv*+byFj2bFJ8a4 zKn|~@0vPb3o3|3GB=aWg?=aGp=z7kbr(J4(nDSQ-ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&n#U=zA|Kn7n3O8-^C=ZS_G&Lp+qovm$!R``FSq)7!+sjk97KYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!J1={-3L&ieH*{RjC#mj1l^=RpZfQs}QM z_E^|@FZl9?33tK|vV&lkwWQuv)y8wX|36m_!NY5{E{L=j_RD@B(k%trbdwwnpr$K; z**BGmP8zZncqY+4qmADaGZ9M5RqWneMkjJ|Zwo}^+eYwHWW#iC_55K{sN0sp19r`> zQ>AZhAp|>i%8#A`19_f<%ldfL=3mfofWy#iGJ}eFy9k&bE*ojodV!s-fyc6map?H( zL3kC`WBd~B^b>%&91yNXrRCg{6koFpNf*>jNAYbG``WzI2d24UhkVo3ha49s(Ipxh zDj5p&P_9GhW7@-YgA`h2P--`n9M^lhXsRrD@IQ){7rzCycK*6CB^G-7glME&E!KvwK` zo&Q9j$zOiac{%x-%=5Lmzh&aQdnW3c_ARW4SN0b)DXIUGF72U+z_59>yFj2bFJ8a4 z6khoC0!pa&%e4{nC8u)i?qJiwxqO+Ks$4pFp8r=~ud(n`WXc{1h3{woF5L+!DtPvt z`(-;vYbNBncV1A6*Z(#IvaoUbR>^9D?W2uo5-c57HI}00{~7-5d+SNpsiD#LD{g&X zBgTrAzt;paawL^IAahJZJmWUVMpzBxAafO3T_qmB{4JG|>SrR&E+BR-9-ksDo-P9W z9M9@+2Do){&u9)1{(urq9m|iCTm78h%7c|2pPK+Sh51C1sugM>CBDvZLl9IfaGgy8 zNnJg$K|=F9Kt&+Od`l~&n#U=zA|Kn7n3O8-^C=ZS_6+CGcRh^cLKYR&@NG(Jk2Kp zqH`H(NyVN`@B)C!e7HL+`Zj(e)5KDjC55{P+xB`Fi6R5rdzl{utDq~0 z%3ivRM>C%Ue;@NTu7bcm4i!R>SC;WvPXR+^YaUtR+I-DdY?ym#^z&;`q0o@R3uUB% zR;)=}HUKVb%#Nf#2XV!LvDJOt*JR6OjpF$|Ip5kT>k610D@tvKe~H4R-N3McbMC9} zL2wt;X8%Iyu1u5AGZm+Ip8m<6HFn|$OKtl|PNr%XhQfo^4WN-o)_0GE*Gv^A(;!P8 zcpL(&S@J~cYuHTacRylKQQw=ek^?1rBlPQ?oRxw13r|gQ|9D~$DcMpy+)C6U6xl0I zEwrMq=~re>{yYqU`f#o)%(BiV{al|kK=}c6LOY;yyf~Cn-_CvKIBHd1UizuPK6vkY zmDz3a+ze#GW?|-l&i1>auQRZzXEfJ?-y!SITJ{HZd#S#|ApO3IvHU~2w{D?Jc1fVB zR002>QcL;w$gvjlaQJ$$wPcCG?|Pii_gii(ocUB=tNo4;NyQo`ikCh1Kc`OvCRYEj z^kEc5R6D8i4-0yR?&=R4zq?ubFq3)yy^exxA}%yqal)+Q_Zac-aPl{lgs#=cDMxV| zdaK&6u-i00TPV?LFMdf$IOIChDH}KIEJGP{QDH~Yr3{9J?ou)7Cu%YEtlU=*+(s0R zM#-QSHth~pofJQ5RdP~W0L_@sSG0l(LvG`o3R0K#DaN%pzD=(yK!oqX^PT^bl`XU(t-I literal 0 HcmV?d00001 -- Gitee From a5f83a30f79b9527a7d008e644ddf8768e28c79a Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Wed, 28 Jan 2026 16:18:00 +0800 Subject: [PATCH 10/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=B7=AF=E7=94=B1?= =?UTF-8?q?=E5=90=8D=E7=A7=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ds_pynative.yaml | 2 +- markdown.md | 176 ----------- .../parallel_core/transformer_config.py | 2 +- .../parallel_core/transformer_config_utils.py | 2 +- .../pynative/transformers/moe/moe_layer.py | 10 +- .../pynative/transformers/moe/router.py | 2 +- .../transformers/moe/test_moe_layer.py | 99 ------ .../pynative/transformers/moe/test_router.py | 289 ------------------ 8 files changed, 9 insertions(+), 573 deletions(-) delete mode 100644 markdown.md delete mode 100644 mindformers/pynative/transformers/moe/test_moe_layer.py delete mode 100644 mindformers/pynative/transformers/moe/test_router.py diff --git a/ds_pynative.yaml b/ds_pynative.yaml index 1a9b81642..b7081e834 100644 --- a/ds_pynative.yaml +++ b/ds_pynative.yaml @@ -235,7 +235,7 @@ model: num_experts_per_tok: 8 n_shared_experts: 1 num_copy_experts: 8 - use_longcat_router: True + use_topk_router_with_load_balancing: True moe_expected_ffn_experts: 2.0 # Default best value: top-k * FFN/(FFN + COPY) moe_router_bias_update_rate: 0.001 moe_shared_expert_intermediate_size: 2048 diff --git a/markdown.md b/markdown.md deleted file mode 100644 index f8119370e..000000000 --- a/markdown.md +++ /dev/null @@ -1,176 +0,0 @@ -基于 LongCat-Flash 技术报告，**计算预算控制 (Computational Budget Control)** 和 **负载均衡控制 (Load Balance Control)** 是为了确保“零计算专家”机制能够稳定、高效运行而设计的两项关键技术。 - -引入零计算专家后，模型需要解决两个核心问题： - -1. **宏观上**：如何确保模型总体的计算量（平均激活参数量）符合预期，既不超支也不偷懒？（计算预算控制） -2. **微观上**：如何防止某些计算型专家过载，同时适配零计算专家的特殊性？（负载均衡控制） - -以下是这两项控制机制的详细说明： - -### 1. 计算预算控制 (Computational Budget Control) - -这套机制的目标是**精细控制零计算专家的平均选择比例**，防止模型在训练过程中倾向于“偷懒”（过度使用零计算专家导致欠拟合）或“超支”（过度使用计算专家导致效率下降。 - - - -- 机制核心：与训练目标（Loss）解耦的**专家偏好（Bias）**项 $b_i$ 来动态调整路由分数。这就好比给每个专家一个“基础分”，根据它最近的忙碌程度动态调整，让它更容易或更难被选中。 - -- 更新公式 - - 专家偏好 $b_i$ 的更新遵循以下规则（类似于 PID 控制器）： - - $$\Delta b_{i} = \mu \left( \frac{K_{e}}{K} \cdot \frac{1}{N} - \frac{T_{i}}{K T_{all}} \right)$$ - - - **适用对象**：仅针对 $N$ 个**标准 FFN 专家**进行更新。 - - **变量含义**： - - $\mu$：调节速率。 - - $K_e$：**期望**激活的 FFN 专家数量（预算目标）。 - - $K$：总共选择的专家数量（Top-K）。 - - $T_i$：分配给专家 $i$ 的 Token 数量。 - - $T_{all}$：总 Token 数量。 - -- 效果 - - 该机制确保了模型在训练过程中，平均激活的 FFN 专家数量能够收敛到预设值（例如 $K_e$ 对应的 27B 参数量），波动幅度小于 1% 。同时，它允许标准差保持在较高水平，这意味着虽然整体预算受控，但模型仍保留了为不同难度的 Token 动态分配不同算力的能力 - ------- - -### 2. 负载均衡控制 (Load Balance Control) - -这套机制的目标是**防止设备间的计算负载不均**。虽然预算控制解决了全局的计算量问题，但如果某个 GPU 上的专家特别热门，会导致该设备成为瓶颈（木桶效应）。 - -- 机制核心：设备级负载均衡 Loss ($\mathcal{L}_{LB}$) - - 为了防止极端的不平衡，模型引入了一个辅助 Loss。LongCat-Flash 对传统的负载均衡 Loss 进行了修改，以适应零计算专家的存在 。 - -- 分组策略 - - 假设 $N$ 个标准 FFN 专家被分布在 $D$ 个设备（或组）上，每个组有 $G = N/D$ 个专家： - - - **标准专家**：归入前 $1$ 到 $D$ 组。 - - **零计算专家**：被强制归入第 **$D+1$ 组**（额外的一个虚拟组）。 - -- 计算公式 - - $$\mathcal{L}_{LB} = \alpha \sum_{j=1}^{D+1} f_{j} P_{j}$$ - - 其中： - - - $f_j$ 是该组专家被选中的频率（对于 $D+1$ 组，即零计算专家的选择频率）。 - - $P_j$ 是该组专家的平均路由概率。 - -- 效果 - - 通过这种设计，Loss 函数不仅平衡了物理设备（前 $D$ 组）之间的负载，还隐式地维护了计算型专家与零计算专家之间的比例平衡（趋近于 $\frac{K_{e}}{K-K_{e}}$）。这确保了在训练大规模模型时，不会因为专家的冷热不均导致计算效率下降。 - -### 总结 - -- **计算预算控制**是“调节阀”，利用 PID 算法动态调整基础分，确保模型整体“不偷懒也不超支”，维持平均 27B 的参数激活量。 -- **负载均衡控制**是“调度员”，通过辅助 Loss 确保任务在不同 GPU 间均匀分配，并将零计算专家作为独立的一类进行管理，防止局部拥堵。 - - - -### 1. 计算预算控制 (Computational Budget Control) - -这一机制的目标是确保模型在训练过程中，**平均**激活的 FFN 专家数量能够收敛到一个预设的期望值（$K_e$），从而控制整体的计算开销（即激活参数量）。 - -- **核心手段：基于 PID 控制器的自适应专家偏置** 龙猫引入了一个专家特定的偏置项 $b_i$，该偏置会根据专家最近的利用率动态更新。这个更新规则借鉴了控制理论中的 **PID 控制器**（Proportional-Integral-Derivative Controller）思想 。 - - - - - -- **偏置更新公式** 对于第 $i$ 个专家，其偏置 $b_i$ 在每一步的增量 $\Delta b_i$ 计算如下 ： - - - - - - $$\Delta b_i = \begin{cases} \mu \left( \frac{K_e}{K} \cdot \frac{1}{N} - \frac{T_i}{K T_{all}} \right), & \text{if } 1 \le i \le N \text{ (FFN Experts)} \\ 0, & \text{if } N < i \le N+Z \text{ (Zero-Computation Experts)} \end{cases}$$ - - - **$\mu$**：偏置适应率（Bias adaptation rate）。 - - **$K_e$**：期望激活的 FFN 专家数量（例如平均 8 个）。 - - **$K$**：每 Token 总共选择的专家数（例如 12 个）。 - - **$T_i$**：路由到第 $i$ 个专家的 Token 数量。 - - **$T_{all}$**：全局 Batch 中的 Token 总数。 - -- **机制亮点：零计算专家的“豁免权”** 该机制的一个关键设计是**不更新零计算专家的偏置** 。 - - - - - - - **逻辑：** 零计算专家本质上是恒等映射（Identity Mapping），不需要像 FFN 专家那样竞争特定的语义特征。 - - **效果：** 通过只调节 $N$ 个 FFN 专家的偏置，强制它们竞争有限的“计算名额”。当所有 FFN 专家都达到其目标利用率时，剩下的概率空间自然会被零计算专家填充，从而自动满足全局约束。 - -- **收敛效果** 实验表明，在大约 20B token 的训练后，各层的平均专家激活数收敛到了期望值，波动小于 1% 。 - - - - - -### 2. 负载均衡控制 (Load Balance Control) - -这一机制的目标是防止计算负载在不同设备（GPU）之间分配不均，同时妥善处理零计算专家带来的特殊分组需求。 - -- **核心手段：分组负载均衡 Loss ($\mathcal{L}_{LB}$)** 为了防止某些专家组（Expert Group）过载，论文引入了设备级的负载均衡损失，并专门为零计算专家分配了一个独立的组 。 - - - - - -- **分组策略** 假设共有 $N$ 个 FFN 专家，被均匀分配到 $D$ 个组中（每组 $G=N/D$ 个专家）。龙猫将所有 **$Z$ 个零计算专家单独归入第 $D+1$ 组** 。 - - - - - -- **Loss 计算公式** 负载均衡损失定义为 ： - - - - - - $$\mathcal{L}_{LB} = \alpha \sum_{j=1}^{D+1} f_j P_j$$ - - 其中： - - - **$f_j$**（第 $j$ 组的选择频率）：表示一个 Batch 中有多少比例的 Token 选择了该组。 - - - 对于 FFN 组 ($1 \le j \le D$)，$f_j$ 归一化时分母包含 $K_e$（期望 FFN 专家数） 。 - - - - - - - 对于零计算专家组 ($j=D+1$)，$f_j$ 归一化时分母包含 $K - K_e$（期望零计算专家数） 。 - - - - - - - - - **$P_j$**（第 $j$ 组的路由概率）：该组内所有专家路由分数的平均和 。 - - - - - - - **$\alpha$**：平衡系数。 - -- **调节目标** 通过这种设计，当损失函数收敛时，模型会倾向于将 FFN 专家与零计算专家的选择比例维持在 $\frac{K_e}{K - K_e}$ 附近 。这意味着模型既实现了设备间的负载均衡，又保证了零计算专家被“按需”选中，而不是被边缘化或过度使用。 - - - - - -\[ \begin{aligned} \text{MoE}(x_t) &= \sum_{i=1}^{N+Z} g_i \, E_i(x_t), \\ g_i &= \begin{cases} R(x_t)_i, & \text{if } R(x_t)_i \in \text{TopK}\bigl(R(x_t)_i + b_i \mid 1 \leq i \leq N+Z, K\bigr), \\ 0, & \text{otherwise}, \end{cases} \\ E_i(x_t) &= \begin{cases} \text{FFN}_i(x_t), & \text{if } 1 \leq i \leq N, \\ x_t, & \text{if } N < i \leq N+Z, \end{cases} \end{aligned} \tag{1} - - -$$ -\[ \begin{aligned} \text{MoE}(x_t) &= \sum_{i=1}^{N+Z} g_i \, E_i(x_t), \\ g_i &= \begin{cases} R(x_t)_i, & \text{if } R(x_t)_i \in \text{TopK}\bigl(R(x_t)_i + b_i \mid 1 \leq i \leq N+Z, K\bigr), \\ 0, & \text{otherwise}, \end{cases} -\\ E_i(x_t) &= \begin{cases} \text{FFN}_i(x_t), & \text{if } 1 \leq i \leq N, \\ x_t, & \text{if } N < i \leq N+Z, \end{cases} \end{aligned} \tag{1} \] -$$ -\noindent where \(R\) denotes the softmax router, \(b_i\) is the expert bias corresponding to the \(i\)-th expert, and \(K\) denotes the number of experts selected per token. \ No newline at end of file diff --git a/mindformers/parallel_core/transformer_config.py b/mindformers/parallel_core/transformer_config.py index 3030952dc..dabf8f5ff 100644 --- a/mindformers/parallel_core/transformer_config.py +++ b/mindformers/parallel_core/transformer_config.py @@ -356,7 +356,7 @@ class TransformerConfig(ModelParallelConfig, MFModelConfig): num_copy_experts: int = 0 - use_longcat_router: bool = True + use_topk_router_with_load_balancing: bool = True moe_expected_ffn_experts: float = 2.0 # Default best value: top-k * FFN/(FFN + COPY) diff --git a/mindformers/parallel_core/transformer_config_utils.py b/mindformers/parallel_core/transformer_config_utils.py index 7d48113f2..2d0889e4c 100644 --- a/mindformers/parallel_core/transformer_config_utils.py +++ b/mindformers/parallel_core/transformer_config_utils.py @@ -383,7 +383,7 @@ COMMON_CONFIG_MAPPING = { "expert_relocation_initial_iteration": "expert_relocation_initial_iteration", "expert_relocation_freq": "expert_relocation_freq", "num_copy_experts": "num_copy_experts", - "use_longcat_router": "use_longcat_router", + "use_topk_router_with_load_balancing": "use_topk_router_with_load_balancing", "moe_expected_ffn_experts": "moe_expected_ffn_experts", "moe_router_bias_update_rate": "moe_router_bias_update_rate", diff --git a/mindformers/pynative/transformers/moe/moe_layer.py b/mindformers/pynative/transformers/moe/moe_layer.py index ed39080c5..1429f517e 100644 --- a/mindformers/pynative/transformers/moe/moe_layer.py +++ b/mindformers/pynative/transformers/moe/moe_layer.py @@ -20,7 +20,7 @@ from mindspore.common.parameter import Parameter from mindformers.parallel_core.transformer_config import TransformerConfig from mindformers.pynative.layers.linear import Linear from mindformers.pynative.transformers.mlp import MLPSubmodules -from .router import LongCatRouter, TopKRouter +from .router import TopKRouterWithLoadBalancing, TopKRouter from .experts import GroupedMLP from .shared_experts import SharedExpertMLP @@ -37,10 +37,10 @@ class MoELayer(nn.Cell): self.top_k = config.moe_router_topk # Router - # Prefer LongCatRouter when copy experts or aux loss is enabled - use_longcat_router = self.config.use_longcat_router - if use_longcat_router: - self.router = LongCatRouter(config) + # Prefer TopKRouterWithLoadBalancing when copy experts or aux loss is enabled + use_topk_router_with_load_balancing = self.config.use_topk_router_with_load_balancing + if use_topk_router_with_load_balancing: + self.router = TopKRouterWithLoadBalancing(config) else: self.router = TopKRouter(config) diff --git a/mindformers/pynative/transformers/moe/router.py b/mindformers/pynative/transformers/moe/router.py index 0044afc97..0d99e31b3 100644 --- a/mindformers/pynative/transformers/moe/router.py +++ b/mindformers/pynative/transformers/moe/router.py @@ -222,7 +222,7 @@ class TopKRouter(nn.Cell): return top_scores, selected_experts_indices, num_tokens_per_expert -class LongCatRouter(nn.Cell): +class TopKRouterWithLoadBalancing(nn.Cell): """ LongCat-Flash routing mechanism with Computational Budget Control and Load Balance Control. diff --git a/mindformers/pynative/transformers/moe/test_moe_layer.py b/mindformers/pynative/transformers/moe/test_moe_layer.py deleted file mode 100644 index 24c7d6062..000000000 --- a/mindformers/pynative/transformers/moe/test_moe_layer.py +++ /dev/null @@ -1,99 +0,0 @@ -# Copyright 2026 Huawei Technologies Co., Ltd -# ... (License header) ... -"""MoE Layer implementation.""" -from mindspore import nn, Tensor, mint, ops -import mindspore as ms -from mindspore.common.parameter import Parameter - -from mindformers.parallel_core.transformer_config import TransformerConfig -from mindformers.pynative.layers.linear import Linear -from mindformers.pynative.transformers.mlp import MLPSubmodules - -# 【注意】这里必须导入新的 LongCatRouter，因为它返回 4 个值 -from .router import LongCatRouter -from .experts import GroupedMLP -from .shared_experts import SharedExpertMLP - - -class MoELayer(nn.Cell): - """ - MoE Layer that combines LongCatRouter, Grouped Experts, and Shared Experts. - """ - - def __init__(self, config: TransformerConfig): - super().__init__() - self.config = config - self.num_experts = config.num_moe_experts - self.top_k = config.moe_router_topk - - # 【改动】使用新的 LongCatRouter - # 它内部管理了 Computational Budget Control (PID) 和 Load Balance Loss - self.router = LongCatRouter(config) - - # Experts (保持不变，支持 copy experts) - self.experts = GroupedMLP(config) - - # Shared Experts (保持不变) - self.shared_experts = None - if config.shared_expert_num > 0: - submodules = MLPSubmodules( - linear_fc1=Linear, - linear_fc2=Linear - ) - self.shared_experts = SharedExpertMLP(config, submodules) - - # Buffers for logging only (Optional) - # 实际的 bias 和 tokens count 现在主要由 Router 内部维护和使用 - self.tokens_per_expert = Parameter( - mint.zeros(self.num_experts, dtype=ms.float32), - name="tokens_per_expert", - requires_grad=False - ) - - # Mint operators - self.reshape = mint.reshape - self.add = mint.add - - def construct(self, hidden_states: Tensor): - """ - Forward pass for MoELayer. - Args: - hidden_states (Tensor): Input tensor of shape (bs, slen, dim) - """ - bs, slen, dim = hidden_states.shape - x_flat = self.reshape(hidden_states, (-1, dim)) - - # 【定义 aux_loss】 - # 调用 self.router (LongCatRouter)，它返回 4 个值： - # 1. top_scores: 路由权重 - # 2. selected_experts_indices: 选中的专家索引 - # 3. num_tokens_per_expert: 每个专家的 token 数量统计（用于 logging 或其他用途） - # 4. aux_loss: 计算好的负载均衡 Loss (Scalar) - top_scores, selected_experts_indices, num_tokens_per_expert, aux_loss = self.router( - x_flat, - training=self.training - ) - - # 统计 Expert 使用情况 (可选，仅用于兼容旧逻辑或日志) - if self.tokens_per_expert is not None: - self.tokens_per_expert.add_(num_tokens_per_expert) - - # 执行专家计算 - routed_output = self.experts(hidden_states, top_scores, selected_experts_indices) - - # 执行共享专家计算 - shared_output = None - if self.shared_experts is not None: - shared_output, _ = self.shared_experts(hidden_states) - - out_experts = self.reshape(routed_output, (bs, slen, dim)) - - # 结果融合 - if shared_output is not None: - final_out = self.add(shared_output, out_experts) - else: - final_out = out_experts - - # 返回最终输出和 aux_loss - # 这里的 aux_loss 就是上面从 router 返回的那个变量 - return final_out, aux_loss \ No newline at end of file diff --git a/mindformers/pynative/transformers/moe/test_router.py b/mindformers/pynative/transformers/moe/test_router.py deleted file mode 100644 index 4bfc04a63..000000000 --- a/mindformers/pynative/transformers/moe/test_router.py +++ /dev/null @@ -1,289 +0,0 @@ -# Copyright 2026 Huawei Technologies Co., Ltd -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================ -"""LongCat Router Mechanism for MoE""" -from typing import Tuple, Optional - -from mindspore import nn, Tensor, mint, ops -from mindspore.common.parameter import Parameter -from mindspore.common import dtype as mstype - -from mindformers.parallel_core.transformer_config import TransformerConfig -from mindformers.parallel_core.utils.init_method import init_method_normal - - -class LongCatRouter(nn.Cell): - """ - Implements the LongCat-Flash routing mechanism with Computational Budget Control - and Load Balance Control. - - Mechanism 1: Computational Budget Control (PID-controlled Bias) - - Adjusts expert bias dynamically to maintain expected activation rates. - - Updates are EXCLUDED for Zero-Computation (Copy) experts. - - Mechanism 2: Load Balance Control - - Computes auxiliary loss based on groups. - - FFN experts are divided into D groups. - - Zero-Computation experts form the (D+1)-th group. - - Args: - config (TransformerConfig): Configuration object. - """ - - def __init__(self, config: TransformerConfig): - super().__init__() - self.config = config - - # Dimensions - self.hidden_size = config.hidden_size - self.num_total_experts = config.num_moe_experts - self.top_k = config.moe_router_topk - - # LongCat specific configurations - # N: FFN experts, Z: Zero-Computation experts - self.num_copy_experts = config.num_copy_experts - self.num_ffn_experts = self.num_total_experts - self.num_copy_experts - - if self.num_ffn_experts <= 0: - raise ValueError(f"num_ffn_experts must be positive, got {self.num_ffn_experts}") - - # Ke: Expected number of activated FFN experts - # Usually defined in config, fallback to top_k * ratio if not present - self.expected_ffn_k = getattr(config, 'moe_expected_ffn_experts', float(self.top_k * self.num_ffn_experts / self.num_total_experts)) - - # PID Controller parameters - self.bias_update_rate = getattr(config, 'moe_router_bias_update_rate', 0.001) - - # Load Balancing parameters - # D: Number of FFN groups - self.num_ffn_groups = config.moe_router_num_groups if config.moe_router_num_groups else 1 - self.aux_loss_coeff = config.moe_aux_loss_coeff - - # Routing parameters - self.score_func = config.moe_router_score_function - self.route_norm = config.norm_topk_prob - self.route_scale = ( - config.moe_router_topk_scaling_factor - if config.moe_router_topk_scaling_factor is not None - else 1.0 - ) - - # Learnable Router Weight - self.weight = Parameter( - init_method_normal(0.02)((self.num_total_experts, self.hidden_size)), - name='weight' - ) - - # Expert Bias (Controlled by PID, detached from Gradient Descent) - self.expert_bias = Parameter( - mint.zeros(self.num_total_experts, dtype=mstype.float32), - name="expert_bias", - requires_grad=False - ) - - # Ops - self.linear = mint.nn.functional.linear - self.sigmoid = mint.nn.functional.sigmoid - self.softmax = mint.nn.functional.softmax - self.cast = ops.cast - self.topk = mint.topk - self.gather = mint.gather - self.mul = mint.mul - self.div = mint.div - self.sum = mint.sum - self.histc = mint.histc - self.ones_like = mint.ones_like - self.reshape = mint.reshape - self.mean = mint.mean - self.cat = mint.cat - - def update_expert_bias(self, num_tokens_per_expert: Tensor, total_tokens: int): - """ - PID Controller for Computational Budget Control. - Formula: delta_b_i = mu * (Target_Rate - Current_Rate) - Only updates FFN experts (indices 0 to N-1). Zero-Computation experts are skipped. - """ - if self.training and self.bias_update_rate > 0: - # Current selection rate: Ti / (K * T_all) - # Note: We use K * T_all as denominator to normalize to [0, 1] per expert slot - current_rate = num_tokens_per_expert / (self.top_k * total_tokens) - - # Target selection rate for FFN experts: Ke / (K * N) - target_rate_ffn = self.expected_ffn_k / (self.top_k * self.num_ffn_experts) - - # Calculate error: Target - Current - # We construct a target tensor that only has values for FFN experts - # For Copy experts, we set error to 0 so bias doesn't change - target_rates = self.ones_like(current_rate) * target_rate_ffn - - # Mask out Copy Experts (Last Z experts) - # Assuming experts are ordered [FFN_0 ... FFN_N-1, Copy_0 ... Copy_Z-1] - if self.num_copy_experts > 0: - ffn_mask = mint.zeros(self.num_total_experts, dtype=mstype.bool_) - ffn_mask[:self.num_ffn_experts] = True - - # Zero out targets for copy experts (logic: no update) - # We achieve "no update" by making error term 0 effectively or masking update - pass # Implementation below uses mask on update directly - - error = target_rates - current_rate - update_step = self.bias_update_rate * error - - # Apply mask: Set update to 0 for Copy Experts - if self.num_copy_experts > 0: - mask = mint.zeros(self.num_total_experts, dtype=mstype.float32) - mask[:self.num_ffn_experts] = 1.0 - update_step = self.mul(update_step, mask) - - # Update bias in-place - ops.assign_add(self.expert_bias, update_step) - - def compute_load_balancing_loss(self, router_probs: Tensor, expert_indices: Tensor) -> Tensor: - """ - Grouped Load Balancing Loss. - Groups 1..D: FFN Experts. - Group D+1: Zero-Computation Experts. - - L_LB = alpha * sum_{j=1}^{D+1} (f_j * P_j) - """ - if self.aux_loss_coeff <= 0: - return mint.zeros((), dtype=router_probs.dtype) - - bs_slen = router_probs.shape[0] - - # 1. Calculate f_j (Frequency of selection per group) - # Flatten selected indices - selected_flat = self.reshape(expert_indices, (-1,)) - - # Count hits per expert - expert_counts = self.histc( - selected_flat, - bins=self.num_total_experts, - min=0, - max=self.num_total_experts - ) # Shape [Total_Experts] - - # Aggregation for FFN Groups - # Assuming FFN experts are 0..N-1, split into D groups - experts_per_ffn_group = self.num_ffn_experts // self.num_ffn_groups - - ffn_counts = expert_counts[:self.num_ffn_experts] - ffn_counts_reshaped = self.reshape(ffn_counts, (self.num_ffn_groups, experts_per_ffn_group)) - group_counts_ffn = self.sum(ffn_counts_reshaped, dim=1) # Shape [D] - - # Aggregation for Copy Group (Group D+1) - if self.num_copy_experts > 0: - copy_counts = expert_counts[self.num_ffn_experts:] - group_count_copy = self.sum(copy_counts).unsqueeze(0) # Shape [1] - all_group_counts = self.cat((group_counts_ffn, group_count_copy)) # Shape [D+1] - else: - all_group_counts = group_counts_ffn - - # Normalize counts to frequencies f_j - # FFN groups normalized by (Ke * T) ? Paper implies slightly different normalization, - # usually it is fraction of total selections. Let's use standard fraction within TopK. - # f_j = (tokens in group) / (Total Tokens * TopK) - # Note: LongCat paper Eq 3/4/5 suggests specific normalization denominators. - # Here we implement generic LB loss: sum(f_j * P_j) * N_groups - f_j = all_group_counts / (bs_slen * self.top_k) - - # 2. Calculate P_j (Sum of probabilities per group) - # Sum probs across batch for each expert - expert_prob_sum = self.sum(router_probs, dim=0) # Shape [Total_Experts] - - ffn_probs = expert_prob_sum[:self.num_ffn_experts] - ffn_probs_reshaped = self.reshape(ffn_probs, (self.num_ffn_groups, experts_per_ffn_group)) - group_probs_ffn = self.sum(ffn_probs_reshaped, dim=1) - - if self.num_copy_experts > 0: - copy_probs = expert_prob_sum[self.num_ffn_experts:] - group_prob_copy = self.sum(copy_probs).unsqueeze(0) - all_group_probs = self.cat((group_probs_ffn, group_prob_copy)) - else: - all_group_probs = group_probs_ffn - - # Normalize P_j (average probability per token) - P_j = all_group_probs / bs_slen - - # 3. Compute Loss - # Multiply and Sum - # We multiply by number of groups to keep magnitude similar to standard LB loss - num_groups = self.num_ffn_groups + (1 if self.num_copy_experts > 0 else 0) - loss = self.aux_loss_coeff * num_groups * self.sum(f_j * P_j) - - return loss - - def construct( - self, x: Tensor, training: bool = True - ) -> Tuple[Tensor, Tensor, Tensor, Tensor]: - """ - Returns: - top_scores: [BS*Seq, K] - selected_indices: [BS*Seq, K] - num_tokens_per_expert: [Num_Experts] - aux_loss: Scalar tensor - """ - bs_slen, _ = x.shape - router_dtype = self.config.moe_router_dtype - - # 1. Compute Router Scores - x_cast = self.cast(x, router_dtype) - weight = self.cast(self.weight, router_dtype) - logits = self.linear(x_cast, weight) # [BS*Seq, Total_Experts] - - # 2. Apply Softmax/Sigmoid - if self.score_func == "sigmoid": - probs = self.sigmoid(self.cast(logits, mstype.float32)) - elif self.score_func == "softmax": - probs = self.softmax(self.cast(logits, mstype.float32), dim=1) - else: - raise NotImplementedError(f"Unknown score function {self.score_func}") - - # 3. Add Bias (Computational Budget Control) - # bias is updated by PID but applied here for routing - probs_for_routing = probs + self.expert_bias - - # 4. TopK Selection - # Note: No node-limited routing here, standard TopK over all experts (FFN + Copy) - _, selected_indices = self.topk( - probs_for_routing, k=self.top_k, dim=-1, sorted=False - ) - selected_indices = self.cast(selected_indices, mstype.int64) - - # Gather real probabilities (without bias) for gating - top_scores = self.gather(probs, dim=1, index=selected_indices) - - # 5. Normalize and Scale - if self.route_norm: - denominator = self.sum(top_scores, dim=-1, keepdim=True) + 1e-20 - top_scores = self.div(top_scores, denominator) - - top_scores = self.mul(top_scores, self.route_scale) - - # 6. Statistics - num_tokens_per_expert = self.histc( - selected_indices, - bins=self.num_total_experts, - min=0, - max=self.num_total_experts, - ) - - # 7. Update Bias (PID Control) - if self.training and training: - self.update_expert_bias(num_tokens_per_expert, bs_slen) - - # 8. Compute Aux Loss (Load Balance Control) - aux_loss = self.compute_load_balancing_loss(probs, selected_indices) - - return top_scores, selected_indices, num_tokens_per_expert, aux_loss \ No newline at end of file -- Gitee From f040248f0d0eb0d0e056c03d8f8905da04ca7b3d Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Wed, 28 Jan 2026 16:18:28 +0800 Subject: [PATCH 11/12] =?UTF-8?q?=E4=BF=AE=E6=94=B9=E8=B7=AF=E7=94=B1?= =?UTF-8?q?=E5=90=8D=E7=A7=B02?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- mindformers/pynative/transformers/moe/router.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mindformers/pynative/transformers/moe/router.py b/mindformers/pynative/transformers/moe/router.py index 0d99e31b3..6ebbf4e6d 100644 --- a/mindformers/pynative/transformers/moe/router.py +++ b/mindformers/pynative/transformers/moe/router.py @@ -224,7 +224,7 @@ class TopKRouter(nn.Cell): class TopKRouterWithLoadBalancing(nn.Cell): """ - LongCat-Flash routing mechanism with Computational Budget Control and + Routing mechanism with Computational Budget Control and Load Balance Control. Computational Budget Control: @@ -245,7 +245,7 @@ class TopKRouterWithLoadBalancing(nn.Cell): self.num_total_experts = config.num_moe_experts self.top_k = config.moe_router_topk - # LongCat configuration + # examine numbers of experts self.num_copy_experts = self.config.num_copy_experts if self.num_copy_experts < 0 or self.num_copy_experts > self.num_total_experts: raise ValueError( -- Gitee From 1b3d475cd575c362860120aa399002384e691859 Mon Sep 17 00:00:00 2001 From: nie-zhentao Date: Wed, 28 Jan 2026 16:25:33 +0800 Subject: [PATCH 12/12] =?UTF-8?q?=E9=9B=B6=E8=AE=A1=E7=AE=97=E4=B8=93?= =?UTF-8?q?=E5=AE=B6+=E5=AF=B9=E5=BA=94=E6=8E=A7=E5=88=B6=E9=BB=98?= =?UTF-8?q?=E8=AE=A4=E5=85=B3=E9=97=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ds_pynative.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ds_pynative.yaml b/ds_pynative.yaml index b7081e834..1ab6a1ebe 100644 --- a/ds_pynative.yaml +++ b/ds_pynative.yaml @@ -234,8 +234,8 @@ model: n_routed_experts: 16 num_experts_per_tok: 8 n_shared_experts: 1 - num_copy_experts: 8 - use_topk_router_with_load_balancing: True + num_copy_experts: 0 + use_topk_router_with_load_balancing: False moe_expected_ffn_experts: 2.0 # Default best value: top-k * FFN/(FFN + COPY) moe_router_bias_update_rate: 0.001 moe_shared_expert_intermediate_size: 2048 -- Gitee