diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000000000000000000000000000000000000..a8c200329368ddfdbe180fdbc2deda24ed7a9ce4 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,5 @@ +{ + "python-envs.defaultEnvManager": "ms-python.python:conda", + "python-envs.defaultPackageManager": "ms-python.python:conda", + "python-envs.pythonProjects": [] +} \ No newline at end of file diff --git a/ds_pynative.yaml b/ds_pynative.yaml index e76328464fe7c9c9269aaf7c62bdc139f7a5b654..1ab6a1ebec5f5f5da3fd39e3c509d6e5e30e388c 100644 --- a/ds_pynative.yaml +++ b/ds_pynative.yaml @@ -108,7 +108,7 @@ train_dataset: &train_dataset pad: -1 # The token id of `pad` in the dataset. data_path: # Megatron dataset sampling ratio and path. - '1' - - "/home/l00913161/data/deepseek-datasets/mmap_deepseekv3_datasets_text_document" + - "/home/w00932055/dsv4/deepseek-datasets/mmap_deepseekv3_datasets_text_document" input_columns: ["input_ids", "labels", "loss_mask", "position_ids"] construct_args_key: ["input_ids", "labels", "loss_mask", "position_ids"] num_parallel_workers: 8 @@ -234,15 +234,19 @@ model: n_routed_experts: 16 num_experts_per_tok: 8 n_shared_experts: 1 + num_copy_experts: 0 + use_topk_router_with_load_balancing: False + moe_expected_ffn_experts: 2.0 # Default best value: top-k * FFN/(FFN + COPY) + moe_router_bias_update_rate: 0.001 moe_shared_expert_intermediate_size: 2048 moe_grouped_gemm: True moe_router_load_balancing_type: 'seq_aux_loss' - moe_aux_loss_coeff: 0. # 0.001 + moe_aux_loss_coeff: 0.001 # 0.001 scoring_func: 'sigmoid' norm_topk_prob: True moe_token_drop_policy: probs moe_router_enable_expert_bias: True - moe_router_bias_update_rate: 0. # 0.001 + moe_router_bias_update_rate: 0.001 # 0.001 # callbacks callbacks: diff --git a/mindformers/parallel_core/inference/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg b/mindformers/parallel_core/inference/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9caae8fd2cb675dfdf8a44ccba74ba6d91e3a500 Binary files /dev/null and b/mindformers/parallel_core/inference/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/inference/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg b/mindformers/parallel_core/inference/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..16ce900aa1b8f77f49466f1b920d13463035d0aa Binary files /dev/null and b/mindformers/parallel_core/inference/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/inference/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg b/mindformers/parallel_core/inference/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..1baf2a58e2ea105200839d62fe77c1cfd022617e Binary files /dev/null and b/mindformers/parallel_core/inference/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/inference/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg b/mindformers/parallel_core/inference/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..e4be77d193af009a94615b4a4d6f434d9159fc7a Binary files /dev/null and b/mindformers/parallel_core/inference/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/inference/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg b/mindformers/parallel_core/inference/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..004face80aebf1033074f7d5e67c4f79df6884ad Binary files /dev/null and b/mindformers/parallel_core/inference/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/training_graph/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg b/mindformers/parallel_core/training_graph/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9c6e6becd3c866882de6a9d239ebec5a58441804 Binary files /dev/null and b/mindformers/parallel_core/training_graph/base_models/common/embeddings/__init__.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/training_graph/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg b/mindformers/parallel_core/training_graph/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9b4391cde0863c32794e1c5e99b4be8dba48c0e7 Binary files /dev/null and b/mindformers/parallel_core/training_graph/base_models/common/embeddings/language_model_embedding.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/training_graph/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg b/mindformers/parallel_core/training_graph/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..3ee15affdf17506693b557d7c83586bebfc62f39 Binary files /dev/null and b/mindformers/parallel_core/training_graph/base_models/common/embeddings/rope_utils.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/training_graph/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg b/mindformers/parallel_core/training_graph/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..37d86bce7d01e3fa82c49b6439c1b8d0bbcf23e2 Binary files /dev/null and b/mindformers/parallel_core/training_graph/base_models/common/embeddings/rotary_pos_embedding.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/training_graph/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg b/mindformers/parallel_core/training_graph/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c8c1a0b8ea66cbe37ea0d1b9c822180f8270a4e8 Binary files /dev/null and b/mindformers/parallel_core/training_graph/base_models/common/embeddings/yarn_rotary_pos_embedding.py.baiduyun.uploading.cfg differ diff --git a/mindformers/parallel_core/transformer_config.py b/mindformers/parallel_core/transformer_config.py index cd0b6df0e2e207b5b59ad80d2afc236b172493d2..dabf8f5fff5049127caef7fac24faf118d966dd5 100644 --- a/mindformers/parallel_core/transformer_config.py +++ b/mindformers/parallel_core/transformer_config.py @@ -254,7 +254,7 @@ class TransformerConfig(ModelParallelConfig, MFModelConfig): The default is "sub_seq_aux_loss". """ - moe_router_topk: int = 2 + moe_router_topk: int = 4 """Number of experts to route to for each token.""" moe_router_num_groups: Optional[int] = None @@ -354,6 +354,14 @@ class TransformerConfig(ModelParallelConfig, MFModelConfig): moe_apply_probs_on_input: bool = False """Apply probs on input of experts instead of applying after activation and glu.""" + num_copy_experts: int = 0 + + use_topk_router_with_load_balancing: bool = True + + moe_expected_ffn_experts: float = 2.0 # Default best value: top-k * FFN/(FFN + COPY) + + moe_router_bias_update_rate: float = 0.001 + # MindFormers New shared_expert_num: int = 0 """Number of shared experts.""" diff --git a/mindformers/parallel_core/transformer_config_utils.py b/mindformers/parallel_core/transformer_config_utils.py index 3f00c66b54e7a3c4ee18951919d73b5557b0d35f..2d0889e4c9b223b4a472928fe221c76118943710 100644 --- a/mindformers/parallel_core/transformer_config_utils.py +++ b/mindformers/parallel_core/transformer_config_utils.py @@ -382,7 +382,11 @@ COMMON_CONFIG_MAPPING = { "enable_expert_relocation": "enable_expert_relocation", "expert_relocation_initial_iteration": "expert_relocation_initial_iteration", "expert_relocation_freq": "expert_relocation_freq", - + "num_copy_experts": "num_copy_experts", + "use_topk_router_with_load_balancing": "use_topk_router_with_load_balancing", + "moe_expected_ffn_experts": "moe_expected_ffn_experts", + "moe_router_bias_update_rate": "moe_router_bias_update_rate", + # Context Parallel # not changes "context_parallel_algo": ("cp_comm_type", get_cp_comm_type), diff --git a/mindformers/pynative/base_models/gpt/moe_module_specs.py b/mindformers/pynative/base_models/gpt/moe_module_specs.py index 9d540addb1105615113aeb276e7493b2eb851231..09e19ff8cdb7b5977f8b33feb8367df88af9c446 100644 --- a/mindformers/pynative/base_models/gpt/moe_module_specs.py +++ b/mindformers/pynative/base_models/gpt/moe_module_specs.py @@ -31,9 +31,6 @@ def get_moe_module_spec( if num_experts is None: raise ValueError("num_experts cannot be None.") - # experts spec - if not moe_grouped_gemm: - raise NotImplementedError("moe_grouped_gemm = 'False' is not supported now.") shared_experts = SharedExpertMLP diff --git a/mindformers/pynative/transformers/moe/experts.py b/mindformers/pynative/transformers/moe/experts.py index 54c452272500720fead31ce4b2531af81f1887c5..4c68208a3572db7fcfa534ef9080550a6963893b 100644 --- a/mindformers/pynative/transformers/moe/experts.py +++ b/mindformers/pynative/transformers/moe/experts.py @@ -39,6 +39,14 @@ class GroupedMLP(nn.Cell): self.config = config self.num_local_experts = self.config.num_moe_experts self.top_k = self.config.moe_router_topk + self.num_copy_experts = self.config.num_copy_experts + + if self.num_copy_experts < 0 or self.num_copy_experts > self.num_local_experts: + raise ValueError( + f"num_copy_experts must be in [0, num_moe_experts], but got {self.num_copy_experts}." + ) + self.copy_expert_start = self.num_local_experts - self.num_copy_experts + self.num_ffn_experts = self.copy_expert_start if self.config.moe_apply_probs_on_input: if self.config.moe_router_topk == 1: @@ -69,10 +77,10 @@ class GroupedMLP(nn.Cell): # parameters self.weight1 = Parameter( - self.init_method([self.num_local_experts * self.hidden_size, self.moe_ffn_hidden_size]), + self.init_method([self.num_ffn_experts * self.hidden_size, self.moe_ffn_hidden_size]), name='w1') self.weight2 = Parameter( - self.init_method([self.num_local_experts * self.config.moe_ffn_hidden_size, self.hidden_size]), + self.init_method([self.num_ffn_experts * self.config.moe_ffn_hidden_size, self.hidden_size]), name='w2') self.cast = ops.cast @@ -183,6 +191,13 @@ class GroupedMLP(nn.Cell): # Process each expert out_experts_splits = [] for expert_idx, x_expert in enumerate(x_splits): + # CopyExpert: identity mapping (no FFN), keep routing weights + if expert_idx >= self.copy_expert_start: + h = x_expert + if not self.config.moe_apply_probs_on_input: + h = self.mul(h, permuted_probs_splits[expert_idx].reshape(-1, 1)) + out_experts_splits.append(h) + continue h = self.matmul(x_expert, w1[expert_idx]) h1, h2 = self.chunk(h, 2, -1) h1 = self.activation_func(h1) @@ -206,23 +221,47 @@ class GroupedMLP(nn.Cell): if self.moe_use_experts_for_loop: return self._run_experts_for_loop(w1, w2, permuted_local_hidden_states, tokens_per_expert, permuted_probs) - # Original grouped_mm implementation - tokens_per_expert = self.cumsum(tokens_per_expert, dim=0, dtype=ms.int64) - fc1_output = GroupedMatmul(split_item=3, group_type=0)( - [permuted_local_hidden_states], [w1], None, None, None, None, None, tokens_per_expert)[0] + # Split tokens into non-copy experts and copy experts to avoid extra compute + counts_list = tokens_per_expert.asnumpy().tolist() + non_copy_experts = self.num_ffn_experts + non_copy_tokens = sum(counts_list[:non_copy_experts]) + copy_tokens = sum(counts_list[non_copy_experts:]) if self.num_copy_experts > 0 else 0 - if self.gated_linear_unit: - if self.activation_type == 'fusedswiglu': - intermediate_parallel = self.activation_func(fc1_output, -1).reshape((-1, w2.shape[1])) + outputs = [] + if non_copy_experts > 0 and non_copy_tokens > 0: + # Run GroupedGEMM only for non-copy experts + non_copy_input = permuted_local_hidden_states[:non_copy_tokens] + non_copy_probs = permuted_probs[:non_copy_tokens] + tokens_per_expert_nc = tokens_per_expert[:non_copy_experts] + tokens_per_expert_nc = self.cumsum(tokens_per_expert_nc, dim=0, dtype=ms.int64) + + fc1_output = GroupedMatmul(split_item=3, group_type=0)( + [non_copy_input], [w1], None, None, None, None, None, tokens_per_expert_nc)[0] + + if self.gated_linear_unit: + if self.activation_type == 'fusedswiglu': + intermediate_parallel = self.activation_func(fc1_output, -1).reshape((-1, w2.shape[1])) + else: + x0, x1 = self.chunk(fc1_output, 2, -1) + act_out = self.activation_func(x0) + intermediate_parallel = self.mul(act_out, x1) else: - x0, x1 = self.chunk(fc1_output, 2, -1) - act_out = self.activation_func(x0) - intermediate_parallel = self.mul(act_out, x1) - else: - intermediate_parallel = self.activation_func(fc1_output) - - permuted_probs = self.cast(permuted_probs, intermediate_parallel.dtype) - intermediate_parallel = self.mul(intermediate_parallel, self.unsqueeze(permuted_probs, -1)) - fc2_output = GroupedMatmul(split_item=3, group_type=0)( - [intermediate_parallel], [w2], None, None, None, None, None, tokens_per_expert)[0] - return fc2_output + intermediate_parallel = self.activation_func(fc1_output) + + non_copy_probs = self.cast(non_copy_probs, intermediate_parallel.dtype) + intermediate_parallel = self.mul(intermediate_parallel, self.unsqueeze(non_copy_probs, -1)) + fc2_output = GroupedMatmul(split_item=3, group_type=0)( + [intermediate_parallel], [w2], None, None, None, None, None, tokens_per_expert_nc)[0] + outputs.append(fc2_output) + + if self.num_copy_experts > 0 and copy_tokens > 0: + copy_segment = permuted_local_hidden_states[non_copy_tokens: non_copy_tokens + copy_tokens] + if not self.config.moe_apply_probs_on_input: + copy_probs = self.cast(permuted_probs[non_copy_tokens: non_copy_tokens + copy_tokens], + permuted_local_hidden_states.dtype) + copy_segment = self.mul(copy_segment, copy_probs.reshape(-1, 1)) + outputs.append(copy_segment) + + if outputs: + return self.cat(outputs, dim=0) + return permuted_local_hidden_states[:0] diff --git a/mindformers/pynative/transformers/moe/moe_layer.py b/mindformers/pynative/transformers/moe/moe_layer.py index 2290d92f5c4fe36d0ed3acf98fba2e9c3a8cdaab..1429f517e0931631a091661df081f16d7ad754d8 100644 --- a/mindformers/pynative/transformers/moe/moe_layer.py +++ b/mindformers/pynative/transformers/moe/moe_layer.py @@ -20,7 +20,7 @@ from mindspore.common.parameter import Parameter from mindformers.parallel_core.transformer_config import TransformerConfig from mindformers.pynative.layers.linear import Linear from mindformers.pynative.transformers.mlp import MLPSubmodules -from .router import TopKRouter +from .router import TopKRouterWithLoadBalancing, TopKRouter from .experts import GroupedMLP from .shared_experts import SharedExpertMLP @@ -37,7 +37,12 @@ class MoELayer(nn.Cell): self.top_k = config.moe_router_topk # Router - self.router = TopKRouter(config) + # Prefer TopKRouterWithLoadBalancing when copy experts or aux loss is enabled + use_topk_router_with_load_balancing = self.config.use_topk_router_with_load_balancing + if use_topk_router_with_load_balancing: + self.router = TopKRouterWithLoadBalancing(config) + else: + self.router = TopKRouter(config) # Experts self.experts = GroupedMLP(config) @@ -94,7 +99,15 @@ class MoELayer(nn.Cell): bs, slen, dim = hidden_states.shape x_flat = self.reshape(hidden_states, (-1, dim)) - top_scores, selected_experts_indices, num_tokens_per_expert = self.router(x_flat, self.expert_bias) + if isinstance(self.router, TopKRouter): + top_scores, selected_experts_indices, num_tokens_per_expert = self.router( + x_flat, self.expert_bias + ) + aux_loss = None + else: + top_scores, selected_experts_indices, num_tokens_per_expert, aux_loss = self.router( + x_flat, training=self.training + ) self.tokens_per_expert.add_(num_tokens_per_expert) @@ -111,4 +124,4 @@ class MoELayer(nn.Cell): else: final_out = out_experts - return final_out, None + return final_out, aux_loss diff --git a/mindformers/pynative/transformers/moe/router.py b/mindformers/pynative/transformers/moe/router.py index 8c8d469c38490e9a0b33d0e13961b5310cf7026d..6ebbf4e6d1901c1e5b5da72d9747511f1b1affe7 100644 --- a/mindformers/pynative/transformers/moe/router.py +++ b/mindformers/pynative/transformers/moe/router.py @@ -220,3 +220,280 @@ class TopKRouter(nn.Cell): ) return top_scores, selected_experts_indices, num_tokens_per_expert + + +class TopKRouterWithLoadBalancing(nn.Cell): + """ + Routing mechanism with Computational Budget Control and + Load Balance Control. + + Computational Budget Control: + - PID-like bias update to maintain expected FFN activation rate. + - Bias updates are excluded for zero-computation (copy) experts. + + Load Balance Control: + - Auxiliary loss computed over expert groups. + - FFN experts are divided into D groups; copy experts form group D+1. + """ + + def __init__(self, config: TransformerConfig): + super().__init__() + self.config = config + + # Dimensions + self.hidden_size = config.hidden_size + self.num_total_experts = config.num_moe_experts + self.top_k = config.moe_router_topk + + # examine numbers of experts + self.num_copy_experts = self.config.num_copy_experts + if self.num_copy_experts < 0 or self.num_copy_experts > self.num_total_experts: + raise ValueError( + f"num_copy_experts must be in [0, num_moe_experts], but got {self.num_copy_experts}." + ) + self.num_ffn_experts = self.num_total_experts - self.num_copy_experts + if self.num_ffn_experts <= 0: + raise ValueError( + f"num_ffn_experts must be positive, got {self.num_ffn_experts}" + ) + + # Expected number of activated FFN experts + expected_ffn_k = getattr( + config, + "moe_expected_ffn_experts", + float(self.top_k * self.num_ffn_experts / self.num_total_experts), + ) + try: + self.expected_ffn_k = float(expected_ffn_k) + except (TypeError, ValueError) as exc: + raise ValueError( + f"moe_expected_ffn_experts must be a number, but got {expected_ffn_k}" + ) from exc + + # Bias update rate for computational budget control + self.bias_update_rate = self.config.moe_router_bias_update_rate + # PID gains and integral decay + self.pid_p = float(getattr(config, "moe_router_pid_p", 1.0)) + self.pid_i = float(getattr(config, "moe_router_pid_i", 0.01)) + self.pid_d = float(getattr(config, "moe_router_pid_d", 0.05)) + self.pid_i_decay = float(getattr(config, "moe_router_pid_i_decay", 0.9)) + + # Load balance control (grouped) + self.num_ffn_groups = config.moe_router_num_groups or 1 + if self.num_ffn_groups <= 0: + raise ValueError(f"moe_router_num_groups must be > 0, got {self.num_ffn_groups}") + if self.num_ffn_experts % self.num_ffn_groups != 0: + raise ValueError( + f"num_ffn_experts ({self.num_ffn_experts}) must be divisible by " + f"moe_router_num_groups ({self.num_ffn_groups})" + ) + self.aux_loss_coeff = config.moe_aux_loss_coeff + + # Routing parameters + self.score_func = config.moe_router_score_function + self.route_norm = config.norm_topk_prob + self.route_scale = ( + config.moe_router_topk_scaling_factor + if config.moe_router_topk_scaling_factor is not None + else 1.0 + ) + + # Learnable router weight + self.weight = Parameter( + init_method_normal(0.02)((self.num_total_experts, self.hidden_size)), + name="weight", + ) + + # Expert bias (updated by controller; not trained by gradients) + self.expert_bias = Parameter( + mint.zeros(self.num_total_experts, dtype=mstype.float32), + name="expert_bias", + requires_grad=False, + ) + # PID controller states + self.integral = Parameter( + mint.zeros(self.num_total_experts, dtype=mstype.float32), + name="expert_bias_integral", + requires_grad=False, + ) + self.previous_error = Parameter( + mint.zeros(self.num_total_experts, dtype=mstype.float32), + name="expert_bias_prev_error", + requires_grad=False, + ) + + # Ops + self.linear = mint.nn.functional.linear + self.sigmoid = mint.nn.functional.sigmoid + self.softmax = mint.nn.functional.softmax + self.cast = ops.cast + self.topk = mint.topk + self.gather = mint.gather + self.mul = mint.mul + self.div = mint.div + self.sum = mint.sum + self.histc = mint.histc + self.ones_like = mint.ones_like + self.reshape = mint.reshape + self.cat = mint.cat + self.zeros = mint.zeros + + def update_expert_bias(self, num_tokens_per_expert: Tensor, total_tokens: int): + """ + Bias update for Computational Budget Control. + delta_b_i = mu * (target_rate - current_rate) + Only updates FFN experts; copy experts are excluded. + """ + if not self.training or self.bias_update_rate <= 0 or total_tokens <= 0: + return + + # Current selection rate: Ti / (K * T_all) + current_rate = self.cast(num_tokens_per_expert, mstype.float32) / ( + self.top_k * total_tokens + ) + + # Target selection rate for FFN experts: Ke / (K * N) + target_rate_ffn = self.expected_ffn_k / (self.top_k * self.num_ffn_experts) + target_rates = self.ones_like(current_rate) * target_rate_ffn + + error = target_rates - current_rate + + # Mask out copy experts (no bias update) + if self.num_copy_experts > 0: + mask = self.zeros((self.num_total_experts,), dtype=mstype.float32) + mask[: self.num_ffn_experts] = 1.0 + error = self.mul(error, mask) + + # PID terms + p_term = self.mul(error, self.pid_p) + integral = self.mul(self.integral, self.pid_i_decay) + self.mul( + error, 1.0 - self.pid_i_decay + ) + i_term = self.mul(integral, self.pid_i) + d_term = self.mul(error - self.previous_error, self.pid_d) + + update_step = self.bias_update_rate * (p_term + i_term + d_term) + + ops.assign_add(self.expert_bias, update_step) + ops.assign(self.integral, integral) + ops.assign(self.previous_error, error) + + def compute_load_balancing_loss(self, router_probs: Tensor, expert_indices: Tensor) -> Tensor: + """ + Grouped load balancing loss: + L_LB = alpha * sum_{j=1}^{D+1} (f_j * P_j) + """ + if self.aux_loss_coeff <= 0: + return self.zeros((), dtype=router_probs.dtype) + + bs_slen = router_probs.shape[0] + if bs_slen == 0: + return self.zeros((), dtype=router_probs.dtype) + + # 1) f_j: frequency of selection per group + selected_flat = self.reshape(expert_indices, (-1,)) + expert_counts = self.histc( + selected_flat, + bins=self.num_total_experts, + min=0, + max=self.num_total_experts, + ) + expert_counts = self.cast(expert_counts, router_probs.dtype) + + experts_per_ffn_group = self.num_ffn_experts // self.num_ffn_groups + ffn_counts = expert_counts[: self.num_ffn_experts] + ffn_counts_reshaped = self.reshape( + ffn_counts, (self.num_ffn_groups, experts_per_ffn_group) + ) + group_counts_ffn = self.sum(ffn_counts_reshaped, dim=1) + + if self.num_copy_experts > 0: + copy_counts = expert_counts[self.num_ffn_experts :] + group_count_copy = self.sum(copy_counts).unsqueeze(0) + all_group_counts = self.cat((group_counts_ffn, group_count_copy)) + else: + all_group_counts = group_counts_ffn + + f_j = all_group_counts / (bs_slen * self.top_k) + + # 2) P_j: probability mass per group + expert_prob_sum = self.sum(router_probs, dim=0) + ffn_probs = expert_prob_sum[: self.num_ffn_experts] + ffn_probs_reshaped = self.reshape( + ffn_probs, (self.num_ffn_groups, experts_per_ffn_group) + ) + group_probs_ffn = self.sum(ffn_probs_reshaped, dim=1) + + if self.num_copy_experts > 0: + copy_probs = expert_prob_sum[self.num_ffn_experts :] + group_prob_copy = self.sum(copy_probs).unsqueeze(0) + all_group_probs = self.cat((group_probs_ffn, group_prob_copy)) + else: + all_group_probs = group_probs_ffn + + P_j = all_group_probs / bs_slen + + num_groups = self.num_ffn_groups + (1 if self.num_copy_experts > 0 else 0) + loss = self.aux_loss_coeff * num_groups * self.sum(f_j * P_j) + return loss + + def construct( + self, x: Tensor, training: bool = True + ) -> Tuple[Tensor, Tensor, Tensor, Tensor]: + """ + Returns: + top_scores: [bs*slen, K] + selected_indices: [bs*slen, K] + num_tokens_per_expert: [num_experts] + aux_loss: scalar + """ + bs_slen, _ = x.shape + router_dtype = self.config.moe_router_dtype + + # 1) Router logits + x_cast = self.cast(x, router_dtype) + weight = self.cast(self.weight, router_dtype) + logits = self.linear(x_cast, weight) + + # 2) Probabilities + if self.score_func == "sigmoid": + probs = self.sigmoid(self.cast(logits, mstype.float32)) + elif self.score_func == "softmax": + probs = self.softmax(self.cast(logits, mstype.float32), dim=1) + else: + raise NotImplementedError(f"Unknown score function {self.score_func}") + + # 3) Add bias for routing only + probs_for_routing = probs + self.expert_bias + + # 4) TopK selection + _, selected_indices = self.topk( + probs_for_routing, k=self.top_k, dim=-1, sorted=False + ) + selected_indices = self.cast(selected_indices, mstype.int64) + + # 5) Gather routing scores + top_scores = self.gather(probs, dim=1, index=selected_indices) + + # 6) Normalize and scale + if self.route_norm: + denominator = self.sum(top_scores, dim=-1, keepdim=True) + 1e-20 + top_scores = self.div(top_scores, denominator) + top_scores = self.mul(top_scores, self.route_scale) + + # 7) Statistics + num_tokens_per_expert = self.histc( + selected_indices, + bins=self.num_total_experts, + min=0, + max=self.num_total_experts, + ) + + # 8) Update bias (PID-like control) + if self.training and training: + self.update_expert_bias(num_tokens_per_expert, bs_slen) + + # 9) Load balance loss + aux_loss = self.compute_load_balancing_loss(probs, selected_indices) + + return top_scores, selected_indices, num_tokens_per_expert, aux_loss \ No newline at end of file diff --git a/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/base_model.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/base_model.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..66f597b6599f74b6deec78db46777952c5252dcb Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/base_model.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/run_parallel.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/run_parallel.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..f3ea0c014aba1445dde72a4665a87498951908fc Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/run_parallel.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/test_llama.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/test_llama.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..bd8ab84dfe6566322b9e545810fc5535c710b739 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_legacy/test_model/test_llama/test_llama.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..2673ff3bee34326a710374461ce155daa5618aae Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/predict_deepseek3_671b.yaml.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/predict_deepseek3_671b.yaml.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c54b469ed0441300758f5d49f6db513a4f5596a5 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/predict_deepseek3_671b.yaml.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_deepseek_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_deepseek_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..ac0bc73d4e6ee92700de02c730ef879235b1de41 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_deepseek_infer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_quant_deepseek_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_quant_deepseek_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..1565e24f9bb9e429f74ae4c739af3a7723bec351 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/run_quant_deepseek_infer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_deepseek_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_deepseek_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9b3147e4cc44f7e925e74bf412f2c9430727cc90 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_deepseek_infer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_quant_deepseek_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_quant_deepseek_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..b7103922c5119ab69931b62f2884a424820351da Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_deepseek3_infer/test_quant_deepseek_infer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_eplb/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_eplb/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..bef7de911a0f5c5b8d43fb77ab990e03ff2fe093 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_eplb/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_eplb/test_moe_eplb.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_eplb/test_moe_eplb.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9ef55fc8798f50d610557c82a73c042919683c0f Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_eplb/test_moe_eplb.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..00d9084d452cf0548488546dd5340d45c320b1f6 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/test_moe_token_permute.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/test_moe_token_permute.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..0a2eccbd9ea3b0657f8db4a8a0bb3ba038adbb3b Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_deepseek3/test_moe_token_permute/test_moe_token_permute.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..3ac51db3b7c4ac2922ba5ce1b814358839ee5f03 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/glm4_moe_infer.yaml.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/glm4_moe_infer.yaml.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8049c9fcc34c0a206ca12a82a2323f740c36c555 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/glm4_moe_infer.yaml.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/run_glm4_moe.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/run_glm4_moe.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..be632016d96edc0616468ffa3d63ee206d876ea0 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/run_glm4_moe.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/test_glm4_moe_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/test_glm4_moe_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c751417db70d47da43948c4a553e63d5b18bd5d9 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_glm4_moe/test_glm4_moe_infer/test_glm4_moe_infer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..7a9b70fe3f2f0554875166e5e610526ca271c4f1 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/qwen3_0_6b_infer.yaml.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/qwen3_0_6b_infer.yaml.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..72cd201a514e0def4d308f6dec86499201fe4bca Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/qwen3_0_6b_infer.yaml.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/run_qwen3.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/run_qwen3.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..7cfa7c0adfdb2471c0c047663ff360bafeb1ce31 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/run_qwen3.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/test_qwen3_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/test_qwen3_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..69936bc8f64e6500d267a10f4c668add369b2339 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_qwen3/test_qwen3_infer/test_qwen3_infer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..395ed56f5c37fcab695bae941ec08da7b58af805 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/qwen3_moe_30b_a3b_infer.yaml.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/qwen3_moe_30b_a3b_infer.yaml.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..48fbe44827fbf521d64ebe8483e6b8c5bcb8c943 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/qwen3_moe_30b_a3b_infer.yaml.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/run_qwen3_moe.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/run_qwen3_moe.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..de7a3114f8d1a86aaeb5d4707810086033694610 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/run_qwen3_moe.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/test_qwen3_moe_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/test_qwen3_moe_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..29530b796c4055c38e1a136feed261c221ff9824 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_qwen3_moe/test_qwen3_moe_infer/test_qwen3_moe_infer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..d2cae9af0066104e04c2102cd203becf450a1d53 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/run_telechat2_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/run_telechat2_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..a564e1f93e72c9ef18f8d9380dce5cdd485aa1ae Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/run_telechat2_infer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/telechat2_infer.yaml.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/telechat2_infer.yaml.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..0ee25639f0adf90a6b23097a525e1d72e5b68ac2 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/telechat2_infer.yaml.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/test_telechat2_infer.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/test_telechat2_infer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..07111a5776ccdbd4821b5de796c3e6de9ba293e1 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_model/test_telechat2/test_telechat2_infer/test_telechat2_infer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_inference/test_base_models/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_inference/test_base_models/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..77f6d033a857a9072577c154b4f705da9822c4c2 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_parallel_core/test_inference/test_base_models/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8fb68f1402decbd1ab6d2a07ffca905319b55a5b Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_base_models/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_base_models/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..97086c49a55d985a801dc18d91e4f1d22340c14e Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_base_models/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_tensor_parallel/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_tensor_parallel/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..88970d419ad03cbab9409752d2ce97cf6c925453 Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_tensor_parallel/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/__init__.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..ebcb347bf5e81df6ba98df506211ce05839944cd Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/test_vocab_parallel_cross_entropy_parallel.py.baiduyun.uploading.cfg b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/test_vocab_parallel_cross_entropy_parallel.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..0c163c6c5e8edec73a1579bbeed93bf4c39a6ffc Binary files /dev/null and b/tests/st/test_multi_cards_cases/test_parallel_core/test_train_graph/test_vocab_parallel_crossentropy_parallel/test_vocab_parallel_cross_entropy_parallel.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..ac4d55b64b43e0d8dc0dcd479be7c043e6c1b09e Binary files /dev/null and b/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..27668c2953e6ddc0a1413033f7ffda7bbd354a59 Binary files /dev/null and b/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader_broadcast.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader_broadcast.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c7b3e29a8bab03953ceb529b908b6077b0b8472c Binary files /dev/null and b/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader_broadcast.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader_streaming.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader_streaming.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..b1fc3ff7160fad80743a52b3f44f78320b387019 Binary files /dev/null and b/tests/st/test_ut/test_dataset/test_dataloader/test_hf_dataloader/test_hf_dataloader_streaming.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_inference/test_base_models/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_inference/test_base_models/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..d65b68a2e9f95911327a7c2c730a0a90e2cd1034 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_inference/test_base_models/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_inference/test_common/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_inference/test_common/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..78ad5a468ee5a2d3c221c2f09de832953fb58056 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_inference/test_common/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..52aaaa4ac22b4ecc76cc9d52d1d7cd83ea68f441 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_inference/test_transformer/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_inference/test_utils/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_inference/test_utils/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..20d423b9deddda353af6dfa4a11372111f024a39 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_inference/test_utils/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_inference/test_utils/test_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_inference/test_utils/test_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8ff9709aecb4b4448c7282d0a5847c2c31af9492 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_inference/test_utils/test_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..44c0aaa1c852eca16b7a16758ed07cbd80849d9c Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..6810be0e287ec1d0e09e806b72a12ecbb20bd4d0 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_activation.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_activation.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..4b08320731166e3ff199c45f6db298f71dab93a4 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_activation.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_fused_swiglu.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_fused_swiglu.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..44893c4bbf18ffc7820c696c1dac442efe4092a8 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_training_graph/test_activation/test_fused_swiglu.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_transformer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_transformer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..81c7a6dc03a6e9fae879d82182d53a9c282c0b8f Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_training_graph/test_transformer/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..5deb6bf3fa7931b0f17f8ca181d027e48845c2a5 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8bf4ba5f0c30b598caaed27eea4a351045093b18 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/run_vocab_parallel_cross_entropy.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/run_vocab_parallel_cross_entropy.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..f56b7bf6845702dd232cf041b0b66e40dfa72911 Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/run_vocab_parallel_cross_entropy.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/test_vocab_parallel_cross_entropy.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/test_vocab_parallel_cross_entropy.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..830497a26da57804841ad94cc55edec07f3c58ea Binary files /dev/null and b/tests/st/test_ut/test_parallel_core/test_training_graph/test_vocab_parallel_crossentropy/test_vocab_parallel_cross_entropy.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_base_models/test_embeddings/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_base_models/test_embeddings/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..4ddc636b39f28956e7ab2764ef73bbd10ea99e2d Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_base_models/test_embeddings/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layer/test_activation/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layer/test_activation/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c02035522a93edb079e667380065c42abcfd8898 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layer/test_activation/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layer/test_activation/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layer/test_activation/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c5a773d39d18e7a79c864927ca990ebfe3eb5612 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layer/test_activation/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layer/test_activation/test_activation.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layer/test_activation/test_activation.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..ce59c654ef30386e5bcdc1150982efc7a00e422a Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layer/test_activation/test_activation.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_activation/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_activation/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..1dc05b9b4328db8094a07b02ccf1d94b3124a3b6 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_activation/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_activation/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_activation/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..189db90766e15b60a588012dbf146eb3af1af925 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_activation/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_activation/test_activation.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_activation/test_activation.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..646eac3747e0da437c3650cf569666ddeed09bb0 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_activation/test_activation.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..fe4f482f979c60d17cd922ac7df992f82ca36423 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..45bde5f6ffdbf6b62de5d9f1fdedee6cfbcd9ea7 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/run_causal_mask_generate.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/run_causal_mask_generate.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..9a3972dd8691ec37d8d826743f7d66ede49867b6 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/run_causal_mask_generate.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/test_causal_mask_generate.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/test_causal_mask_generate.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..2d9585b1d02aed617f64861431b760388cff0dcd Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_causal_mask_generate/test_causal_mask_generate.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..c8460f8ebf3a870e4b7d20dc0db3bf1d1f97c10a Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..5de89cd6f112bc22725e31b51cc13fcd97b9f66a Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/test_flash_attention.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/test_flash_attention.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..708bfc0c495f7fc57dc0efd9831d3734c0e27621 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_flash_attention/test_flash_attention.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_linear/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_linear/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..175c6ad2b21f414e4565b32ad403a7898a4d07e8 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_linear/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_linear/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_linear/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..f47b70ea32ea2ab9d496716876241c9c0409f87a Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_linear/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_linear/run_linear.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_linear/run_linear.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..3c7c1d77074635b130a5d922297f44ef5d6aa4f2 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_linear/run_linear.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_linear/test_linear.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_linear/test_linear.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..6525b8c12de0f1dadd04f1967ba0f6e462e3339b Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_linear/test_linear.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_norm/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_norm/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..ec0e56280a7226a25fafb766a4a29f67b83c15fc Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_norm/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_norm/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_norm/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..7903b6721a69d62c6d71b669e80ef0e89fc65ad5 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_norm/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_layers/test_norm/test_fused_norm.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_layers/test_norm/test_fused_norm.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..8a44c6e87c5249eb76335785562831ad1d52a9db Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_layers/test_norm/test_fused_norm.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_mlp/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_mlp/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..f2ca5d6ee6db03ad4d5f1f65743f39a1785ef8ea Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_mlp/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_mlp/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_mlp/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..bc683399dddd9ba04be244b698eddb950bbab874 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_mlp/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_mlp/run_mlp.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_mlp/run_mlp.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..734825e40466e8278f98e07b44fc428825d607ed Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_mlp/run_mlp.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_mlp/test_mlp.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_mlp/test_mlp.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..e77fac55522fe0c99bd31266c6633f221cdd48b5 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_mlp/test_mlp.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_moe/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_moe/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..b345ef986f19e4d529c3ea7c406ec5351a633c0c Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_moe/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..f325770624063ab952fadb46a25c21a55893b131 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..68c295097ef03a724b6b1409be485bd5d6e2615e Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/run_transformer_block.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/run_transformer_block.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..fbf317ff5aa6b48099b13e258ca8fcb2bae7ef29 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/run_transformer_block.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/test_transformer_block.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/test_transformer_block.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..d5169dd2dd91f16d5e2f3f0a0d0ead6b1522c24b Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_block/test_transformer_block.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/__init__.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/__init__.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..f2e4b8caba6088548f714e18bd3dd22375ab4649 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/__init__.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/data_gen_utils.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/data_gen_utils.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..6403f73652daacf81537c7297753969caf622dce Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/data_gen_utils.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/run_transformer_layer.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/run_transformer_layer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..4582b30f9723bc048126955d15509d5bfd7218d6 Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/run_transformer_layer.py.baiduyun.uploading.cfg differ diff --git a/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/test_transformer_layer.py.baiduyun.uploading.cfg b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/test_transformer_layer.py.baiduyun.uploading.cfg new file mode 100644 index 0000000000000000000000000000000000000000..17b903621cf8092c45106d2c883cdf3de64010ed Binary files /dev/null and b/tests/st/test_ut/test_pynative/test_transformers/test_transformer_layer/test_transformer_layer.py.baiduyun.uploading.cfg differ