diff --git a/ds_pynative.yaml b/ds_pynative.yaml index 1ab6a1ebec5f5f5da3fd39e3c509d6e5e30e388c..b7081e834e04548859df5a90116252abc8bc55e3 100644 --- a/ds_pynative.yaml +++ b/ds_pynative.yaml @@ -234,8 +234,8 @@ model: n_routed_experts: 16 num_experts_per_tok: 8 n_shared_experts: 1 - num_copy_experts: 0 - use_topk_router_with_load_balancing: False + num_copy_experts: 8 + use_topk_router_with_load_balancing: True moe_expected_ffn_experts: 2.0 # Default best value: top-k * FFN/(FFN + COPY) moe_router_bias_update_rate: 0.001 moe_shared_expert_intermediate_size: 2048