diff --git a/mindformers/experimental/infer/core/parallel_paged_attention_mgr.py b/mindformers/experimental/infer/core/parallel_paged_attention_mgr.py
index 85aeb1a29bd3b29d272daf21881ad40a7363cbb1..37ca16488bb372f3972cb7ee342c8980766f30fa 100644
--- a/mindformers/experimental/infer/core/parallel_paged_attention_mgr.py
+++ b/mindformers/experimental/infer/core/parallel_paged_attention_mgr.py
@@ -17,9 +17,9 @@
 import math
 
 import mindspore.common.dtype as mstype
-from mindspore import nn, Parameter
 from mindspore import ops as P
-from mindspore.common.initializer import initializer
+from mindspore import nn
+from mindformers.experimental.infer.core.utils import create_empty_parameter
 
 
 class ParallelPagedAttentionMgr(nn.Cell):
@@ -45,10 +45,20 @@ class ParallelPagedAttentionMgr(nn.Cell):
         self.value_cache = None
         self.npu_mem_size = npu_mem_size
         if self.npu_mem_size > 0:
-            self.key_cache = Parameter(initializer('zeros', kv_shape, compute_dtype), name="key_cache",
-                                       requires_grad=False)
-            self.value_cache = Parameter(initializer('zeros', kv_shape, compute_dtype), name="value_cache",
-                                         requires_grad=False)
+            self.key_cache = create_empty_parameter(
+                shape=kv_shape,
+                dtype=compute_dtype,
+                device="CPU",
+                name="key_cache",
+                requires_grad=False,
+            )
+            self.value_cache = create_empty_parameter(
+                shape=kv_shape,
+                dtype=compute_dtype,
+                device="CPU",
+                name="value_cache",
+                requires_grad=False,
+            )
         self.reshape_and_cache = P.auto_generate.ReshapeAndCache()
         self.paged_attention = P.auto_generate.PagedAttention(self.n_heads,
                                                               self.scale_value,