diff --git a/mindspore_serving/server/_servable_local.py b/mindspore_serving/server/_servable_local.py
index 5cdaea063dfde05ff30bb9b97873a59b6c28b830..a063754d17417f4b08bc9adb4e5b7c42047a0a63 100644
--- a/mindspore_serving/server/_servable_local.py
+++ b/mindspore_serving/server/_servable_local.py
@@ -21,14 +21,13 @@ import subprocess
 
 from mindspore_serving import log as logger
 from mindspore_serving.server.common import check_type, get_abs_path
-from mindspore_serving.server.worker import init_mindspore, get_newest_version_number
+from mindspore_serving.server.worker import get_newest_version_number
 from mindspore_serving.server._servable_common import ServableContextDataBase
 
 
 def _get_device_type(target_device_type):
     """Get device type supported, this will load libmindspore.so or libmindspore-lite.so"""
     # Get Device type: Ascend, Gpu, Cpu
-    init_mindspore.set_mindspore_cxx_env()
     args = f"{sys.executable} -c 'from mindspore_serving._mindspore_serving import Worker_;" \
            f"device_type=Worker_.get_device_type(\"{target_device_type}\");" \
            f"print(\"#get_device_type_result=\", device_type, \"#\", sep=\"\")'"
@@ -48,7 +47,6 @@ def _get_device_type(target_device_type):
 def _all_reuse_device():
     """Get device type supported, this will load libmindspore.so or libmindspore-lite.so"""
     # Whether allow reuse device, for Ascend910 return False, other return True
-    init_mindspore.set_mindspore_cxx_env()
     args = f"{sys.executable} -c 'from mindspore_serving._mindspore_serving import Worker_;" \
            f"reuse_flag=Worker_.support_reuse_device();" \
            f"print(\"#get_reuse_flag_result=\", reuse_flag, \"#\", sep=\"\")'"
diff --git a/mindspore_serving/server/_server.py b/mindspore_serving/server/_server.py
index f5b4346a629707f0c44e12921aac9c1e8ad94154..a2979e1a7fddd07fd88a10fc134700b3bec5bdcb 100644
--- a/mindspore_serving/server/_server.py
+++ b/mindspore_serving/server/_server.py
@@ -19,6 +19,7 @@ import threading
 import signal
 
 import mindspore_serving.log as logger
+from mindspore_serving.server.worker.init_mindspore import set_mindspore_cxx_env
 from mindspore_serving.server.master import start_master_server, stop_on_except, stop, at_stop_list, only_model_stage
 from mindspore_serving.server._servable_common import WorkerContext
 from mindspore_serving.server._servable_local import ServableStartConfig, ServableContextData, merge_config
@@ -69,6 +70,7 @@ def start_servables(servable_configs):
                 f"The item of parameter '{servable_configs}' should be ServableStartConfig, but actually "
                 f"{type(config)}")
 
+    set_mindspore_cxx_env()
     # merge ServableStartConfig with same servable name and running version number
     try:
         servable_configs = merge_config(servable_configs)
@@ -99,7 +101,6 @@ def start_servables(servable_configs):
     start_master_server(address=master_address)
 
     signal.signal(signal.SIGCHLD, signal.SIG_IGN)
-
     worker_list = _start_workers_with_devices(master_address, servable_configs)
     has_device_workers = bool(worker_list)
     _listening_workers_when_startup(worker_list)
diff --git a/mindspore_serving/server/worker/init_mindspore.py b/mindspore_serving/server/worker/init_mindspore.py
index c5b18946fccb829e9a5107a5c550f6aede543dfb..47c790049ecc9f01148c766e698edccedf817446 100644
--- a/mindspore_serving/server/worker/init_mindspore.py
+++ b/mindspore_serving/server/worker/init_mindspore.py
@@ -65,18 +65,23 @@ def set_mindspore_cxx_env():
         return
     _flag_set_mindspore_cxx_env = True
 
+    ld_lib_path = os.getenv('LD_LIBRARY_PATH', "")
+    check_version_and_try_set_env_lib()  # try set env LD_LIBRARY_PATH
+    logger.info(f"Update env LD_LIBRARY_PATH from '{ld_lib_path}' to '{os.getenv('LD_LIBRARY_PATH')}'")
+
+    ld_lib_path = os.getenv('LD_LIBRARY_PATH', "")
     ms_dir = get_mindspore_whl_path()
     if not ms_dir:
         logger.info(f"find mindspore failed, LD_LIBRARY_PATH will not add MindSpore lib path")
         return
     check_mindspore_version(ms_dir)
     ms_dir = os.path.join(ms_dir, "lib")
-    ld_lib_path = os.getenv('LD_LIBRARY_PATH', "")
+
     if ld_lib_path:
-        os.environ['LD_LIBRARY_PATH'] = ld_lib_path + ":" + ms_dir
+        if ms_dir not in ld_lib_path.split(":"):
+            os.environ['LD_LIBRARY_PATH'] = ld_lib_path + ":" + ms_dir
     else:
         os.environ['LD_LIBRARY_PATH'] = ms_dir
-    check_version_and_try_set_env_lib()  # try set env LD_LIBRARY_PATH
     logger.info(f"Update env LD_LIBRARY_PATH from '{ld_lib_path}' to '{os.getenv('LD_LIBRARY_PATH')}'")