From e49ebe2e93e7e3435c632f6d7a8b1b455373c1da Mon Sep 17 00:00:00 2001 From: andy_git Date: Sat, 19 Apr 2025 12:09:54 +0800 Subject: [PATCH] add syn create vg --- src/cmd/dsscmd.c | 10 +- src/cmd/dsscmd_volume.c | 65 ++++--- src/common/dss_meta_buf.c | 5 +- src/common/dss_session.c | 4 +- src/common/dss_session.h | 2 - src/common/persist/dss_ctrl_def.h | 35 ++-- src/common/persist/dss_diskgroup.c | 177 ++++++++++--------- src/common/persist/dss_diskgroup.h | 12 +- src/common/persist/dss_redo_recovery.c | 76 +++++++++ src/common/persist/dss_redo_recovery.h | 1 + src/common_api/dss_api_impl.c | 225 ++++++++++++------------ src/common_api/dss_api_impl.h | 7 +- src/common_api/dss_cli_conn.c | 86 ++++++++++ src/common_api/dss_cli_conn.h | 6 +- src/service/dss_instance.c | 227 +++++++++++++++++++------ src/service/dss_instance.h | 20 ++- src/service/dss_service.c | 8 +- src/service/dssserver.c | 25 +-- src/tbox/dsstbox.c | 8 +- 19 files changed, 652 insertions(+), 347 deletions(-) diff --git a/src/cmd/dsscmd.c b/src/cmd/dsscmd.c index 231640c..3928362 100644 --- a/src/cmd/dsscmd.c +++ b/src/cmd/dsscmd.c @@ -1635,8 +1635,8 @@ static status_t query_latch_remain_proc(void) DSS_PRINT_INF("Begin to query latch remain.\n"); int64 inst_id; status_t status = cm_str2bigint(cmd_query_latch_remain_args[DSS_ARG_IDX_0].input_args, &inst_id); - DSS_RETURN_IFERR2( - status, DSS_PRINT_ERROR("inst_id:%s is not a valid int64.\n", cmd_query_latch_remain_args[DSS_ARG_IDX_0].input_args)); + DSS_RETURN_IFERR2(status, + DSS_PRINT_ERROR("inst_id:%s is not a valid int64.\n", cmd_query_latch_remain_args[DSS_ARG_IDX_0].input_args)); char *home = cmd_query_latch_remain_args[DSS_ARG_IDX_2].input_args; int64 type = DSS_LATCH_ALL; if (cmd_query_latch_remain_args[DSS_ARG_IDX_1].inputed) { @@ -2273,7 +2273,7 @@ static status_t showdisk_proc(void) DSS_PRINT_ERROR("Failed to load config info!\n"); return status; } - status = dss_load_vg_conf_info(&g_vgs_info, inst_cfg); + status = dss_load_vg_conf_info(inst_cfg); if (status != CM_SUCCESS) { LOG_DEBUG_ERR("Failed to load vg info from config, errcode is %d.\n", status); return status; @@ -3613,8 +3613,8 @@ static status_t getcfg_proc(void) if (conn == NULL) { return CM_ERROR; } - char value[DSS_PARAM_BUFFER_SIZE] = {0}; - status_t status = dss_getcfg_impl(conn, name, value, DSS_PARAM_BUFFER_SIZE); + char value[CM_PARAM_BUFFER_SIZE] = {0}; + status_t status = dss_getcfg_impl(conn, name, value, CM_PARAM_BUFFER_SIZE); if (status != CM_SUCCESS) { if (strlen(value) != 0 && cm_str_equal_ins(name, "SSL_PWD_CIPHERTEXT")) { LOG_DEBUG_ERR("Failed to get cfg, name is %s, value is ***.\n", name); diff --git a/src/cmd/dsscmd_volume.c b/src/cmd/dsscmd_volume.c index d71a993..181daba 100644 --- a/src/cmd/dsscmd_volume.c +++ b/src/cmd/dsscmd_volume.c @@ -46,7 +46,7 @@ static void dss_set_ctrl_checksum(dss_ctrl_t *ctrl) // NOTE:only called by create vg, no need to record redo log static status_t vg_initialize_resource(dss_vg_info_item_t *vg_item, gft_node_t *parent_node) { - status_t status = + status_t status = dss_alloc_ft_node_when_create_vg(vg_item, parent_node, DSS_RECYLE_DIR_NAME, GFT_PATH, DSS_FT_NODE_FLAG_SYSTEM); if (status != CM_SUCCESS) { LOG_RUN_ERR("Failed to allocate .recycle file when create vg %s.", vg_item->vg_name); @@ -229,7 +229,7 @@ static status_t dss_set_vg_ctrl( DSS_THROW_ERROR(ERR_ALLOC_MEMORY, sizeof(dss_ctrl_t), "vg_ctrl"); return CM_ERROR; } - (void)memset_s(vg_ctrl, sizeof(dss_ctrl_t), 0 , sizeof(dss_ctrl_t)); + (void)memset_s(vg_ctrl, sizeof(dss_ctrl_t), 0, sizeof(dss_ctrl_t)); vg_item->dss_ctrl = vg_ctrl; do { dss_volume_t volume; @@ -269,29 +269,44 @@ status_t dss_create_vg(const char *vg_name, const char *volume_name, dss_config_ dss_static_assert_info(); LOG_RUN_INF("[VOL][CV] Begin to create vg %s.", vg_name); - status = dss_load_vg_conf_info(&g_vgs_info, inst_cfg); + status = dss_load_vg_conf_info(inst_cfg); if (status != CM_SUCCESS) { LOG_DEBUG_ERR( - "[VOL][CV] Failed to load vg info from config, vg name is %s, volume name is %s, errcode is %d.\n", - vg_name, volume_name, status); + "[VOL][CV] Failed to load vg info from config, vg name is %s, volume name is %s, errcode is %d.\n", vg_name, + volume_name, status); return status; } dss_vg_info_item_t *vg_item = dss_find_vg_item(vg_name); - if (vg_item == NULL) { - dss_free_vg_info(); - LOG_DEBUG_ERR( - "[VOL][CV] Failed to find vg info from config, vg name is %s, volume name is %s, errcode is %d.\n", - vg_name, volume_name, status); - DSS_THROW_ERROR(ERR_DSS_VG_CREATE, vg_name, "Failed to find vg info from config"); - return CM_ERROR; - } + // for old version, cfg dss_vg_conf.ini first, then create + if (vg_item != NULL) { + if (vg_item->entry_path[0] == '\0' || cm_strcmpi(vg_item->entry_path, volume_name) != 0) { + dss_free_vg_info(); + DSS_THROW_ERROR( + ERR_DSS_VG_CREATE, vg_name, "Failed to cmp super-block name with entry_path config in dss_vg_conf.\n"); + return CM_ERROR; + } + } else { + // no need check cfg from dss_vg_conf.ini frist, after create success, update the dss_vg_conf.ini for adding vg + // dyn + if (g_vgs_info->group_num >= DSS_MAX_VOLUME_GROUP_NUM) { + LOG_DEBUG_ERR( + "[VOL][CV] Failed to create vg over limit, vg name is %s, volume name is %s.\n", vg_name, volume_name); + return CM_ERROR; + } - if (vg_item->entry_path[0] == '\0' || cm_strcmpi(vg_item->entry_path, volume_name) != 0) { - dss_free_vg_info(); - DSS_THROW_ERROR( - ERR_DSS_VG_CREATE, vg_name, "Failed to cmp super-block name with entry_path config in dss_vg_conf.\n"); - return CM_ERROR; + vg_item = &g_vgs_info->volume_group[g_vgs_info->group_num]; + (void)memcpy_s(vg_item->vg_name, DSS_MAX_NAME_LEN, vg_name, DSS_MAX_NAME_LEN); + (void)memcpy_s(vg_item->entry_path, DSS_MAX_VOLUME_PATH_LEN, volume_name, DSS_MAX_VOLUME_PATH_LEN); + + // refresh g_vgs_info->group_num from dssserver if using dsscmd with -i + uint32 group_num = g_vgs_info->group_num + 1; + bool32 check_ret = CM_FALSE; + CM_RETURN_IFERR(dss_check_dup_vg(g_vgs_info, group_num, &check_ret)); + if (check_ret) { + LOG_DEBUG_ERR("[VOL][CV] more than one volume group name or more than one entry-paths"); + return CM_ERROR; + } } status = dss_set_vg_ctrl(vg_name, volume_name, vg_item, inst_cfg, size); @@ -301,7 +316,7 @@ status_t dss_create_vg(const char *vg_name, const char *volume_name, dss_config_ return CM_SUCCESS; } -static dss_vg_info_item_t* dss_find_vg_item_inner(dss_vg_info_t *vg_info, const char *vg_name, bool32 *is_first_vg) +static dss_vg_info_item_t *dss_find_vg_item_inner(dss_vg_info_t *vg_info, const char *vg_name, bool32 *is_first_vg) { *is_first_vg = CM_FALSE; for (uint32_t i = 0; i < vg_info->group_num; i++) { @@ -316,7 +331,7 @@ static dss_vg_info_item_t* dss_find_vg_item_inner(dss_vg_info_t *vg_info, const } static status_t dss_modify_volume_head( - dss_vg_info_item_t* vg_item, const char *vol_path, uint32 id, volume_modify_type_e type) + dss_vg_info_item_t *vg_item, const char *vol_path, uint32 id, volume_modify_type_e type) { #ifndef WIN32 char buf[DSS_DISK_UNIT_SIZE] __attribute__((__aligned__(DSS_ALIGN_SIZE))); @@ -340,7 +355,7 @@ static status_t dss_modify_volume_head( vol_head->software_version = 0; break; case VOLUME_MODIFY_REPLACE: - case VOLUME_MODIFY_ROLLBACK: + case VOLUME_MODIFY_ROLLBACK: break; default: LOG_DEBUG_ERR("Invalid volume modify type: %u.", type); @@ -389,7 +404,7 @@ static status_t dss_add_volume_inner(dss_vg_info_item_t *vg_item, const char *vo CM_RETURN_IFERR_EX(dss_cmp_volume_head(vg_item, vol_path, id), dss_close_volume(&vg_item->volume_handle[id])); uint64 vol_size = dss_get_volume_size(&vg_item->volume_handle[id]); dss_close_volume(&vg_item->volume_handle[id]); - if (vol_size == DSS_INVALID_64) { + if (vol_size == DSS_INVALID_64) { LOG_DEBUG_ERR("Failed to get volume size when add volume:%s.", vol_path); return CM_ERROR; } @@ -424,7 +439,7 @@ static status_t dss_add_volume_inner(dss_vg_info_item_t *vg_item, const char *vo return CM_SUCCESS; } -static status_t dss_remove_volume_inner(dss_vg_info_item_t* vg_item, const char *vol_path) +static status_t dss_remove_volume_inner(dss_vg_info_item_t *vg_item, const char *vol_path) { LOG_RUN_INF("Begin to remove volume, vg_name is %s, vol_path is %s.", vg_item->vg_name, vol_path); uint32 id; @@ -503,7 +518,7 @@ static status_t dss_replace_prepare_new_volume( return CM_SUCCESS; } -static status_t dss_replace_volume_to_disk(dss_vg_info_item_t* vg_item, const char *old_vol, const char *new_vol, +static status_t dss_replace_volume_to_disk(dss_vg_info_item_t *vg_item, const char *old_vol, const char *new_vol, uint32 id, uint64 new_size, dss_config_t *inst_cfg) { /* @@ -589,7 +604,7 @@ static status_t dss_replace_volume_inner( dss_volume_t new_volume; status_t ret = dss_open_volume(new_vol, NULL, DSS_CLI_OPEN_FLAG, &new_volume); DSS_RETURN_IFERR2(ret, LOG_DEBUG_ERR("Open volume %s failed.", new_vol)); - ret = dss_replace_prepare_new_volume(vg_item, &new_volume, id, &new_size); + ret = dss_replace_prepare_new_volume(vg_item, &new_volume, id, &new_size); dss_close_volume(&new_volume); if (ret != CM_SUCCESS) { LOG_RUN_ERR("Failed to prepare new volume, vg_name is %s, new_vol is %s.", vg_item->vg_name, new_vol); diff --git a/src/common/dss_meta_buf.c b/src/common/dss_meta_buf.c index 025de27..4427051 100644 --- a/src/common/dss_meta_buf.c +++ b/src/common/dss_meta_buf.c @@ -723,7 +723,6 @@ static status_t dss_add_buffer_cache(dss_session_t *session, dss_vg_info_item_t block_ctrl = DSS_GET_BLOCK_CTRL_FROM_META(meta_addr); block_id_tmp = ((dss_common_block_t *)meta_addr)->id; - block_ctrl->type = type; if ((block_ctrl->hash == hash) && (dss_buffer_cache_key_compare(&block_id_tmp, &add_block_id) == CM_TRUE)) { dss_unlock_shm_meta_bucket(session, &bucket->enque_lock); if (((dss_common_block_t *)meta_addr)->type != type) { @@ -1194,7 +1193,7 @@ void dss_recycle_meta(dss_session_t *session, dss_bg_task_info_t *bg_task_info, LOG_DEBUG_INF("try recycle meta, trigger_enable:%u", (uint32)trigger_enable); // do recycle meta for vg one by one - for (uint32_t i = bg_task_info->vg_id_beg; i < bg_task_info->vg_id_end; i++) { + for (uint32_t i = bg_task_info->my_task_id; i < g_vgs_info->group_num; i += bg_task_info->task_num_max) { dss_recycle_meta_by_vg(session, &g_vgs_info->volume_group[i], recycle_meta_args, trigger_enable); } @@ -1213,7 +1212,7 @@ void dss_buffer_recycle_disable(dss_block_ctrl_t *block_ctrl, bool8 recycle_disa void dss_set_recycle_meta_args_to_vg(dss_bg_task_info_t *bg_task_info) { // do recycle meta for vg one by one - for (uint32_t i = bg_task_info->vg_id_beg; i < bg_task_info->vg_id_end; i++) { + for (uint32_t i = bg_task_info->my_task_id; i < g_vgs_info->group_num; i += bg_task_info->task_num_max) { g_vgs_info->volume_group[i].recycle_meta_desc.task_args = bg_task_info->task_args; } } diff --git a/src/common/dss_session.c b/src/common/dss_session.c index 87642ef..75ed21d 100644 --- a/src/common/dss_session.c +++ b/src/common/dss_session.c @@ -100,9 +100,7 @@ uint32 dss_get_udssession_startid(void) { dss_config_t *inst_cfg = dss_get_inst_cfg(); uint32 start_sid = (uint32)DSS_BACKGROUND_TASK_NUM; - if (inst_cfg->params.nodes_list.inst_cnt > 1) { - start_sid = start_sid + inst_cfg->params.channel_num + inst_cfg->params.work_thread_cnt; - } + start_sid = start_sid + inst_cfg->params.channel_num + inst_cfg->params.work_thread_cnt; return start_sid; } diff --git a/src/common/dss_session.h b/src/common/dss_session.h index ad8b007..e31c7c6 100644 --- a/src/common/dss_session.h +++ b/src/common/dss_session.h @@ -67,8 +67,6 @@ typedef enum st_dss_background_task_type { typedef struct st_dss_bg_task_info { uint32 task_num_max; uint32 my_task_id; - uint32 vg_id_beg; - uint32 vg_id_end; void *task_args; } dss_bg_task_info_t; diff --git a/src/common/persist/dss_ctrl_def.h b/src/common/persist/dss_ctrl_def.h index b036284..168b8da 100644 --- a/src/common/persist/dss_ctrl_def.h +++ b/src/common/persist/dss_ctrl_def.h @@ -273,14 +273,14 @@ typedef struct st_dss_ctrl { char root[DSS_ROOT_FT_DISK_SIZE]; // dss_root_ft_block_t, 8KB union { dss_redo_ctrl_t redo_ctrl; - char redo_ctrl_data[DSS_DISK_UNIT_SIZE]; // 512 + char redo_ctrl_data[DSS_DISK_UNIT_SIZE]; // 512 }; - char reserve1[DSS_CTRL_RESERVE_SIZE1]; // 663K - char disk_latch[DSS_INIT_DISK_LATCH_SIZE]; // INIT DISK LATCH 32KB + char reserve1[DSS_CTRL_RESERVE_SIZE1]; // 663K + char disk_latch[DSS_INIT_DISK_LATCH_SIZE]; // INIT DISK LATCH 32KB union { struct { - char disk_lock[DSS_LOCK_SHARE_DISK_SIZE]; // share disk lock, 32KB + 512, align with 8K - char reserve4[DSS_CTRL_RESERVE_SIZE4]; // 512 + char disk_lock[DSS_LOCK_SHARE_DISK_SIZE]; // share disk lock, 32KB + 512, align with 8K + char reserve4[DSS_CTRL_RESERVE_SIZE4]; // 512 }; struct { char reserve3[DSS_CTRL_RESERVE_SIZE3]; // 32KB @@ -344,6 +344,18 @@ typedef struct st_dss_log_file_ctrl { uint64 lsn; } dss_log_file_ctrl_t; +typedef struct st_dss_share_vg_item_t { + dss_shared_latch_t vg_latch; + shm_hashmap_t buffer_cache; + uint32 objectid; + uint32 id; + uint32 all_vg_item_cnt; // only vg_item 0 will be setted, for api add vg item info dyn + char vg_name[DSS_MAX_NAME_LEN]; // added for syn to api + char entry_path[DSS_MAX_VOLUME_PATH_LEN]; // the manger volume path, dded for syn to api + char reserv[280]; // align 512 + dss_ctrl_t dss_ctrl; +} dss_share_vg_item_t; + typedef struct st_dss_vg_info_item_t { uint32 id; char vg_name[DSS_MAX_NAME_LEN]; @@ -367,11 +379,15 @@ typedef struct st_dss_vg_info_item_t { dss_block_ctrl_task_desc_t recycle_meta_desc; // for recycle meta uint32 objectid; uint32 space_alarm; + dss_share_vg_item_t *share_vg_item; } dss_vg_info_item_t; typedef struct st_dss_vg_info_t { dss_vg_info_item_t volume_group[DSS_MAX_VOLUME_GROUP_NUM]; uint32_t group_num; + uint32 dest_vg_num; // may more than group_num because added by dyn + uint32 inited_vg_num; // may more than group_num becasue added by dyn + struct stat cfg_stat; } dss_vg_info_t; typedef struct st_dss_vol_handles_t { @@ -388,14 +404,5 @@ typedef struct st_dss_vg_conf_t { char entry_path[DSS_MAX_VOLUME_PATH_LEN]; // the manager volume path } dss_vg_conf_t; -typedef struct st_dss_share_vg_item_t { - dss_shared_latch_t vg_latch; - shm_hashmap_t buffer_cache; - uint32 objectid; - uint32 id; - char reserve[412]; // align 512 - dss_ctrl_t dss_ctrl; -} dss_share_vg_item_t; - #pragma pack() #endif // __DSS_CTRL_DEF_H__ diff --git a/src/common/persist/dss_diskgroup.c b/src/common/persist/dss_diskgroup.c index 78b54b1..3da855d 100644 --- a/src/common/persist/dss_diskgroup.c +++ b/src/common/persist/dss_diskgroup.c @@ -32,16 +32,18 @@ #include "cm_utils.h" #include "dss_io_fence.h" #include "dss_open_file.h" -#include "dss_diskgroup.h" - #ifndef WIN32 #include #endif +#include +#include "dss_diskgroup.h" #include "dss_meta_buf.h" #include "dss_fs_aux.h" #include "dss_syn_meta.h" #include "dss_thv.h" +#include "cm_memory.h" + #ifdef __cplusplus extern "C" { #endif @@ -107,6 +109,7 @@ void dss_set_master_id(uint32 id) void dss_set_server_flag(void) { g_is_dss_server = DSS_TRUE; + CM_MFENCE; } int32 dss_get_server_status_flag(void) @@ -169,7 +172,6 @@ status_t dss_read_vg_config_file(const char *file_name, char *buf, uint32 *buf_l return status; } -status_t dss_parse_vg_config(dss_vg_info_t *config, char *buf, uint32 buf_len); status_t dss_load_vg_conf_inner(dss_vg_info_t *vgs_info, const dss_config_t *inst_cfg) { char vg_config_path[DSS_FILE_PATH_MAX_LENGTH]; @@ -186,34 +188,60 @@ status_t dss_load_vg_conf_inner(dss_vg_info_t *vgs_info, const dss_config_t *ins return status; } - status = dss_parse_vg_config(vgs_info, file_buf, len); + status = dss_parse_vg_config(vgs_info, file_buf, len, CM_FALSE); return status; } -status_t dss_load_vg_conf_info(dss_vg_info_t **vgs, const dss_config_t *inst_cfg) +void dss_refresh_vg_info_update_vg_num(dss_vg_info_t *vgs_info) +{ + LOG_RUN_INF( + "refresh vg with new item(s), old vg num [%u], cur_vg_num [%u]", vgs_info->group_num, vgs_info->dest_vg_num); + // update local vg info + uint32 old_vg_num = vgs_info->group_num; + vgs_info->group_num = vgs_info->dest_vg_num; + + // update this for api to syn + for (uint32 i = 0; i < old_vg_num; i++) { + vgs_info->volume_group[i].share_vg_item->all_vg_item_cnt = vgs_info->group_num; + LOG_RUN_INF("refresh vg update vg[%d] all_vg_item_cnt [%u]", i, + vgs_info->volume_group[i].share_vg_item->all_vg_item_cnt); + } +} + +status_t dss_init_vg_info() { - dss_vg_info_t *vgs_info = dss_malloc_vg_info(); + dss_vg_info_t *vgs_info = dss_get_vg_info_ptr(); bool32 result = (bool32)(vgs_info != NULL); DSS_RETURN_IF_FALSE2(result, DSS_THROW_ERROR(ERR_ALLOC_MEMORY, sizeof(dss_vg_info_t), "dss_load_vg_conf_info")); errno_t errcode = memset_s(vgs_info, sizeof(dss_vg_info_t), 0, sizeof(dss_vg_info_t)); result = (bool32)(errcode == EOK); - DSS_RETURN_IF_FALSE3(result, DSS_FREE_POINT(vgs_info), CM_THROW_ERROR(ERR_SYSTEM_CALL, errcode)); + DSS_RETURN_IF_FALSE3(result, dss_free_vg_info(), CM_THROW_ERROR(ERR_SYSTEM_CALL, errcode)); - status_t status = dss_load_vg_conf_inner(vgs_info, inst_cfg); - if (status != CM_SUCCESS) { - dss_free_vg_info(); - return CM_ERROR; - } - - for (uint32 i = 0; i < vgs_info->group_num; i++) { + for (uint32 i = 0; i < DSS_MAX_VOLUME_GROUP_NUM; i++) { for (size_t j = 0; j < DSS_MAX_VOLUMES; j++) { vgs_info->volume_group[i].id = i; vgs_info->volume_group[i].volume_handle[j].handle = DSS_INVALID_HANDLE; vgs_info->volume_group[i].volume_handle[j].unaligned_handle = DSS_INVALID_HANDLE; } } - *vgs = vgs_info; + + return CM_SUCCESS; +} + +status_t dss_load_vg_conf_info(const dss_config_t *inst_cfg) +{ + status_t status = dss_init_vg_info(); + if (status != CM_SUCCESS) { + return CM_ERROR; + } + dss_vg_info_t *vgs_info = dss_get_vg_info_ptr(); + status = dss_load_vg_conf_inner(vgs_info, inst_cfg); + if (status != CM_SUCCESS) { + dss_free_vg_info(); + return CM_ERROR; + } + return CM_SUCCESS; } @@ -223,6 +251,24 @@ void dss_free_vg_info() DSS_FREE_POINT(g_vgs_info) } +dss_vg_info_t *dss_get_vg_info_ptr() +{ + if (g_vgs_info == NULL) { + dss_vg_info_t *new_vg_info = cm_malloc(sizeof(dss_vg_info_t)); + if (new_vg_info == NULL) { + LOG_DEBUG_ERR("Failed to malloc space for g_vgs_info"); + return NULL; + } + errno_t ret = memset_s(new_vg_info, sizeof(dss_vg_info_t), 0, sizeof(dss_vg_info_t)); + if (ret != CM_SUCCESS) { + LOG_DEBUG_ERR("Failed to memset for g_vgs_info"); + return NULL; + } + g_vgs_info = new_vg_info; + } + return g_vgs_info; +} + dss_vg_info_item_t *dss_find_vg_item(const char *vg_name) { for (uint32_t i = 0; i < g_vgs_info->group_num; i++) { @@ -384,25 +430,20 @@ status_t dss_load_vg_info_and_recover_core(uint32 i, bool8 need_recovery) return CM_SUCCESS; } -status_t dss_load_vg_info_and_recover(bool8 need_recovery) +// load dss ctrl from disk and recover when start, if load from remote ,just check +status_t dss_load_vg_info_and_recover_with_range(uint32 vg_beg, uint32 vg_end, bool8 need_recovery) { - for (uint32_t i = 0; i < g_vgs_info->group_num; i++) { + for (uint32_t i = vg_beg; i < vg_end; i++) { status_t status = dss_load_vg_info_and_recover_core(i, need_recovery); if (status != CM_SUCCESS) { LOG_RUN_ERR("DSS instance failed to load vg:%s!", g_vgs_info->volume_group[i].vg_name); return status; } + LOG_RUN_INF("load vg and recover item(s), vg [%u], vg_name [%s]", i, g_vgs_info->volume_group[i].vg_name); } return CM_SUCCESS; } -static void dss_free_shm_hashmap_memory(uint32 num) -{ - for (uint32 i = 0; i <= num; i++) { - shm_hashmap_destroy(g_vgs_info->volume_group[i].buffer_cache, i); - } -} - status_t dss_alloc_vg_item_redo_log_buf(dss_vg_info_item_t *vg_item) { LOG_RUN_INF("Begin to alloc redo log buf of vg %s.", vg_item->vg_name); @@ -419,7 +460,7 @@ status_t dss_alloc_vg_item_redo_log_buf(dss_vg_info_item_t *vg_item) return CM_SUCCESS; } -static dss_share_vg_item_t *dss_get_vg_item_by_id(bool32 is_server, uint32 id) +dss_share_vg_item_t *dss_get_vg_item_by_id(bool32 is_server, uint32 id) { dss_share_vg_item_t *vg_item = NULL; if (is_server) { @@ -441,56 +482,6 @@ static dss_share_vg_item_t *dss_get_vg_item_by_id(bool32 is_server, uint32 id) } return NULL; } -status_t dss_get_vg_info() -{ - bool32 is_server = dss_is_server(); - dss_config_t *inst_cfg = dss_get_inst_cfg(); - status_t status = dss_load_vg_conf_info(&g_vgs_info, inst_cfg); - DSS_RETURN_IF_ERROR(status); - for (uint32 i = 0; i < g_vgs_info->group_num; i++) { - dss_share_vg_item_t *vg_item = dss_get_vg_item_by_id(is_server, i); - if (vg_item == NULL) { - LOG_RUN_ERR("Failed to get vg_item %u from shm!", i); - DSS_THROW_ERROR(ERR_DSS_GA_INIT, "failed to get vg_item %u from shm!", i); - return CM_ERROR; - } - g_vgs_info->volume_group[i].objectid = vg_item->objectid; - g_vgs_info->volume_group[i].buffer_cache = &vg_item->buffer_cache; - g_vgs_info->volume_group[i].dss_ctrl = &vg_item->dss_ctrl; - g_vgs_info->volume_group[i].vg_latch = &vg_item->vg_latch; - vg_item->id = g_vgs_info->volume_group[i].id; - if (!is_server) { - continue; - } - g_vgs_info->volume_group[i].stack.buff = (char *)cm_malloc_align(DSS_ALIGN_SIZE, DSS_MAX_STACK_BUF_SIZE); - bool32 result = (bool32)(g_vgs_info->volume_group[i].stack.buff != NULL); - DSS_RETURN_IF_FALSE3(result, - LOG_DEBUG_ERR("malloc stack failed, align size:%u, size:%u.", DSS_ALIGN_SIZE, DSS_MAX_STACK_BUF_SIZE), - DSS_THROW_ERROR(ERR_ALLOC_MEMORY, DSS_MAX_STACK_BUF_SIZE, "volume group stack buff")); - g_vgs_info->volume_group[i].stack.size = DSS_MAX_STACK_BUF_SIZE; - int32 ret = shm_hashmap_init(&vg_item->buffer_cache, i, dss_buffer_cache_key_compare); - if (ret != CM_SUCCESS) { - if (i != 0) { - dss_free_shm_hashmap_memory(i - 1); - } - DSS_FREE_POINT(g_vgs_info->volume_group[i].stack.buff); - LOG_RUN_ERR("DSS instance failed to initialize buffer cache, %d!", ret); - DSS_THROW_ERROR(ERR_DSS_GA_INIT, "failed to init hashmap of vg %s", g_vgs_info->volume_group[i].vg_name); - return CM_ERROR; - } - cm_bilist_init(&g_vgs_info->volume_group[i].open_file_list); - cm_bilist_init(&g_vgs_info->volume_group[i].syn_meta_desc.bilist); - status = dss_alloc_vg_item_redo_log_buf(&g_vgs_info->volume_group[i]); - if (status != CM_SUCCESS) { - dss_free_shm_hashmap_memory(i); - DSS_FREE_POINT(g_vgs_info->volume_group[i].stack.buff); - return CM_ERROR; - } - g_vgs_info->volume_group[i].space_alarm = DSS_VG_SPACE_ALARM_INIT; - } - LOG_RUN_INF("DSS succeed to init vgs in memory."); - return status; -} status_t dss_check_entry_path(char *entry_path1, char *entry_path2, bool32 *result) { @@ -529,7 +520,7 @@ status_t dss_check_dup_vg(dss_vg_info_t *config, uint32 vg_no, bool32 *result) return CM_SUCCESS; } -status_t dss_parse_vg_config(dss_vg_info_t *config, char *buf, uint32 buf_len) +status_t dss_parse_vg_config(dss_vg_info_t *config, char *buf, uint32 buf_len, bool32 is_refresh) { uint32 line_no; text_t text, line, comment, name, value; @@ -577,8 +568,27 @@ status_t dss_parse_vg_config(dss_vg_info_t *config, char *buf, uint32 buf_len) return CM_ERROR; } - CM_RETURN_IFERR(cm_text2str(&name, config->volume_group[vg_no].vg_name, DSS_MAX_NAME_LEN)); - CM_RETURN_IFERR(cm_text2str(&value, config->volume_group[vg_no].entry_path, DSS_MAX_VOLUME_PATH_LEN)); + if (is_refresh && (vg_no < config->dest_vg_num)) { + if (cm_compare_text_str(&name, config->volume_group[vg_no].vg_name) != 0) { + DSS_THROW_ERROR(ERR_DSS_CONFIG_LOAD, "volume group name [%s] not the same as before [%s].", name.str, + config->volume_group[vg_no].vg_name); + return CM_ERROR; + } + if (cm_compare_text_str(&value, config->volume_group[vg_no].entry_path) != 0) { + DSS_THROW_ERROR(ERR_DSS_CONFIG_LOAD, "volume entry_path name [%s] not the same as before [%s].", + value.str, config->volume_group[vg_no].entry_path); + return CM_ERROR; + } + } + + // when not change the exist cfg, record only find vg after last find + if (vg_no >= config->dest_vg_num) { + CM_RETURN_IFERR(cm_text2str(&name, config->volume_group[vg_no].vg_name, DSS_MAX_NAME_LEN)); + CM_RETURN_IFERR(cm_text2str(&value, config->volume_group[vg_no].entry_path, DSS_MAX_VOLUME_PATH_LEN)); + LOG_RUN_INF("parsed vg id[%d] vg_name[%s] entry_path[%s].", vg_no, config->volume_group[vg_no].vg_name, + config->volume_group[vg_no].entry_path); + } + vg_no++; CM_RETURN_IFERR(dss_check_dup_vg(config, vg_no, &check_ret)); if (check_ret) { @@ -589,7 +599,14 @@ status_t dss_parse_vg_config(dss_vg_info_t *config, char *buf, uint32 buf_len) comment.str = text.str; comment.len = 0; } - config->group_num = vg_no; + if (!is_refresh) { + config->group_num = vg_no; + } + + if (vg_no > config->dest_vg_num) { + config->dest_vg_num = vg_no; + } + return CM_SUCCESS; } @@ -2163,7 +2180,7 @@ status_t dss_read_volume_4standby(const char *vg_name, uint32 volume_id, int64 o bool32 dss_meta_syn(dss_session_t *session, dss_bg_task_info_t *bg_task_info) { bool32 finish = CM_TRUE; - for (uint32_t i = bg_task_info->vg_id_beg; i < bg_task_info->vg_id_end; i++) { + for (uint32_t i = bg_task_info->my_task_id; i < g_vgs_info->group_num; i += bg_task_info->task_num_max) { bool32 cur_finish = dss_syn_buffer_cache(session, &g_vgs_info->volume_group[i]); if (!cur_finish && !finish) { finish = CM_FALSE; diff --git a/src/common/persist/dss_diskgroup.h b/src/common/persist/dss_diskgroup.h index adec014..f627470 100644 --- a/src/common/persist/dss_diskgroup.h +++ b/src/common/persist/dss_diskgroup.h @@ -81,16 +81,22 @@ typedef struct st_dss_allvg_vlm_space_t { typedef handle_t dss_directory_t; // dss_dir_t +status_t dss_check_dup_vg(dss_vg_info_t *config, uint32 vg_no, bool32 *result); // create vg only use in tool status_t dss_create_vg(const char *vg_name, const char *volume_name, dss_config_t *inst_cfg, uint32 size); -status_t dss_load_vg_conf_info(dss_vg_info_t **vgs, const dss_config_t *inst_cfg); +status_t dss_read_vg_config_file(const char *file_name, char *buf, uint32 *buf_len, bool32 read_only); +status_t dss_parse_vg_config(dss_vg_info_t *config, char *buf, uint32 buf_len, bool32 is_refresh); +status_t dss_load_vg_conf_info(const dss_config_t *inst_cfg); +dss_vg_info_t *dss_get_vg_info_ptr(); void dss_free_vg_info(); dss_vg_info_item_t *dss_find_vg_item(const char *vg_name); dss_vg_info_item_t *dss_find_vg_item_by_id(uint32 vg_id); +dss_share_vg_item_t *dss_get_vg_item_by_id(bool32 is_server, uint32 id); -status_t dss_get_vg_info(); -status_t dss_load_vg_info_and_recover(bool8 need_recovery); +status_t dss_init_vg_info(); status_t dss_load_vg_ctrl(dss_vg_info_item_t *vg_item, bool32 is_lock); +void dss_refresh_vg_info_update_vg_num(dss_vg_info_t *vgs_info); +status_t dss_load_vg_info_and_recover_with_range(uint32 vg_beg, uint32 vg_end, bool8 need_recovery); status_t dss_load_vg_ctrl_part(dss_vg_info_item_t *vg_item, int64 offset, void *buf, int32 size, bool32 *remote); status_t dss_check_refresh_core(dss_vg_info_item_t *vg_item); diff --git a/src/common/persist/dss_redo_recovery.c b/src/common/persist/dss_redo_recovery.c index 06cca57..84f1a88 100644 --- a/src/common/persist/dss_redo_recovery.c +++ b/src/common/persist/dss_redo_recovery.c @@ -497,3 +497,79 @@ status_t dss_recover_from_offset_inner(dss_session_t *session, dss_vg_info_item_ LOG_RUN_INF("[RECOVERY]Succeed to recovery."); return CM_SUCCESS; } + +status_t dss_check_vg_ctrl_valid(dss_vg_info_item_t *vg_item) +{ + dss_ctrl_t *dss_ctrl = vg_item->dss_ctrl; + if (!DSS_VG_IS_VALID(dss_ctrl)) { + DSS_RETURN_IFERR2(CM_ERROR, DSS_THROW_ERROR(ERR_DSS_VG_CHECK_NOT_INIT)); + } + return CM_SUCCESS; +} + +status_t dss_recover_from_offset(dss_session_t *session, dss_vg_info_item_t *vg_item) +{ + bool8 need_recovery = CM_FALSE; + /* 1、load offset batch 2、check batch valid 3、if batch valid, used 4、if batch invalid,just end */ + LOG_RUN_INF("[RECOVERY]Try to load log buf to recover"); + if (dss_load_log_buffer_from_offset(vg_item, &need_recovery) != CM_SUCCESS) { + return CM_ERROR; + } + if (need_recovery) { + char *log_buf = vg_item->log_file_ctrl.log_buf; + status_t status = dss_recover_from_offset_inner(session, vg_item, log_buf); + if (status != CM_SUCCESS) { + return CM_ERROR; + } + } + return CM_SUCCESS; +} + +status_t dss_recover_from_slot(dss_session_t *session, dss_vg_info_item_t *vg_item) +{ + bool8 need_recovery = CM_FALSE; + if (dss_load_log_buffer_from_slot(vg_item, &need_recovery) != CM_SUCCESS) { + return CM_ERROR; + } + if (need_recovery) { + char *log_buf = vg_item->log_file_ctrl.log_buf; + status_t status = dss_recover_from_slot_inner(session, vg_item, log_buf); + if (status != CM_SUCCESS) { + return CM_ERROR; + } + return status; + } + return CM_SUCCESS; +} + +status_t dss_recover_redo_log_with_range(dss_session_t *session, uint32 vg_beg, uint32 vg_end) +{ + status_t status; + for (uint32 i = vg_beg; i < vg_end; i++) { + dss_vg_info_item_t *vg_item = &g_vgs_info->volume_group[i]; + if (dss_check_vg_ctrl_valid(vg_item) != CM_SUCCESS) { + LOG_RUN_ERR("[RECOVERY]Failed to check valid of vg %s.", vg_item->vg_name); + return CM_ERROR; + } + uint32 software_version = dss_get_software_version(&vg_item->dss_ctrl->vg_info); + if (software_version < DSS_SOFTWARE_VERSION_2) { + status = dss_recover_from_slot(session, vg_item); + if (status != CM_SUCCESS) { + LOG_RUN_ERR("[RECOVERY]Failed to recover from vg %s.", vg_item->vg_name); + return CM_ERROR; + } + } else { + status = dss_load_redo_ctrl(vg_item); + if (status != CM_SUCCESS) { + LOG_RUN_ERR("[RECOVERY]Failed to load redo ctrl of vg %s.", vg_item->vg_name); + return CM_ERROR; + } + status = dss_recover_from_offset(session, vg_item); + if (status != CM_SUCCESS) { + LOG_RUN_ERR("[RECOVERY]Failed to recover from vg %s.", vg_item->vg_name); + return CM_ERROR; + } + } + } + return status; +} diff --git a/src/common/persist/dss_redo_recovery.h b/src/common/persist/dss_redo_recovery.h index 0246b41..c563218 100644 --- a/src/common/persist/dss_redo_recovery.h +++ b/src/common/persist/dss_redo_recovery.h @@ -39,6 +39,7 @@ status_t dss_recover_from_slot_inner(dss_session_t *session, dss_vg_info_item_t status_t dss_load_log_buffer_from_offset(dss_vg_info_item_t *vg_item, bool8 *need_recovery); status_t dss_load_log_buffer_from_slot(dss_vg_info_item_t *vg_item, bool8 *need_recovery); status_t dss_read_redolog_from_disk(dss_vg_info_item_t *vg_item, uint32 volume_id, int64 offset, char *buf, int32 size); +status_t dss_recover_redo_log_with_range(dss_session_t *session, uint32 vg_beg, uint32 vg_end); #ifdef __cplusplus } diff --git a/src/common_api/dss_api_impl.c b/src/common_api/dss_api_impl.c index 38067ef..021582c 100644 --- a/src/common_api/dss_api_impl.c +++ b/src/common_api/dss_api_impl.c @@ -57,6 +57,98 @@ typedef struct str_files_rw_ctx { int64 offset; } files_rw_ctx_t; +static status_t dss_syn_vg_item_from_server(uint32 vg_id_vg, uint32 vg_id_end) +{ + for (uint32 i = vg_id_vg; i < vg_id_end; i++) { + dss_share_vg_item_t *share_vg_item = dss_get_vg_item_by_id(CM_FALSE, i); + if (share_vg_item == NULL) { + LOG_RUN_ERR("Failed to get vg item %u from shm!", i); + DSS_THROW_ERROR(ERR_DSS_GA_INIT, "failed to get vg_item %u from shm!", i); + return CM_ERROR; + } + + // syn the info from server by shm + (void)memcpy_s(g_vgs_info->volume_group[i].vg_name, DSS_MAX_NAME_LEN, share_vg_item->vg_name, DSS_MAX_NAME_LEN); + (void)memcpy_s(g_vgs_info->volume_group[i].entry_path, DSS_MAX_VOLUME_PATH_LEN, share_vg_item->entry_path, + DSS_MAX_VOLUME_PATH_LEN); + g_vgs_info->volume_group[i].share_vg_item = share_vg_item; + g_vgs_info->volume_group[i].objectid = share_vg_item->objectid; + g_vgs_info->volume_group[i].buffer_cache = &share_vg_item->buffer_cache; + g_vgs_info->volume_group[i].dss_ctrl = &share_vg_item->dss_ctrl; + g_vgs_info->volume_group[i].vg_latch = &share_vg_item->vg_latch; + g_vgs_info->volume_group[i].id = share_vg_item->id; + + (void)cm_attach_shm(SHM_TYPE_HASH, g_vgs_info->volume_group[i].buffer_cache->shm_id, 0, CM_SHM_ATTACH_RW); + + LOG_RUN_INF("success syn vg item id [%u] vg_name [%s] entry_path[%s] from server", i, + g_vgs_info->volume_group[i].vg_name, g_vgs_info->volume_group[i].entry_path); + } + return CM_SUCCESS; +} + +static status_t dss_init_vg_item_from_server() +{ + status_t status = dss_init_vg_info(); + if (status != CM_SUCCESS) { + LOG_RUN_ERR("Failed to init vg info"); + return CM_ERROR; + } + + // get the first vg info by shm from server + dss_share_vg_item_t *share_vg_item = dss_get_vg_item_by_id(CM_FALSE, 0); + if (share_vg_item == NULL) { + LOG_RUN_ERR("Failed to get vg_item %u from shm!", 0); + DSS_THROW_ERROR(ERR_DSS_GA_INIT, "failed to get vg_item %u from shm!", 0); + return CM_ERROR; + } + + status = dss_syn_vg_item_from_server(0, share_vg_item->all_vg_item_cnt); + if (status != CM_SUCCESS) { + LOG_RUN_ERR("Failed to refresh vg item"); + return CM_ERROR; + } + g_vgs_info->group_num = share_vg_item->all_vg_item_cnt; + return CM_SUCCESS; +} + +static status_t dss_refresh_vg_item_from_server() +{ + status_t status = + dss_syn_vg_item_from_server(g_vgs_info->group_num, g_vgs_info->volume_group[0].share_vg_item->all_vg_item_cnt); + if (status != CM_SUCCESS) { + LOG_RUN_ERR("Failed to refresh vg_item"); + return CM_ERROR; + } + + if (g_vgs_info->group_num != g_vgs_info->volume_group[0].share_vg_item->all_vg_item_cnt) { + g_vgs_info->group_num = g_vgs_info->volume_group[0].share_vg_item->all_vg_item_cnt; + LOG_RUN_INF("success syn vg item to server to:%d", g_vgs_info->group_num); + } + + return CM_SUCCESS; +} + +static dss_vg_info_item_t *dss_find_vg_item_with_refresh(const char *vg_name) +{ + dss_env_t *dss_env = dss_get_env(); + if (!dss_env->initialized) { + return NULL; + } + // may server add vg items + if (g_vgs_info->group_num < g_vgs_info->volume_group[0].share_vg_item->all_vg_item_cnt) { + dss_latch_x(&dss_env->latch); + if (g_vgs_info->group_num < g_vgs_info->volume_group[0].share_vg_item->all_vg_item_cnt) { + status_t status = dss_refresh_vg_item_from_server(); + if (status != CM_SUCCESS) { + dss_unlatch(&dss_env->latch); + return NULL; + } + } + dss_unlatch(&dss_env->latch); + } + return dss_find_vg_item(vg_name); +} + status_t dss_load_ctrl_sync(dss_conn_t *conn, const char *vg_name, uint32 index) { CM_RETURN_IFERR(dss_check_name(vg_name)); @@ -456,92 +548,6 @@ status_t dss_apply_refresh_file_table(dss_conn_t *conn, dss_dir_t *dir) return status; } -static inline void dss_init_conn(dss_conn_t *conn) -{ - conn->flag = CM_FALSE; - conn->cli_vg_handles = NULL; - conn->session = NULL; -} - -status_t dss_alloc_conn(dss_conn_t **conn) -{ - dss_conn_t *_conn = (dss_conn_t *)cm_malloc_align(DSSAPI_BLOCK_SIZE, sizeof(dss_conn_t)); - if (_conn != NULL) { - dss_init_conn(_conn); - *conn = _conn; - return CM_SUCCESS; - } - - return CM_ERROR; -} - -void dss_free_conn(dss_conn_t *conn) -{ - DSS_FREE_POINT(conn); - return; -} - -static status_t dss_check_url_format(const char *url, text_t *uds) -{ - uint32 len = (uint32)strlen(url); - if (len <= uds->len) { - return CM_ERROR; - } - - return (cm_strcmpni(url, uds->str, uds->len) != 0) ? CM_ERROR : CM_SUCCESS; -} - -status_t dss_connect(const char *server_locator, dss_conn_opt_t *options, dss_conn_t *conn) -{ - if (server_locator == NULL) { - DSS_THROW_ERROR(ERR_DSS_UDS_INVALID_URL, "NULL", 0); - return CM_ERROR; - } - - if ((conn->flag == CM_TRUE) && (conn->pipe.link.uds.closed == CM_FALSE)) { - return CM_SUCCESS; - } - - conn->flag = CM_FALSE; - text_t uds = {"UDS:", 4}; - if (dss_check_url_format(server_locator, &uds) != CM_SUCCESS) { - DSS_THROW_ERROR(ERR_DSS_UDS_INVALID_URL, server_locator, strlen(server_locator)); - return ERR_DSS_UDS_INVALID_URL; - } - conn->cli_vg_handles = NULL; - conn->pipe.options = 0; - int32 timeout = options != NULL ? options->timeout : g_dss_uds_conn_timeout; - conn->pipe.connect_timeout = timeout < 0 ? DSS_UDS_CONNECT_TIMEOUT : timeout; - conn->pipe.socket_timeout = DSS_UDS_SOCKET_TIMEOUT; - conn->pipe.link.uds.sock = CS_INVALID_SOCKET; - conn->pipe.link.uds.closed = CM_TRUE; - conn->pipe.type = CS_TYPE_DOMAIN_SCOKET; - conn->session = NULL; - status_t ret = cs_connect_ex( - server_locator, &conn->pipe, NULL, (const char *)(server_locator + uds.len), (const char *)CM_NULL_TEXT.str); - if (ret != CM_SUCCESS) { - LOG_DEBUG_ERR("connect server failed, uds path:%s", server_locator); - return ret; - } - dss_init_packet(&conn->pack, conn->pipe.options); - - conn->flag = CM_TRUE; - - return CM_SUCCESS; -} - -void dss_disconnect(dss_conn_t *conn) -{ - dss_set_thv_run_ctx_item(DSS_THV_RUN_CTX_ITEM_SESSION, NULL); - if (conn->flag == CM_TRUE) { - cs_disconnect(&conn->pipe); - dss_free_packet_buffer(&conn->pack); - conn->flag = CM_FALSE; - } - - return; -} - status_t dss_init_vol_handle_sync(dss_conn_t *conn) { if (!conn->flag) { @@ -557,25 +563,13 @@ status_t dss_init_vol_handle_sync(dss_conn_t *conn) return CM_ERROR; } - status_t status; dss_cli_vg_handles_t *cli_vg_handles = (dss_cli_vg_handles_t *)(conn->cli_vg_handles); - - int cli_flags = DSS_CLI_OPEN_FLAG; - for (uint32 i = 0; i < g_vgs_info->group_num; i++) { + for (uint32 i = 0; i < DSS_MAX_VOLUME_GROUP_NUM; i++) { for (uint32 vid = 0; vid < DSS_MAX_VOLUMES; ++vid) { cli_vg_handles->vg_vols[i].volume_handle[vid].handle = DSS_INVALID_HANDLE; cli_vg_handles->vg_vols[i].volume_handle[vid].unaligned_handle = DSS_INVALID_HANDLE; cli_vg_handles->vg_vols[i].volume_handle[vid].id = vid; } - - status = dss_init_vol_handle(&g_vgs_info->volume_group[i], cli_flags, &cli_vg_handles->vg_vols[i]); - if (status != CM_SUCCESS) { - for (int32 j = (int32)(i - 1); j >= 0; j--) { - dss_destroy_vol_handle(&g_vgs_info->volume_group[j], &cli_vg_handles->vg_vols[j], DSS_MAX_VOLUMES); - } - DSS_FREE_POINT(conn->cli_vg_handles); - return status; - } } cli_vg_handles->group_num = g_vgs_info->group_num; @@ -678,8 +672,8 @@ status_t dss_cli_session_lock(dss_conn_t *conn, dss_session_t *session) conn->cli_info.connect_time); LOG_RUN_ERR("Failed to check session %u, session thread id is %u, connect_time is %llu, conn thread id is %u, " "connect_time is %llu", - session->id, session->cli_info.thread_id, session->cli_info.connect_time, conn->cli_info.thread_id, - conn->cli_info.connect_time); + session->id, session->cli_info.thread_id, session->cli_info.connect_time, conn->cli_info.thread_id, + conn->cli_info.connect_time); cm_spin_unlock(&session->shm_lock); LOG_DEBUG_INF("Succeed to unlock session %u shm lock", session->id); return CM_ERROR; @@ -734,7 +728,7 @@ status_t dss_remove_dir_impl(dss_conn_t *conn, const char *dir, bool32 recursive static dss_dir_t *dss_open_dir_impl_core(dss_conn_t *conn, dss_find_node_t *find_node) { - dss_vg_info_item_t *vg_item = dss_find_vg_item(find_node->vg_name); + dss_vg_info_item_t *vg_item = dss_find_vg_item_with_refresh(find_node->vg_name); if (vg_item == NULL) { LOG_RUN_ERR("Failed to find vg, %s.", find_node->vg_name); DSS_THROW_ERROR(ERR_DSS_VG_NOT_EXIST, find_node->vg_name); @@ -925,7 +919,7 @@ status_t dss_find_vg_by_file_path(const char *path, dss_vg_info_item_t **vg_item status_t status = dss_get_name_from_path(path, &beg_pos, vg_name); DSS_RETURN_IFERR2(status, LOG_DEBUG_ERR("Failed to get name from path:%s, status:%d.", path, status)); - *vg_item = dss_find_vg_item(vg_name); + *vg_item = dss_find_vg_item_with_refresh(vg_name); if (*vg_item == NULL) { LOG_DEBUG_ERR("Failed to find VG:%s.", vg_name); DSS_THROW_ERROR(ERR_DSS_VG_NOT_EXIST, vg_name); @@ -967,7 +961,7 @@ gft_node_t *dss_get_node_by_path_impl(dss_conn_t *conn, const char *path) if (dss_get_ftid_by_path_on_server(conn, path, &ftid, (char *)vg_name) != CM_SUCCESS) { return NULL; } - dss_vg_info_item_t *vg_item = dss_find_vg_item(vg_name); + dss_vg_info_item_t *vg_item = dss_find_vg_item_with_refresh(vg_name); if (vg_item == NULL) { LOG_DEBUG_ERR("Failed to find vg,vg name %s.", vg_name); DSS_THROW_ERROR(ERR_DSS_VG_NOT_EXIST, vg_name); @@ -999,9 +993,9 @@ status_t dss_init_file_context( return CM_SUCCESS; } -/* +/* 1 after extend success, will generate new linked list -context[file_run_ctx->files->group_num - 1] [0]->context[file_run_ctx->files->group_num - 1] +context[file_run_ctx->files->group_num - 1] [0]->context[file_run_ctx->files->group_num - 1] [1]->...->context[file_run_ctx->files->group_num - 1] [DSS_FILE_CONTEXT_PER_GROUP - 1] 2 insert new linked list head into the old linked list */ @@ -1090,7 +1084,7 @@ status_t dss_open_file_impl(dss_conn_t *conn, const char *file_path, int flag, i LOG_DEBUG_INF("dss begin to open file, file path:%s, flag:%d", file_path, flag); DSS_RETURN_IF_ERROR(dss_check_device_path(file_path)); DSS_RETURN_IF_ERROR(dss_open_file_on_server(conn, file_path, flag, &find_node)); - dss_vg_info_item_t *vg_item = dss_find_vg_item(find_node->vg_name); + dss_vg_info_item_t *vg_item = dss_find_vg_item_with_refresh(find_node->vg_name); if (vg_item == NULL) { LOG_RUN_ERR("Failed to find vg, vg name %s.", find_node->vg_name); DSS_THROW_ERROR(ERR_DSS_VG_NOT_EXIST, find_node->vg_name); @@ -1565,9 +1559,9 @@ status_t dss_read_write_file_core(dss_rw_param_t *param, void *buf, int32 size, volume.name_p, vol_offset, real_size, node->name, node->size, node->written_size); #if defined(_DEBUG) && !defined(OPENGAUSS) if (CM_STR_EQUAL(context->vg_item->vg_name, "dss_data") && !CM_STR_BEGIN_WITH(node->name, "ctrl")) { - LOG_DEBUG_INF("dss pwrite file %s, vol_offset:%lld, head:%u-%u", node->name, vol_offset, - *(uint16 *)((char*)buf + sizeof(uint32)), *(uint32 *)buf); - } + LOG_DEBUG_INF("dss pwrite file %s, vol_offset:%lld, head:%u-%u", node->name, vol_offset, + *(uint16 *)((char *)buf + sizeof(uint32)), *(uint32 *)buf); + } #endif status = dss_write_volume(&volume, (int64)vol_offset, buf, real_size); } @@ -2035,16 +2029,11 @@ status_t dss_init(uint32 max_open_files, char *home) #endif } CM_RETURN_IFERR(dss_init_shm(dss_env, home)); - status_t status = dss_get_vg_info(); + CM_RETURN_IFERR(dss_init_files(dss_env, max_open_files)); + status_t status = dss_init_vg_item_from_server(); if (status != CM_SUCCESS) { return dss_init_err_proc(dss_env, CM_TRUE, CM_TRUE, "Failed to get shared vg info", status); } - CM_RETURN_IFERR(dss_init_files(dss_env, max_open_files)); - - for (int32_t i = 0; i < (int32_t)g_vgs_info->group_num; i++) { - dss_vg_info_item_t *item = &g_vgs_info->volume_group[i]; - (void)cm_attach_shm(SHM_TYPE_HASH, item->buffer_cache->shm_id, 0, CM_SHM_ATTACH_RW); - } status = cm_create_thread(dss_heartbeat_entry, SIZE_K(512), NULL, &dss_env->thread_heartbeat); if (status != CM_SUCCESS) { @@ -2405,7 +2394,7 @@ status_t get_au_size_impl(dss_conn_t *conn, int handle, long long *au_size) status_t dss_compare_size_equal_impl(const char *vg_name, long long *au_size) { - dss_vg_info_item_t *vg_item = dss_find_vg_item(vg_name); + dss_vg_info_item_t *vg_item = dss_find_vg_item_with_refresh(vg_name); if (vg_name == NULL || vg_item == NULL) { dss_free_vg_info(); LOG_DEBUG_ERR("Failed to find vg info from config, vg name is null\n"); diff --git a/src/common_api/dss_api_impl.h b/src/common_api/dss_api_impl.h index c62b391..776c717 100644 --- a/src/common_api/dss_api_impl.h +++ b/src/common_api/dss_api_impl.h @@ -38,7 +38,7 @@ extern "C" { #endif -typedef struct st_dss_conn dss_conn_t; +typedef struct st_dss_conn dss_conn_t; typedef struct st_dss_conn_opt dss_conn_opt_t; typedef struct st_dss_rw_param { @@ -191,20 +191,17 @@ typedef struct st_dss_query_hotpatch_recv_info { dss_hp_info_view_t *hp_info_view; // Location of output buffer must be specified before decoding. } dss_query_hotpatch_recv_info_t; - #define DSSAPI_BLOCK_SIZE 512 #define DSS_HOME "DSS_HOME" #define SYS_HOME "HOME" #define DSS_DEFAULT_UDS_PATH "UDS:/tmp/.dss_unix_d_socket" -#define SESSION_LOCK_TIMEOUT 500 // tickets +#define SESSION_LOCK_TIMEOUT 500 // tickets status_t dss_load_ctrl_sync(dss_conn_t *conn, const char *vg_name, uint32 index); status_t dss_add_or_remove_volume(dss_conn_t *conn, const char *vg_name, const char *volume_name, uint8 cmd); status_t dss_kick_host_sync(dss_conn_t *conn, int64 kick_hostid); status_t dss_alloc_conn(dss_conn_t **conn); void dss_free_conn(dss_conn_t *conn); -status_t dss_connect(const char *server_locator, dss_conn_opt_t *options, dss_conn_t *conn); -void dss_disconnect(dss_conn_t *conn); // NOTE:just for dsscmd because not support many threads in one process. status_t dss_connect_ex(const char *server_locator, dss_conn_opt_t *options, dss_conn_t *conn); diff --git a/src/common_api/dss_cli_conn.c b/src/common_api/dss_cli_conn.c index cfcc1c2..17dde61 100644 --- a/src/common_api/dss_cli_conn.c +++ b/src/common_api/dss_cli_conn.c @@ -229,6 +229,92 @@ void dss_leave_api(dss_conn_t *conn, bool32 get_api_volume_error) } } +void dss_init_conn(dss_conn_t *conn) +{ + conn->flag = CM_FALSE; + conn->cli_vg_handles = NULL; + conn->session = NULL; +} + +status_t dss_alloc_conn(dss_conn_t **conn) +{ + dss_conn_t *_conn = (dss_conn_t *)cm_malloc_align(DSSAPI_BLOCK_SIZE, sizeof(dss_conn_t)); + if (_conn != NULL) { + dss_init_conn(_conn); + *conn = _conn; + return CM_SUCCESS; + } + + return CM_ERROR; +} + +void dss_free_conn(dss_conn_t *conn) +{ + DSS_FREE_POINT(conn); + return; +} + +static status_t dss_check_url_format(const char *url, text_t *uds) +{ + uint32 len = (uint32)strlen(url); + if (len <= uds->len) { + return CM_ERROR; + } + + return (cm_strcmpni(url, uds->str, uds->len) != 0) ? CM_ERROR : CM_SUCCESS; +} + +status_t dss_connect(const char *server_locator, dss_conn_opt_t *options, dss_conn_t *conn) +{ + if (server_locator == NULL) { + DSS_THROW_ERROR(ERR_DSS_UDS_INVALID_URL, "NULL", 0); + return CM_ERROR; + } + + if ((conn->flag == CM_TRUE) && (conn->pipe.link.uds.closed == CM_FALSE)) { + return CM_SUCCESS; + } + + conn->flag = CM_FALSE; + text_t uds = {"UDS:", 4}; + if (dss_check_url_format(server_locator, &uds) != CM_SUCCESS) { + DSS_THROW_ERROR(ERR_DSS_UDS_INVALID_URL, server_locator, strlen(server_locator)); + return ERR_DSS_UDS_INVALID_URL; + } + conn->cli_vg_handles = NULL; + conn->pipe.options = 0; + int32 timeout = options != NULL ? options->timeout : g_dss_uds_conn_timeout; + conn->pipe.connect_timeout = timeout < 0 ? DSS_UDS_CONNECT_TIMEOUT : timeout; + conn->pipe.socket_timeout = DSS_UDS_SOCKET_TIMEOUT; + conn->pipe.link.uds.sock = CS_INVALID_SOCKET; + conn->pipe.link.uds.closed = CM_TRUE; + conn->pipe.type = CS_TYPE_DOMAIN_SCOKET; + conn->session = NULL; + status_t ret = cs_connect_ex( + server_locator, &conn->pipe, NULL, (const char *)(server_locator + uds.len), (const char *)CM_NULL_TEXT.str); + if (ret != CM_SUCCESS) { + LOG_DEBUG_ERR("connect server failed, uds path:%s", server_locator); + return ret; + } + dss_init_packet(&conn->pack, conn->pipe.options); + + conn->flag = CM_TRUE; + + return CM_SUCCESS; +} + +void dss_disconnect(dss_conn_t *conn) +{ + dss_set_thv_run_ctx_item(DSS_THV_RUN_CTX_ITEM_SESSION, NULL); + if (conn->flag == CM_TRUE) { + cs_disconnect(&conn->pipe); + dss_free_packet_buffer(&conn->pack); + conn->flag = CM_FALSE; + } + + return; +} + #ifdef __cplusplus } #endif diff --git a/src/common_api/dss_cli_conn.h b/src/common_api/dss_cli_conn.h index a8b3e5f..79e830e 100644 --- a/src/common_api/dss_cli_conn.h +++ b/src/common_api/dss_cli_conn.h @@ -70,10 +70,12 @@ status_t dss_try_conn(dss_conn_opt_t *options, dss_conn_t *conn); void dss_clt_env_init(void); status_t dss_enter_api(dss_conn_t **conn); void dss_leave_api(dss_conn_t *conn, bool32 get_api_volume_error); +status_t dss_connect(const char *server_locator, dss_conn_opt_t *options, dss_conn_t *conn); +void dss_disconnect(dss_conn_t *conn); +void dss_init_conn(dss_conn_t *conn); #ifdef __cplusplus } #endif - -#endif // __DSS_CLI_CONN_H__ +#endif // __DSS_CLI_CONN_H__ diff --git a/src/service/dss_instance.c b/src/service/dss_instance.c index ef38ef1..2fe0e09 100644 --- a/src/service/dss_instance.c +++ b/src/service/dss_instance.c @@ -122,80 +122,196 @@ static status_t dss_init_thread(dss_instance_t *inst) return CM_SUCCESS; } -status_t dss_check_vg_ctrl_valid(dss_vg_info_item_t *vg_item) +static void dss_free_shm_hashmap_memory(uint32 num) { - dss_ctrl_t *dss_ctrl = vg_item->dss_ctrl; - if (!DSS_VG_IS_VALID(dss_ctrl)) { - DSS_RETURN_IFERR2(CM_ERROR, DSS_THROW_ERROR(ERR_DSS_VG_CHECK_NOT_INIT)); + for (uint32 i = 0; i < num && i < DSS_MAX_VOLUME_GROUP_NUM; i++) { + shm_hashmap_destroy(g_vgs_info->volume_group[i].buffer_cache, i); } - return CM_SUCCESS; } -status_t dss_recover_from_offset(dss_session_t *session, dss_vg_info_item_t *vg_item) +status_t dss_init_vg_info_by_server_with_range(dss_vg_info_t *vgs_info, uint32 vg_beg, uint32 vg_end) { - bool8 need_recovery = CM_FALSE; - /* 1、load offset batch 2、check batch valid 3、if batch valid, used 4、if batch invalid,just end */ - LOG_RUN_INF("[RECOVERY]Try to load log buf to recover"); - if (dss_load_log_buffer_from_offset(vg_item, &need_recovery) != CM_SUCCESS) { - return CM_ERROR; - } - if (need_recovery) { - char *log_buf = vg_item->log_file_ctrl.log_buf; - status_t status = dss_recover_from_offset_inner(session, vg_item, log_buf); + status_t status; + for (uint32 i = vg_beg; i < vg_end; i++) { + dss_share_vg_item_t *share_vg_item = dss_get_vg_item_by_id(CM_TRUE, i); + if (share_vg_item == NULL) { + LOG_RUN_ERR("Failed to get vg_item %u from shm!", i); + DSS_THROW_ERROR(ERR_DSS_GA_INIT, "failed to get vg_item %u from shm!", i); + return CM_ERROR; + } + + // set these info for api and local + share_vg_item->id = vgs_info->volume_group[i].id; + share_vg_item->all_vg_item_cnt = vg_end; + (void)memcpy_s(share_vg_item->vg_name, DSS_MAX_NAME_LEN, vgs_info->volume_group[i].vg_name, DSS_MAX_NAME_LEN); + (void)memcpy_s(share_vg_item->entry_path, DSS_MAX_VOLUME_PATH_LEN, vgs_info->volume_group[i].entry_path, + DSS_MAX_VOLUME_PATH_LEN); + int32 ret = shm_hashmap_init(&share_vg_item->buffer_cache, i, dss_buffer_cache_key_compare); + if (ret != CM_SUCCESS) { + if (i != 0) { + dss_free_shm_hashmap_memory(i - 1); + } + LOG_RUN_ERR("DSS instance failed to initialize buffer cache, %d!", ret); + DSS_THROW_ERROR(ERR_DSS_GA_INIT, "failed to init hashmap of vg %s", vgs_info->volume_group[i].vg_name); + return CM_ERROR; + } + + // set this info for local + vgs_info->volume_group[i].share_vg_item = share_vg_item; + vgs_info->volume_group[i].objectid = share_vg_item->objectid; + vgs_info->volume_group[i].buffer_cache = &share_vg_item->buffer_cache; + // at here, has not load dss_ctrl from disk + vgs_info->volume_group[i].dss_ctrl = &share_vg_item->dss_ctrl; + vgs_info->volume_group[i].vg_latch = &share_vg_item->vg_latch; + vgs_info->volume_group[i].stack.size = DSS_MAX_STACK_BUF_SIZE; + vgs_info->volume_group[i].stack.buff = (char *)cm_malloc_align(DSS_ALIGN_SIZE, DSS_MAX_STACK_BUF_SIZE); + if (vgs_info->volume_group[i].stack.buff == NULL) { + dss_free_shm_hashmap_memory(i); + LOG_DEBUG_ERR("malloc stack failed, align size:%u, size:%u.", DSS_ALIGN_SIZE, DSS_MAX_STACK_BUF_SIZE); + DSS_THROW_ERROR(ERR_ALLOC_MEMORY, DSS_MAX_STACK_BUF_SIZE, "volume group stack buff"); + return CM_ERROR; + } + + cm_bilist_init(&vgs_info->volume_group[i].open_file_list); + cm_bilist_init(&vgs_info->volume_group[i].syn_meta_desc.bilist); + + status = dss_alloc_vg_item_redo_log_buf(&vgs_info->volume_group[i]); if (status != CM_SUCCESS) { + dss_free_shm_hashmap_memory(i); + DSS_FREE_POINT(vgs_info->volume_group[i].stack.buff); return CM_ERROR; } + + vgs_info->volume_group[i].space_alarm = DSS_VG_SPACE_ALARM_INIT; + LOG_RUN_INF("success init vg item id [%u] vg_name [%s] entry_path[%s] by server", i, + vgs_info->volume_group[i].vg_name, vgs_info->volume_group[i].entry_path); } + return CM_SUCCESS; } -status_t dss_recover_from_slot(dss_session_t *session, dss_vg_info_item_t *vg_item) +status_t dss_init_server_vg_info() +{ + dss_config_t *inst_cfg = dss_get_inst_cfg(); + status_t status = dss_load_vg_conf_info(inst_cfg); + DSS_RETURN_IF_ERROR(status); + + dss_vg_info_t *vgs_info = dss_get_vg_info_ptr(); + status = dss_init_vg_info_by_server_with_range(vgs_info, 0, vgs_info->group_num); + DSS_RETURN_IF_ERROR(status); + + LOG_RUN_INF("DSS success to init vgs in memory."); + return status; +} + +status_t dss_refresh_server_vg_conf_inner(dss_vg_info_t *vgs_info, dss_config_t *inst_cfg) { - bool8 need_recovery = CM_FALSE; - if (dss_load_log_buffer_from_slot(vg_item, &need_recovery) != CM_SUCCESS) { + char vg_config_path[DSS_FILE_PATH_MAX_LENGTH]; + status_t status; + struct stat cur_stat; + + int32 errcode = sprintf_s(vg_config_path, DSS_FILE_PATH_MAX_LENGTH, "%s/cfg/%s", inst_cfg->home, DSS_VG_CONF_NAME); + bool32 result = (bool32)(errcode != -1); + DSS_RETURN_IF_FALSE2(result, CM_THROW_ERROR(ERR_SYSTEM_CALL, errcode)); + + if (stat(vg_config_path, &cur_stat) < 0) { + LOG_RUN_ERR("stat for file:%s fail.", vg_config_path); return CM_ERROR; } - if (need_recovery) { - char *log_buf = vg_item->log_file_ctrl.log_buf; - status_t status = dss_recover_from_slot_inner(session, vg_item, log_buf); - if (status != CM_SUCCESS) { - return CM_ERROR; - } + + if (vgs_info->inited_vg_num == 0) { + // assume refresh by the startup + vgs_info->inited_vg_num = vgs_info->group_num; + vgs_info->dest_vg_num = vgs_info->group_num; + vgs_info->cfg_stat = cur_stat; + LOG_RUN_INF("refresh vg ready new item(s), group_num [%u], des_vg_num [%u] inited_vg_num[%u]", + vgs_info->group_num, vgs_info->dest_vg_num, vgs_info->inited_vg_num); + return CM_SUCCESS; + } + + // the same file with same size, assme the same conten, only support add cfg, then size must change + if ((vgs_info->cfg_stat.st_dev == cur_stat.st_dev) && (vgs_info->cfg_stat.st_ino == cur_stat.st_ino) && + (vgs_info->cfg_stat.st_size == cur_stat.st_size)) { + // no updt vgs_info.cfg_stat for check exception + return CM_SUCCESS; + } + + // the init file has been changed, may exist new vg + uint32 len = DSS_MAX_CONFIG_FILE_SIZE; + status = dss_read_vg_config_file(vg_config_path, inst_cfg->config.file_buf, &len, DSS_TRUE); + if (status != CM_SUCCESS) { return status; } + + status = dss_parse_vg_config(vgs_info, inst_cfg->config.file_buf, len, CM_TRUE); + if (status != CM_SUCCESS) { + return status; + } + LOG_RUN_INF( + "refresh vg parse new item(s), group_num [%u], dest_vg_num [%u]", vgs_info->group_num, vgs_info->dest_vg_num); + // not find more new + if (vgs_info->dest_vg_num <= vgs_info->group_num) { + vgs_info->cfg_stat = cur_stat; + + return CM_SUCCESS; + } + + if (vgs_info->dest_vg_num > vgs_info->inited_vg_num) { + // init the local resource for the new vg + status = dss_init_vg_info_by_server_with_range(vgs_info, vgs_info->group_num, vgs_info->dest_vg_num); + DSS_RETURN_IF_ERROR(status); + vgs_info->inited_vg_num = vgs_info->dest_vg_num; + LOG_RUN_INF("refresh vg inited new item(s), inited_vg_num [%u], dest_vg_num [%u]", vgs_info->inited_vg_num, + vgs_info->dest_vg_num); + } + vgs_info->cfg_stat = cur_stat; + + // at here, all the vg in [group_nu, dest_vg_num] not loaded from disk, SHOULD NOT been seen by the api + // only after loaded from disk and do finish to to recovery, will call dss_refresh_vg_info_updt_vg_num to + // make these are seen by api return CM_SUCCESS; } -status_t dss_recover_from_instance(dss_session_t *session, dss_instance_t *inst) +status_t dss_refresh_server_vg_info() { + dss_vg_info_t *vgs_info = dss_get_vg_info_ptr(); + dss_config_t *inst_cfg = dss_get_inst_cfg(); + return dss_refresh_server_vg_conf_inner(vgs_info, inst_cfg); +} + +static status_t dss_load_vg_info_and_recover(bool8 need_recovery) +{ + dss_vg_info_t *vgs_info = dss_get_vg_info_ptr(); + return dss_load_vg_info_and_recover_with_range(0, vgs_info->group_num, need_recovery); +} + +status_t dss_refresh_load_vg_info_and_recover(dss_session_t *session, dss_instance_t *inst) +{ + if (inst->status != DSS_STATUS_OPEN) { + return CM_ERROR; + } + status_t status; - for (uint32 i = 0; i < g_vgs_info->group_num; i++) { - dss_vg_info_item_t *vg_item = &g_vgs_info->volume_group[i]; - if (dss_check_vg_ctrl_valid(vg_item) != CM_SUCCESS) { - LOG_RUN_ERR("[RECOVERY]Failed to check valid of vg %s.", vg_item->vg_name); - return CM_ERROR; - } - uint32 software_version = dss_get_software_version(&vg_item->dss_ctrl->vg_info); - if (software_version < DSS_SOFTWARE_VERSION_2) { - status = dss_recover_from_slot(session, vg_item); - if (status != CM_SUCCESS) { - LOG_RUN_ERR("[RECOVERY]Failed to recover from vg %s.", vg_item->vg_name); - return CM_ERROR; - } - } else { - status = dss_load_redo_ctrl(vg_item); - if (status != CM_SUCCESS) { - LOG_RUN_ERR("[RECOVERY]Failed to load redo ctrl of vg %s.", vg_item->vg_name); - return CM_ERROR; - } - status = dss_recover_from_offset(session, vg_item); - if (status != CM_SUCCESS) { - LOG_RUN_ERR("[RECOVERY]Failed to recover from vg %s.", vg_item->vg_name); - return CM_ERROR; - } - } + status = dss_refresh_server_vg_info(); + DSS_RETURN_IF_ERROR(status); + + dss_vg_info_t *vgs_info = dss_get_vg_info_ptr(); + if (vgs_info->group_num == vgs_info->dest_vg_num) { + return CM_SUCCESS; } - return status; + + // try to recover new add vg + status = dss_load_vg_info_and_recover_with_range(vgs_info->group_num, vgs_info->dest_vg_num, CM_FALSE); + DSS_RETURN_IF_ERROR(status); + + dss_refresh_vg_info_update_vg_num(vgs_info); + return CM_SUCCESS; +} + +status_t dss_recover_from_instance(dss_session_t *session, dss_instance_t *inst) +{ + dss_vg_info_t *vgs_info = dss_get_vg_info_ptr(); + return dss_recover_redo_log_with_range(session, 0, vgs_info->group_num); } bool32 dss_config_cm() @@ -218,7 +334,7 @@ static status_t dss_init_inst_handle_session(dss_instance_t *inst) static status_t instance_init_core(dss_instance_t *inst) { - status_t status = dss_get_vg_info(); + status_t status = dss_init_server_vg_info(); DSS_RETURN_IFERR2(status, DSS_THROW_ERROR(ERR_DSS_GA_INIT, "DSS instance failed to get vg info.")); status = dss_init_session_pool(dss_get_max_total_session_cnt()); DSS_RETURN_IFERR2(status, DSS_THROW_ERROR(ERR_DSS_GA_INIT, "DSS instance failed to initialize sessions.")); @@ -514,11 +630,12 @@ void dss_uninit_cm(dss_instance_t *inst) void dss_free_log_ctrl() { - if (g_vgs_info == NULL) { + dss_vg_info_t *vgs_info = dss_get_vg_info_ptr(); + if (vgs_info == NULL) { return; } - for (uint32 i = 0; i < g_vgs_info->group_num; i++) { - dss_vg_info_item_t *vg_item = &g_vgs_info->volume_group[i]; + for (uint32 i = 0; i < vgs_info->group_num; i++) { + dss_vg_info_item_t *vg_item = &vgs_info->volume_group[i]; if (vg_item != NULL && vg_item->log_file_ctrl.log_buf != NULL) { DSS_FREE_POINT(vg_item->log_file_ctrl.log_buf); } diff --git a/src/service/dss_instance.h b/src/service/dss_instance.h index 32063f5..b11d373 100644 --- a/src/service/dss_instance.h +++ b/src/service/dss_instance.h @@ -75,7 +75,7 @@ typedef struct st_dss_instance { int32 lock_fd; latch_t switch_latch; dss_config_t inst_cfg; - dss_instance_status_e status; + volatile dss_instance_status_e status; uds_lsnr_t lsnr; latch_t uds_lsnr_latch; reactors_t reactors; @@ -87,12 +87,13 @@ typedef struct st_dss_instance { spinlock_t inst_work_lock; int32 cluster_proto_vers[DSS_MAX_INSTANCES]; bool8 is_maintain; - bool8 is_cleaning; - bool8 no_grab_lock; - bool8 is_releasing_lock; - bool8 is_checking; - bool8 reserve[3]; - bool32 is_join_cluster; + volatile bool8 is_cleaning; + volatile bool8 no_grab_lock; + volatile bool8 is_releasing_lock; + volatile bool8 is_checking; + volatile bool8 is_handle_main_wait; + bool8 reserve[2]; + volatile bool32 is_join_cluster; dss_session_t *handle_session; dss_bg_task_info_t syn_meta_task[DSS_META_SYN_BG_TASK_NUM_MAX]; @@ -132,6 +133,11 @@ void dss_meta_syn_proc(thread_t *thread); void dss_recycle_meta_proc(thread_t *thread); void dss_alarm_check_proc(thread_t *thread); +status_t dss_init_vg_info_by_server_with_range(dss_vg_info_t *vgs_info, uint32 vg_beg, uint32 vg_end); +status_t dss_init_server_vg_info(); +status_t dss_refresh_server_vg_info(); +status_t dss_refresh_load_vg_info_and_recover(dss_session_t *session, dss_instance_t *inst); + #ifdef __cplusplus } #endif diff --git a/src/service/dss_service.c b/src/service/dss_service.c index ae7f841..cf5d22e 100644 --- a/src/service/dss_service.c +++ b/src/service/dss_service.c @@ -988,7 +988,7 @@ void dss_wait_session_pause(dss_instance_t *inst) void dss_wait_background_pause(dss_instance_t *inst) { LOG_DEBUG_INF("Begin to set background paused."); - while (inst->is_cleaning || inst->is_checking) { + while (inst->is_cleaning || inst->is_checking || inst->is_handle_main_wait) { cm_sleep(1); } LOG_DEBUG_INF("Succeed to pause background task."); @@ -1171,7 +1171,7 @@ static status_t dss_process_set_main_inst(dss_session_t *session) return CM_ERROR; } if (!cm_latch_timed_x( - &g_dss_instance.switch_latch, session->id, DSS_PROCESS_REMOTE_INTERVAL, LATCH_STAT(LATCH_SWITCH))) { + &g_dss_instance.switch_latch, session->id, DSS_PROCESS_REMOTE_INTERVAL, LATCH_STAT(LATCH_SWITCH))) { LOG_RUN_INF("[SWITCH] Spin switch lock timed out, just continue."); continue; } @@ -1306,8 +1306,8 @@ static status_t dss_process_enable_upgrades(dss_session_t *session) dss_config_t *cfg = dss_get_inst_cfg(); uint32 curr_id = (uint32)(cfg->params.inst_id); dss_get_version_output_t get_version_output = {.all_same = DSS_TRUE, .min_version = DSS_PROTO_VERSION}; - DSS_RETURN_IF_ERROR(dss_set_audit_resource(session->audit_info.resource, DSS_AUDIT_MODIFY, - "enable upgrades", curr_id)); + DSS_RETURN_IF_ERROR( + dss_set_audit_resource(session->audit_info.resource, DSS_AUDIT_MODIFY, "enable upgrades", curr_id)); int ret = dss_bcast_get_protocol_version(&get_version_output); if (ret != CM_SUCCESS) { // If any node return ERR_DSS_UNSUPPORTED_CMD, we assume old node exists. diff --git a/src/service/dssserver.c b/src/service/dssserver.c index e5f99f9..bad73df 100644 --- a/src/service/dssserver.c +++ b/src/service/dssserver.c @@ -150,6 +150,8 @@ static void handle_main_wait(void) if (!g_dss_instance.is_maintain) { dss_check_peer_inst(&g_dss_instance, DSS_INVALID_ID64); } + g_dss_instance.is_handle_main_wait = CM_TRUE; + CM_MFENCE; if (periods == MILLISECS_PER_SECOND * SECONDS_PER_DAY / interval) { periods = 0; dss_ssl_ca_cert_expire(); @@ -159,6 +161,10 @@ static void handle_main_wait(void) } dss_clean_all_sessions_latch(); + + (void)dss_refresh_load_vg_info_and_recover(g_dss_instance.handle_session, &g_dss_instance); + + g_dss_instance.is_handle_main_wait = CM_FALSE; cm_sleep(interval); periods++; } while (CM_TRUE); @@ -203,27 +209,12 @@ static status_t dss_create_bg_task_set(dss_instance_t *inst, char *task_name, ui task_num = max_task_num; } - uint32 vg_per_task = g_vgs_info->group_num / task_num; - uint32 vg_left = g_vgs_info->group_num % task_num; - - uint32 vg_id = 0; - uint32 cur_range = 0; - for (uint32 i = 0; i < task_num; i++) { bg_task_info_set[i].task_num_max = task_num; bg_task_info_set[i].my_task_id = i; - bg_task_info_set[i].vg_id_beg = vg_id; bg_task_info_set[i].task_args = task_args; - if (vg_left > 0) { - cur_range = vg_per_task + 1; - vg_left--; - } else { - cur_range = vg_per_task; - } - bg_task_info_set[i].vg_id_end = bg_task_info_set[i].vg_id_beg + cur_range; - vg_id = bg_task_info_set[i].vg_id_end; - LOG_RUN_INF("task:%s id:%u, vg_range:[%u-%u).", task_name, bg_task_info_set[i].my_task_id, - bg_task_info_set[i].vg_id_beg, bg_task_info_set[i].vg_id_end); + + LOG_RUN_INF("task_num_max:%u, my_task_id:%u, task:%su).", task_num, bg_task_info_set[i].my_task_id, task_name); uint32 work_idx = get_bg_task_idx(i); status_t status = cm_create_thread(bg_task_entry, 0, &(bg_task_info_set[i]), &(inst->threads[work_idx])); diff --git a/src/tbox/dsstbox.c b/src/tbox/dsstbox.c index a14d70b..f86f635 100644 --- a/src/tbox/dsstbox.c +++ b/src/tbox/dsstbox.c @@ -409,9 +409,9 @@ static status_t repair_proc(void) status = dss_repair_fs_aux(&input); } else { DSS_PRINT_RUN_ERROR("[TBOX][REPAIR] Only support -t " - "[fs_block|ft_block|core_ctrl|volume_header|software_version|" - "root_ft_block|volume_ctrl|fs_aux_block], " - "your type is %s.\n", + "[fs_block|ft_block|core_ctrl|volume_header|software_version|" + "root_ft_block|volume_ctrl|fs_aux_block], " + "your type is %s.\n", input.type); status = CM_ERROR; } @@ -571,7 +571,7 @@ static status_t miner_proc(void) DSS_PRINT_ERROR("[TBOX][MINER]DSS init loggers failed!\n"); return status; } - status = dss_load_vg_conf_info(&g_vgs_info, &inst_cfg); + status = dss_load_vg_conf_info(&inst_cfg); if (status != CM_SUCCESS) { DSS_PRINT_ERROR("[TBOX][MINER]Failed to load vg info from config, errcode is %d.\n", status); return status; -- Gitee