diff --git a/src/common/backend/utils/misc/postgresql_single.conf.sample b/src/common/backend/utils/misc/postgresql_single.conf.sample index 60d178295e1b5a9f79ab78c28fb012447fd048ff..ffe049cce44c5700496671817d2812631d15b024 100644 --- a/src/common/backend/utils/misc/postgresql_single.conf.sample +++ b/src/common/backend/utils/misc/postgresql_single.conf.sample @@ -845,4 +845,4 @@ job_queue_processes = 10 # Number of concurrent jobs, optional: [0..1000] #ss_log_level = 7 #ss_log_backup_file_count = 10 #ss_log_max_file_size = 10MB -#ss_parallel_thread_count = 16 \ No newline at end of file +#ss_parallel_thread_count = 16 diff --git a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp index cfcb2eb0aec4f93e96d3a97875903e48195bb58f..4ff2c860df372ee4139687428c304aa75d3e6118 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_callback.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_callback.cpp @@ -1367,10 +1367,10 @@ static int CBStartup(void *db_handle) static int CBRecoveryStandby(void *db_handle, int inst_id) { Assert(inst_id == g_instance.attr.attr_storage.dms_attr.instance_id); - ereport(LOG, (errmsg("[SS reform] Recovery as standby"))); + ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS reform] Recovery as standby"))); if (!SSRecoveryNodes()) { - ereport(WARNING, (errmodule(MOD_DMS), errmsg("Recovery failed in startup first"))); + ereport(WARNING, (errmodule(MOD_DMS), errmsg("[SS reform] Recovery failed"))); return GS_ERROR; } @@ -1382,14 +1382,14 @@ static int CBRecoveryPrimary(void *db_handle, int inst_id) Assert(g_instance.dms_cxt.SSReformerControl.primaryInstId == inst_id || g_instance.dms_cxt.SSReformerControl.primaryInstId == -1); g_instance.dms_cxt.SSRecoveryInfo.in_flushcopy = false; - ereport(LOG, (errmsg("[SS reform] Recovery as primary, will replay xlog from inst:%d", + ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS reform] Recovery as primary, will replay xlog from inst:%d", g_instance.dms_cxt.SSReformerControl.primaryInstId))); /* Release my own lock before recovery */ SSLockReleaseAll(); SSWakeupRecovery(); if (!SSRecoveryNodes()) { - ereport(WARNING, (errmodule(MOD_DMS), errmsg("Recovery failed in startup first"))); + ereport(WARNING, (errmodule(MOD_DMS), errmsg("[SS reform] Recovery failed"))); return GS_ERROR; } @@ -1548,9 +1548,11 @@ static void CBReformStartNotify(void *db_handle, dms_role_t role, unsigned char g_instance.dms_cxt.SSClusterState = NODESTATE_NORMAL; g_instance.dms_cxt.SSRecoveryInfo.reform_ready = false; g_instance.dms_cxt.SSRecoveryInfo.in_flushcopy = false; + g_instance.dms_cxt.SSRecoveryInfo.startup_need_exit_normally = false; g_instance.dms_cxt.resetSyscache = true; if (ss_reform_type == DMS_REFORM_TYPE_FOR_FAILOVER_OPENGAUSS) { g_instance.dms_cxt.SSRecoveryInfo.in_failover = true; + g_instance.dms_cxt.SSRecoveryInfo.recovery_pause_flag = true; if (role == DMS_ROLE_REFORMER) { g_instance.dms_cxt.dw_init = false; // variable set order: SharedRecoveryInProgress -> failover_triggered -> dms_role diff --git a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp index 673e5553bd577c3f52818728532502ad8141b7f5..3c64f1ce8e76bd75777c70de872643ecefb6a37b 100644 --- a/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp +++ b/src/gausskernel/ddes/adapter/ss_dms_recovery.cpp @@ -54,11 +54,45 @@ void SSSavePrimaryInstId(int id) SSSaveReformerCtrl(); } +/** + * find reform failed in recovery phase, maybe other node restart + * pageredo or startup thread may trapped in LockBuffer for page request + * to solve this: reform_proc thread need exit process + * + * reform failed during recovery phase has three situation + * 1) primary restart 2) restart failover 3) alive failover + * + * 1) primary restart 2) restart failover: + * gaussdb will restart + * 3) alive failover: + * try to exit startup thread, + * if success, gaussdb still alive and prepare next reform + * if not, gaussdb need exit cause pageredo or startup may trapped in LockBuffer +*/ bool SSRecoveryNodes() { bool result = false; while (true) { if (dms_reform_failed()) { + if (SS_STANDBY_FAILOVER && !g_instance.dms_cxt.SSRecoveryInfo.restart_failover_flag) { + g_instance.dms_cxt.SSRecoveryInfo.startup_need_exit_normally = true; + } + SendPostmasterSignal(PMSIGNAL_DMS_TERM_STARTUP); + + while (true) { + if (g_instance.pid_cxt.StartupPID == 0) { + ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS reform] reform failed, startup thread exit noramlly " + "during recovery"))); + break; + } + + if (g_instance.dms_cxt.SSRecoveryInfo.recovery_trapped_in_page_request) { + ereport(WARNING, (errmodule(MOD_DMS), errmsg("[SS reform] pageredo or startup thread are trapped " + "in page request during recovery phase, need exit"))); + _exit(0); + } + pg_usleep(5000L); + } result = false; break; } diff --git a/src/gausskernel/process/postmaster/postmaster.cpp b/src/gausskernel/process/postmaster/postmaster.cpp index b8ed975843dbed05b162835ea1e2ceea7d10f729..bb578682ee882603e6e8aed1fa3dd622791e3d71 100644 --- a/src/gausskernel/process/postmaster/postmaster.cpp +++ b/src/gausskernel/process/postmaster/postmaster.cpp @@ -10209,6 +10209,13 @@ static void sigusr1_handler(SIGNAL_ARGS) wal_get_role_string(get_cur_mode())))); } + if (ENABLE_DMS && CheckPostmasterSignal(PMSIGNAL_DMS_TERM_STARTUP)) { + if (g_instance.pid_cxt.StartupPID != 0) { + ereport(LOG, (errmodule(MOD_DMS), errmsg("[SS reform] send to startup term signal"))); + signal_child(g_instance.pid_cxt.StartupPID, SIGTERM); + } + } + if (CheckPromoteSignal()) { handle_promote_signal(); } diff --git a/src/gausskernel/process/threadpool/knl_instance.cpp b/src/gausskernel/process/threadpool/knl_instance.cpp index 4a8ee49b480ae231e8f6c05726318bfd2809a653..6c00699141f140f6646f93742d7065874d51e344 100755 --- a/src/gausskernel/process/threadpool/knl_instance.cpp +++ b/src/gausskernel/process/threadpool/knl_instance.cpp @@ -190,6 +190,8 @@ static void knl_g_dms_init(knl_g_dms_context *dms_cxt) dms_cxt->SSRecoveryInfo.in_failover = false; dms_cxt->SSRecoveryInfo.in_flushcopy = false; dms_cxt->SSRecoveryInfo.no_backend_left = false; + dms_cxt->SSRecoveryInfo.startup_need_exit_normally = false; + dms_cxt->SSRecoveryInfo.recovery_trapped_in_page_request = false; dms_cxt->log_timezone = NULL; pg_atomic_init_u32(&dms_cxt->inDmsThreShmemInitCnt, 0); pg_atomic_init_u32(&dms_cxt->inProcExitCnt, 0); diff --git a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp index 5464e7ccb37e3ff678e72866e7156b6fb54c6001..10dc1252fb65091c5d5f7e03d5345eaa7116b1bf 100755 --- a/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp +++ b/src/gausskernel/storage/access/transam/parallel_recovery/dispatcher.cpp @@ -2481,6 +2481,12 @@ static void HandleStartupProcInterruptsForParallelRedo(void) SendSingalToPageWorker(SIGHUP); } + if (ENABLE_DMS && t_thrd.startup_cxt.shutdown_requested && SmartShutdown != g_instance.status && + g_instance.dms_cxt.SSRecoveryInfo.startup_need_exit_normally) { + crps_destory_ctxs(); + proc_exit(0); + } + /* * Check if we were requested to exit without finishing recovery. */ diff --git a/src/gausskernel/storage/buffer/bufmgr.cpp b/src/gausskernel/storage/buffer/bufmgr.cpp index 93bbda82d2e36dc1cc4803571c60d88d15ee2b67..12c67fff94ceba68285a433572f1a6adb0dd4897 100644 --- a/src/gausskernel/storage/buffer/bufmgr.cpp +++ b/src/gausskernel/storage/buffer/bufmgr.cpp @@ -2463,6 +2463,12 @@ found_branch: TerminateBufferIO(bufHdr, false, 0); // when reform fail, should return InvalidBuffer to reform proc thread if (AmDmsReformProcProcess() && dms_reform_failed()) { + SSUnPinBuffer(bufHdr); + return InvalidBuffer; + } + + if ((AmPageRedoProcess() || AmStartupProcess()) && dms_reform_failed()) { + SSUnPinBuffer(bufHdr); return InvalidBuffer; } @@ -5969,6 +5975,10 @@ retry: LWLockRelease(buf->content_lock); + if ((AmPageRedoProcess() || AmStartupProcess()) && dms_reform_failed()) { + g_instance.dms_cxt.SSRecoveryInfo.recovery_trapped_in_page_request = true; + } + dms_retry_times++; pg_usleep(SSGetBufSleepTime(dms_retry_times)); goto retry; @@ -6074,6 +6084,10 @@ retry: } LWLockRelease(buf->content_lock); + if ((AmPageRedoProcess() || AmStartupProcess()) && dms_reform_failed()) { + g_instance.dms_cxt.SSRecoveryInfo.recovery_trapped_in_page_request = true; + } + dms_retry_times++; pg_usleep(SSGetBufSleepTime(dms_retry_times)); goto retry; diff --git a/src/gausskernel/storage/smgr/segment/segbuffer.cpp b/src/gausskernel/storage/smgr/segment/segbuffer.cpp index 6bab9bc38ae42e884fa35dc9838ba15b1d9d517f..dad28dfe182677c44a2369a907d8f08ef1f8b96b 100644 --- a/src/gausskernel/storage/smgr/segment/segbuffer.cpp +++ b/src/gausskernel/storage/smgr/segment/segbuffer.cpp @@ -614,6 +614,12 @@ Buffer ReadBufferFast(SegSpace *spc, RelFileNode rnode, ForkNumber forkNum, Bloc SegTerminateBufferIO((BufferDesc *)bufHdr, false, 0); // when reform fail, should return InvalidBuffer to reform proc thread if (AmDmsReformProcProcess() && dms_reform_failed()) { + SSUnPinBuffer(bufHdr); + return InvalidBuffer; + } + + if ((AmPageRedoProcess() || AmStartupProcess()) && dms_reform_failed()) { + SSUnPinBuffer(bufHdr); return InvalidBuffer; } diff --git a/src/include/ddes/dms/ss_dms_bufmgr.h b/src/include/ddes/dms/ss_dms_bufmgr.h index a2b67a7cbfe9d1f589d1efc9daa4e2ad07a27c27..07d2d38d70775ff9140b7964902c64b935bba9fc 100644 --- a/src/include/ddes/dms/ss_dms_bufmgr.h +++ b/src/include/ddes/dms/ss_dms_bufmgr.h @@ -83,4 +83,5 @@ void SSMarkBufferDirtyForERTO(RedoBufferInfo* bufferinfo); long SSGetBufSleepTime(int retry_times); SMGR_READ_STATUS SmgrNetPageCheckRead(Oid spcNode, Oid dbNode, Oid relNode, ForkNumber forkNum, BlockNumber blockNo, char *blockbuf); +void SSUnPinBuffer(BufferDesc* buf_desc); #endif diff --git a/src/include/ddes/dms/ss_dms_recovery.h b/src/include/ddes/dms/ss_dms_recovery.h index 90b413f052f223af74e48a579badafa128357868..91dd619bb57873e159f0cc8ca88f57a495870147 100644 --- a/src/include/ddes/dms/ss_dms_recovery.h +++ b/src/include/ddes/dms/ss_dms_recovery.h @@ -58,6 +58,8 @@ typedef struct ss_recovery_info { bool in_failover; // used to detemin failover scenario, especially for the non-promoting node bool in_flushcopy; bool no_backend_left; + bool startup_need_exit_normally; //used in alive failover + bool recovery_trapped_in_page_request; //used in alive failover } ss_recovery_info_t; extern bool SSRecoveryNodes(); diff --git a/src/include/storage/pmsignal.h b/src/include/storage/pmsignal.h index 98d24feebebdd9951f25a00bf25c905ef1be9ca5..6b83ce0adf6e1d10f19e38b5b72fe30273ac7fd8 100644 --- a/src/include/storage/pmsignal.h +++ b/src/include/storage/pmsignal.h @@ -55,6 +55,7 @@ typedef enum { PMSIGNAL_DMS_SWITCHOVER_PROMOTE, /* dms standby switchover promote */ PMSIGNAL_DMS_REFORM, /* dms reform start during PM_RUN */ PMSIGNAL_DMS_REFORM_DONE, /* dms reform done */ + PMSIGNAL_DMS_TERM_STARTUP, /* term startup thread*/ NUM_PMSIGNALS /* Must be last value of enum! */ } PMSignalReason;