From fc63200f355bdb81b0eedea9639c0f249b6479ee Mon Sep 17 00:00:00 2001 From: ryne3366 Date: Tue, 6 May 2025 17:40:34 +0800 Subject: [PATCH] multi-process rekey Signed-off-by: ryne3366 --- BUILD.gn | 1 + ...rohibited-in-multi-process-scenarios.patch | 280 ++++++++++++++++++ ...ion.patch => 0009-Current-bug-fixes.patch} | 39 ++- 3 files changed, 300 insertions(+), 20 deletions(-) create mode 100644 patch/0008-Rekey-is-prohibited-in-multi-process-scenarios.patch rename patch/{0008-BugFix-CurrVersion.patch => 0009-Current-bug-fixes.patch} (88%) diff --git a/BUILD.gn b/BUILD.gn index 3b03c37..d796c22 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -244,6 +244,7 @@ if (is_mingw || is_mac) { cflags = [ "-Wno-error=implicit-function-declaration", "-Wno-implicit-fallthrough", + "-Wno-inconsistent-dllimport", ] part_name = "sqlite" subsystem_name = "thirdparty" diff --git a/patch/0008-Rekey-is-prohibited-in-multi-process-scenarios.patch b/patch/0008-Rekey-is-prohibited-in-multi-process-scenarios.patch new file mode 100644 index 0000000..76eda91 --- /dev/null +++ b/patch/0008-Rekey-is-prohibited-in-multi-process-scenarios.patch @@ -0,0 +1,280 @@ +From ede0e8debe3ce4848df95f8a20de9ddb4f729355 Mon Sep 17 00:00:00 2001 +From: ryne3366 +Date: Tue, 6 May 2025 17:14:22 +0800 +Subject: [PATCH 1/2] rekey multi process + +--- + src/sqlite3.c | 167 +++++++++++++++++++++++++++++++++++++++----------- + 1 file changed, 132 insertions(+), 35 deletions(-) + +diff --git a/src/sqlite3.c b/src/sqlite3.c +index 3a6a372..97c8f86 100644 +--- a/src/sqlite3.c ++++ b/src/sqlite3.c +@@ -6769,6 +6769,10 @@ SQLITE_API int sqlite3_collation_needed16( + ); + + #ifdef SQLITE_HAS_CODEC ++#define F_CODEC_RDLCK 0 ++#define F_CODEC_WRLCK 1 ++#define F_CODEC_UNLCK 2 ++static int CodecFileLock(sqlite3_file *fd, short lockType); + /* + ** Specify the key for an encrypted database. This routine should be + ** called right after sqlite3_open(). +@@ -15811,7 +15815,7 @@ typedef int VList; + #define RESERVED_BYTE (PENDING_BYTE+1) + #define SHARED_FIRST (PENDING_BYTE+2) + #define SHARED_SIZE 510 +- ++#define SHARED_CODEC_BYTE (PENDING_BYTE+2097) + /* + ** Wrapper around OS specific sqlite3_os_init() function. + */ +@@ -40080,6 +40084,9 @@ struct unixInodeInfo { + int nLock; /* Number of outstanding file locks */ + unsigned char eFileLock; /* One of SHARED_LOCK, RESERVED_LOCK etc. */ + unsigned char bProcessLock; /* An exclusive process lock is held */ ++#if SQLITE_HAS_CODEC ++ int codecLock; ++#endif + UnixUnusedFd *pUnused; /* Unused file descriptors to close */ + int nRef; /* Number of pointers to this structure */ + unixShmNode *pShmNode; /* Shared memory associated with this inode */ +@@ -62279,6 +62286,9 @@ SQLITE_PRIVATE int sqlite3PagerClose(Pager *pPager, sqlite3 *db){ + PAGERTRACE(("CLOSE %d\n", PAGERID(pPager))); + IOTRACE(("CLOSE %p\n", pPager)) + sqlite3OsClose(pPager->jfd); ++#if defined(SQLITE_HAS_CODEC) ++ if( pPager->xCodec ) CodecFileLock(pPager->fd, F_CODEC_UNLCK); ++#endif + sqlite3OsClose(pPager->fd); + sqlite3PageFree(pTmp); + sqlite3PcacheClose(pPager->pPCache); +@@ -259020,6 +259030,44 @@ SQLITE_API int sqlite3_stmt_init( + SQLITE_API const char *sqlite3_sourceid(void){ return SQLITE_SOURCE_ID; } + /************************** End of sqlite3.c ******************************/ + ++static inline u8 IsOnlyOneConnection(Pager *pPager) ++{ ++#if SQLITE_OS_UNIX ++ unixFile *fd = (unixFile *)pPager->fd; ++ if (fd == NULL || fd->pInode == NULL || pPager->pVfs == NULL || ++ sqlite3_stricmp(pPager->pVfs->zName, "unix") != 0) { ++ sqlite3_log(SQLITE_WARNING_DUMP, "Check connection go wrong"); ++ return 0; ++ } ++ unixInodeInfo *pInode = fd->pInode; ++ sqlite3_mutex_enter(pInode->pLockMutex); ++ // unix and only one connection exist ++ if (pInode->nRef != 1) { ++ sqlite3_mutex_leave(pInode->pLockMutex); ++ return 0; ++ } ++ sqlite3_mutex_leave(pInode->pLockMutex); ++ return 1; ++#else ++ return 0; ++#endif ++} ++ ++#if SQLITE_OS_UNIX ++#include ++#include ++static inline int OsGetTid(void) ++{ ++#if defined(__linux__) ++ return (int)syscall(__NR_gettid); ++#elif defined(__APPLE__) ++ return (int)syscall(SYS_thread_selfid); ++#else ++ return 0; ++#endif ++} ++#endif ++ + #ifdef SQLITE_HAS_CODEC + /************** Begin file hw_codec_openssl.h *******************************/ + #ifndef EXPOSE_INTERNAL_FUNC +@@ -260133,6 +260181,60 @@ void sqlite3CodecDetach(void *ctx){ + return; + } + ++#if defined(SQLITE_HAS_CODEC) ++static int CodecFileLock(sqlite3_file *fd, short lockType) ++{ ++#if SQLITE_OS_UNIX ++ unixFile *pFile = (unixFile *)fd; ++ unixInodeInfo *pInode = pFile->pInode; ++ if (pInode == NULL) { ++ sqlite3_log(SQLITE_IOERR_RDLOCK, "Codec file lock %d go wrong", lockType); ++ return SQLITE_IOERR_RDLOCK; ++ } ++#if (F_CODEC_RDLCK != F_RDLCK || F_CODEC_WRLCK != F_WRLCK || F_CODEC_UNLCK != F_UNLCK) ++#error "error rd lock type" ++#endif ++ sqlite3_mutex_enter(pInode->pLockMutex); ++ // last conntection release the lock ++ if (lockType == F_UNLCK && pInode->nRef > 1) { ++ sqlite3_mutex_leave(pInode->pLockMutex); ++ return SQLITE_OK; ++ } ++ if (lockType == F_WRLCK && pInode->nRef > 1) { ++ sqlite3_log(SQLITE_BUSY, "Codec file lock wrlock busy ref %d", pInode->nRef); ++ sqlite3_mutex_leave(pInode->pLockMutex); ++ return SQLITE_BUSY; ++ } ++ if (lockType == F_RDLCK && pInode->codecLock != 0 && pInode->codecLock != OsGetTid()) { ++ sqlite3_log(SQLITE_BUSY, "Codec exists file lock wrlock"); ++ sqlite3_mutex_leave(pInode->pLockMutex); ++ return SQLITE_BUSY; ++ } ++ struct flock lock; ++ lock.l_whence = SEEK_SET; ++ lock.l_start = SHARED_CODEC_BYTE; ++ lock.l_len = 1; ++ // F_RDLCK, F_WRLCK, F_UNLCK ++ lock.l_type = lockType; ++ int rc = osSetPosixAdvisoryLock(pFile->h, &lock, pFile); ++ if (rc < 0) { ++ int tErrno = errno; ++ rc = sqliteErrorFromPosixError(tErrno, SQLITE_IOERR_RDLOCK); ++ if (IS_LOCK_ERROR(rc)) { ++ storeLastErrno(pFile, tErrno); ++ } ++ sqlite3_log(rc, "Codec file lock %d: errno = %d", lockType, tErrno); ++ } else { ++ pInode->codecLock = (lockType == F_WRLCK ? OsGetTid() : 0); ++ } ++ sqlite3_mutex_leave(pInode->pLockMutex); ++ return rc; ++#else ++ return 0; ++#endif ++} ++#endif ++ + int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ + if(db == NULL){ + return SQLITE_ERROR; +@@ -260152,6 +260254,15 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ + return SQLITE_ERROR; + } + sqlite3_mutex_enter(db->mutex); ++ Pager *pPager = p->pBt->pPager; ++ if (pPager->fd != NULL && isOpen(pPager->fd)) { ++ int ret = CodecFileLock(pPager->fd, F_CODEC_RDLCK); ++ if(ret != SQLITE_OK){ ++ sqlite3_log(ret, "sqlite3CodecAttach: codec lock error"); ++ sqlite3_free(ctx); ++ return ret; ++ } ++ } + #ifdef SQLITE_CODEC_ATTACH_CHANGED + int rc = sqlite3CodecInitContext(ctx, p, pKey, nKey, nDb); + #else +@@ -260167,12 +260278,10 @@ int sqlite3CodecAttach(sqlite3* db, int nDb, const void *pKey, int nKey){ + p->pBt->btsFlags &= ~BTS_PAGESIZE_FIXED; + sqlite3BtreeSetPageSize(p, ctx->readCtx->codecConst.cipherPageSize, ctx->readCtx->codecConst.reserveSize, 0); + sqlite3BtreeSecureDelete(p, 1); +- if(isOpen(p->pBt->pPager->fd)){ ++ if(isOpen(pPager->fd)){ + sqlite3BtreeSetAutoVacuum(p, SQLITE_DEFAULT_AUTOVACUUM); + } +- + sqlite3_mutex_leave(db->mutex); +- + return SQLITE_OK; + } + +@@ -260238,7 +260347,20 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ + return rc; + } + sqlite3_mutex_enter(db->mutex); +- (void)sqlite3BtreeBeginTrans(p, 1, 0); ++ rc = sqlite3BtreeBeginTrans(p, 1, 0); ++ if(rc != SQLITE_OK){ ++ sqlite3_mutex_leave(db->mutex); ++ sqlite3_log(rc, "sqlite3 rekey: error when begin trans."); ++ return rc; ++ } ++#if SQLITE_OS_UNIX ++ if((rc = CodecFileLock(pPager->fd, F_CODEC_WRLCK)) != SQLITE_OK){ ++ sqlite3_log(rc, "sqlite3_rekey_v2: error when lock."); ++ (void)sqlite3BtreeRollback(p, SQLITE_ABORT_ROLLBACK, 0); ++ sqlite3_mutex_leave(db->mutex); ++ return rc; ++ } ++#endif + sqlite3PagerPagecount(pPager, &pageCount); + // support hmac algo changed by using rekey operation + int oldHmacAlgo = ctx->writeCtx->codecConst.hmacAlgo; +@@ -260273,6 +260395,9 @@ int sqlite3_rekey_v2(sqlite3 *db, const char *zDb, const void *pKey, int nKey){ + } + (void)sqlite3BtreeRollback(p, SQLITE_ABORT_ROLLBACK, 0); + } ++#if SQLITE_OS_UNIX ++ (void)CodecFileLock(pPager->fd, F_CODEC_RDLCK); ++#endif + sqlite3_mutex_leave(db->mutex); + + return rc; +@@ -261597,25 +261722,10 @@ CHK_RESTORE_OUT: + return rc; + } + +-static inline u8 IsConnectionValidForCheck(Pager *pPager) +-{ +-#if SQLITE_OS_UNIX +- unixFile *fd = (unixFile *)pPager->fd; +- // unix and only one connection exist +- if (fd == NULL || fd->pInode == NULL || pPager->pVfs == NULL || +- sqlite3_stricmp(pPager->pVfs->zName, "unix") != 0 || fd->pInode->nRef != 1) { +- return 0; +- } +- return 1; +-#else +- return 0; +-#endif +-} +- + static int MetaDwrOpenAndCheck(Btree *pBt) + { + Pager *pPager = pBt->pBt->pPager; +- if (pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) { ++ if (pPager->memDb || pPager->readOnly || !IsOnlyOneConnection(pPager)) { + return SQLITE_OK; + } + #ifdef SQLITE_HAS_CODEC +@@ -261660,7 +261770,7 @@ DWR_OPEN_OUT: + static void MetaDwrDisable(Btree *pBt) + { + Pager *pPager = pBt->pBt->pPager; +- if (pPager->metaFd == NULL || pPager->memDb || pPager->readOnly || !IsConnectionValidForCheck(pPager)) { ++ if (pPager->metaFd == NULL || pPager->memDb || pPager->readOnly || !IsOnlyOneConnection(pPager)) { + return; + } + #ifdef SQLITE_HAS_CODEC +@@ -261685,19 +261795,6 @@ static void MetaDwrDisable(Btree *pBt) + } + #endif + #if SQLITE_OS_UNIX +-#include +-#include +-static inline int OsGetTid(void) +-{ +-#if defined(__linux__) +- return (int)syscall(__NR_gettid); +-#elif defined(__APPLE__) +- return (int)syscall(SYS_thread_selfid); +-#else +- return 0; +-#endif +-} +- + static void ResetLockStatus(void) + { + (void)memset(&g_lockStatus, 0, sizeof(g_lockStatus)); +-- +2.28.0.windows.1 + diff --git a/patch/0008-BugFix-CurrVersion.patch b/patch/0009-Current-bug-fixes.patch similarity index 88% rename from patch/0008-BugFix-CurrVersion.patch rename to patch/0009-Current-bug-fixes.patch index d63cab1..ad01fb0 100644 --- a/patch/0008-BugFix-CurrVersion.patch +++ b/patch/0009-Current-bug-fixes.patch @@ -1,18 +1,17 @@ -From 0cb4346c98013b6276d1fda601871627bbd9e9c4 Mon Sep 17 00:00:00 2001 -From: Liu Hongyang -Date: Mon, 28 Apr 2025 19:02:48 +0800 -Subject: [PATCH] bugfix +From e66c70c937aadeddebc36fd9cb6873494395960a Mon Sep 17 00:00:00 2001 +From: ryne3366 +Date: Tue, 6 May 2025 17:16:10 +0800 +Subject: [PATCH 2/2] Current bug fixes -Signed-off-by: Liu Hongyang --- src/sqlite3.c | 120 ++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 96 insertions(+), 24 deletions(-) diff --git a/src/sqlite3.c b/src/sqlite3.c -index 3a6a372..6394950 100644 +index 97c8f86..41cac6f 100644 --- a/src/sqlite3.c +++ b/src/sqlite3.c -@@ -45357,7 +45357,7 @@ static int unixOpen( +@@ -45364,7 +45364,7 @@ static int unixOpen( flags |= SQLITE_OPEN_READONLY; openFlags |= O_RDONLY; isReadonly = 1; @@ -21,7 +20,7 @@ index 3a6a372..6394950 100644 pReadonly = findReusableFd(zName, flags); if( pReadonly ){ fd = pReadonly->fd; -@@ -70288,6 +70288,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( +@@ -70298,6 +70298,7 @@ SQLITE_PRIVATE int sqlite3WalFrames( ** If parameter xBusy is not NULL, it is a pointer to a busy-handler ** callback. In this case this function runs a blocking checkpoint. */ @@ -29,7 +28,7 @@ index 3a6a372..6394950 100644 SQLITE_PRIVATE int sqlite3WalCheckpoint( Wal *pWal, /* Wal connection */ sqlite3 *db, /* Check this handle's interrupt flag */ -@@ -70383,7 +70384,12 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( +@@ -70393,7 +70394,12 @@ SQLITE_PRIVATE int sqlite3WalCheckpoint( -1, 0, zMsg, NULL); rc = SQLITE_CORRUPT_REPORT(&context); }else{ @@ -43,7 +42,7 @@ index 3a6a372..6394950 100644 } /* If no error occurred, set the output variables. */ -@@ -206091,6 +206097,39 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ +@@ -206101,6 +206107,39 @@ static int fts3ExprTermOffsetInit(Fts3Expr *pExpr, int iPhrase, void *ctx){ return rc; } @@ -83,7 +82,7 @@ index 3a6a372..6394950 100644 /* ** Implementation of offsets() function. */ -@@ -206127,6 +206166,12 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( +@@ -206137,6 +206176,12 @@ SQLITE_PRIVATE void sqlite3Fts3Offsets( sCtx.iDocid = pCsr->iPrevId; sCtx.pCsr = pCsr; @@ -96,7 +95,7 @@ index 3a6a372..6394950 100644 /* Loop through the table columns, appending offset information to ** string-buffer res for each column. */ -@@ -260730,7 +260775,7 @@ static int PragmaMetaDoubleWrie(sqlite3 *db, int iDb, Parse *parse, const char * +@@ -260855,7 +260900,7 @@ static int PragmaMetaDoubleWrie(sqlite3 *db, int iDb, Parse *parse, const char * sqlite3_mutex_enter(db->mutex); // only support enabled meta double write int rc = MetaDwrOpenAndCheck(pBt); @@ -105,7 +104,7 @@ index 3a6a372..6394950 100644 parse->nErr++; parse->rc = rc; } -@@ -261182,6 +261227,29 @@ static inline const char *GetMetaFilePath(Pager *pPager) +@@ -261307,6 +261352,29 @@ static inline const char *GetMetaFilePath(Pager *pPager) return pPager->metaFd == NULL ? NULL : ((const char *)pPager->metaFd + ROUND8(pPager->pVfs->szOsFile)); } @@ -135,7 +134,7 @@ index 3a6a372..6394950 100644 static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { if (pPager->metaFd || pPager->zFilename == NULL) { return SQLITE_OK; -@@ -261198,15 +261266,10 @@ static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { +@@ -261323,15 +261391,10 @@ static int MetaDwrOpenFile(Pager *pPager, u8 openCreate) { return SQLITE_NOMEM_BKPT; } sqlite3_snprintf(size, metaPath, "%s-dwr", pPager->zFilename); @@ -152,7 +151,7 @@ index 3a6a372..6394950 100644 u32 flags = (SQLITE_OPEN_READWRITE | SQLITE_OPEN_SUPER_JOURNAL); if (openCreate) { flags |= SQLITE_OPEN_CREATE; -@@ -261765,16 +261828,6 @@ static inline void MarkLockStatusByRc(int rc, u32 lockIdx, u32 lockLen, u8 lockT +@@ -261862,16 +261925,6 @@ static inline void MarkLockStatusByRc(int rc, u32 lockIdx, u32 lockLen, u8 lockT } } @@ -169,7 +168,7 @@ index 3a6a372..6394950 100644 static inline const char *IdxToLockName(u32 idx) { const char *lockName[MAX_LOCK_NUM] = {"write", "ckpt", "recover", "read0", -@@ -261791,7 +261844,7 @@ static void DumpHandleLock(char *dumpBuf, int dumpBufLen) +@@ -261888,7 +261941,7 @@ static void DumpHandleLock(char *dumpBuf, int dumpBufLen) for (int i = 0; i < MAX_LOCK_NUM && availLen > DUMP_MAX_STR_LEN; i++) { if (lockStatus[i] != NO_LOCK) { tmp[0] = '\0'; @@ -178,7 +177,7 @@ index 3a6a372..6394950 100644 int len = strlen(tmp); tmp += len; availLen -= len; -@@ -261830,8 +261883,8 @@ static void DumpTrxProcessLocks(unixFile *file, char *dumpBuf, int dumpBufLen) +@@ -261927,8 +261980,8 @@ static void DumpTrxProcessLocks(unixFile *file, char *dumpBuf, int dumpBufLen) sqlite3_log(SQLITE_ERROR, "[SQLite]Inode is null!"); return; } @@ -189,7 +188,7 @@ index 3a6a372..6394950 100644 const char *lockName[DB_LOCK_NUM] = {"pending", "reserved", "shared_first"}; char *tmp = dumpBuf; int availLen = dumpBufLen - 1; -@@ -261851,10 +261904,13 @@ static void DumpTrxProcessLocks(unixFile *file, char *dumpBuf, int dumpBufLen) +@@ -261948,10 +262001,13 @@ static void DumpTrxProcessLocks(unixFile *file, char *dumpBuf, int dumpBufLen) static void DumpWalLocks(unixFile *file, u8 walEnabled, char *dumpBuf, int dumpBufLen) { @@ -205,7 +204,7 @@ index 3a6a372..6394950 100644 unixShmNode *pShmNode = file->pShm->pShmNode; char *tmp = dumpBuf; int availLen = dumpBufLen - 1; -@@ -262789,6 +262845,22 @@ SQLITE_PRIVATE void sqlite3BinlogErrorCallback(sqlite3 *db, int errNo, char *err +@@ -262886,6 +262942,22 @@ SQLITE_PRIVATE void sqlite3BinlogErrorCallback(sqlite3 *db, int errNo, char *err } /************** End of binlog implement ************************************/ @@ -229,5 +228,5 @@ index 3a6a372..6394950 100644 #ifdef SQLITE_EXPORT_SYMBOLS #ifndef SQLITE_CKSUMVFS_STATIC -- -2.25.1 +2.28.0.windows.1 -- Gitee