taosdata
/
TDengine
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix[TD-34340]: enable page recyling and ensure TDB cell size is at least the size of a free cell (#30606)

This commit is contained in:
Bomin Zhang 2025-04-03 13:28:09 +08:00 committed by GitHub
parent 337a4cebf1
commit a031d3ddd4
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 57 additions and 111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
source/dnode/vnode/src/vnd/vnodeSync.c
View File

@ -121,6 +121,7 @@ static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak
bool wait = (code == 0 && vnodeIsMsgBlock(pMsg->msgType));
if (wait) {
if (pVnode->blocked) {
(void)taosThreadMutexUnlock(&pVnode->lock);
return TSDB_CODE_INTERNAL_ERROR;
}
pVnode->blocked = true;

105
source/libs/tdb/src/db/tdbBtree.c
View File

@ -181,12 +181,7 @@ void tdbBtreeClose(SBTree *pBt) {
int tdbBtreeInsert(SBTree *pBt, const void *pKey, int kLen, const void *pVal, int vLen, TXN *pTxn) {
SBTC btc;
SCell *pCell;
void *pBuf;
int szCell;
int szBuf;
int ret;
int idx;
int c;
ret = tdbBtcOpen(&btc, pBt, pTxn);
@ -1663,70 +1658,70 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN *
vLen = pPage->vLen;
}
int nPayload = kLen + vLen;
if (nHeader + nPayload <= pPage->maxLocal) {
return nHeader + nPayload;
} else {
int maxLocal = pPage->maxLocal;
int nSize = kLen + vLen + nHeader;
if (nSize <= pPage->maxLocal) {
// cell size should be at least the size of a free cell, otherwise it
// cannot be added to the free list when dropped.
if (nSize < pPage->pPageMethods->szFreeCell) {
return pPage->pPageMethods->szFreeCell;
}
return nSize;
}
// calc local storage size
int minLocal = pPage->minLocal;
int surplus = minLocal + (nPayload + nHeader - minLocal) % (maxLocal - sizeof(SPgno));
int nLocal = surplus <= maxLocal ? surplus : minLocal;
// handle overflow pages
int maxLocal = pPage->maxLocal;
// free ofp pages' cells
if (dropOfp) {
int ret = 0;
SPgno pgno = *(SPgno *)(pCell + nLocal - sizeof(SPgno));
int nLeft = nPayload - nLocal + sizeof(SPgno) + nHeader;
SPage *ofp;
int bytes;
// calc local storage size
int minLocal = pPage->minLocal;
int surplus = minLocal + (nSize - minLocal) % (maxLocal - sizeof(SPgno));
int nLocal = surplus <= maxLocal ? surplus : minLocal;
while (pgno != 0) {
ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
if (ret < 0) {
return ret;
}
if (!dropOfp) {
return nLocal;
}
SCell *ofpCell = tdbPageGetCell(ofp, 0);
// free ofp pages' cells
SPgno pgno = *(SPgno *)(pCell + nLocal - sizeof(SPgno));
int nLeft = nSize - nLocal + sizeof(SPgno);
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
bytes = nLeft;
} else {
bytes = ofp->maxLocal - sizeof(SPgno);
}
while (pgno != 0) {
SPage *ofp;
int bytes;
int ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt);
if (ret < 0) {
return ret;
}
// SPgno origPgno = pgno;
memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
SCell *ofpCell = tdbPageGetCell(ofp, 0);
ret = tdbPagerWrite(pBt->pPager, ofp);
if (ret < 0) {
tdbError("failed to write page since %s", terrstr());
return ret;
}
/*
tdbPageDropCell(ofp, 0, pTxn, pBt);
*/
// SIntHdr *pIntHdr = (SIntHdr *)(ofp->pData);
// pIntHdr->flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL);
// pIntHdr->pgno = 0;
// ofp->pPager = NULL;
if (nLeft <= ofp->maxLocal - sizeof(SPgno)) {
bytes = nLeft;
} else {
bytes = ofp->maxLocal - sizeof(SPgno);
}
SArray *ofps = pPage->pPager->ofps;
if (ofps) {
if (taosArrayPush(ofps, &ofp) == NULL) {
return terrno;
}
}
// SPgno origPgno = pgno;
memcpy(&pgno, ofpCell + bytes, sizeof(pgno));
tdbPagerReturnPage(pPage->pPager, ofp, pTxn);
ret = tdbPagerWrite(pBt->pPager, ofp);
if (ret < 0) {
tdbError("failed to write page since %s", terrstr());
return ret;
}
nLeft -= bytes;
SArray *ofps = pPage->pPager->ofps;
if (ofps) {
if (taosArrayPush(ofps, &ofp) == NULL) {
return terrno;
}
}
return nLocal;
tdbPagerReturnPage(pPage->pPager, ofp, pTxn);
nLeft -= bytes;
}
return nLocal;
}
// TDB_BTREE_CELL

14
source/libs/tdb/src/db/tdbPage.c
View File

@ -491,15 +491,13 @@ static int tdbPageFree(SPage *pPage, int idx, SCell *pCell, int szCell) {
if (pCell == pPage->pFreeEnd) {
pPage->pFreeEnd += szCell;
TDB_PAGE_CCELLS_SET(pPage, pPage->pFreeEnd - pPage->pData);
} else if (szCell >= TDB_PAGE_FREE_CELL_SIZE(pPage)) {
cellFree = TDB_PAGE_FCELL(pPage);
pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree);
TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData);
} else {
if (szCell >= TDB_PAGE_FREE_CELL_SIZE(pPage)) {
cellFree = TDB_PAGE_FCELL(pPage);
pPage->pPageMethods->setFreeCellInfo(pCell, szCell, cellFree);
TDB_PAGE_FCELL_SET(pPage, pCell - pPage->pData);
} else {
tdbError("tdb/page-free: invalid cell size: %d", szCell);
return TSDB_CODE_INVALID_PARA;
}
tdbError("tdb/page-free: invalid cell size: %d", szCell);
return TSDB_CODE_INVALID_PARA;
}
dest = pPage->pCellIdx + TDB_PAGE_OFFSET_SIZE(pPage) * idx;

43
source/libs/tdb/src/db/tdbPager.c
View File

@ -746,50 +746,15 @@ int tdbPagerInsertFreePage(SPager *pPager, SPage *pPage, TXN *pTxn) {
int code = 0;
SPgno pgno = TDB_PAGE_PGNO(pPage);
#if ENABLE_PAGE_RECYLING
if (pPager->frps) {
if (taosArrayPush(pPager->frps, &pgno) == NULL) {
return terrno;
}
pPage->pPager = NULL;
return code;
}
pPager->frps = taosArrayInit(8, sizeof(SPgno));
if (pPager->frps == NULL) {
return terrno;
}
// memset(pPage->pData, 0, pPage->pageSize);
tdbTrace("tdb/insert-free-page: tbc recycle page: %d.", pgno);
// printf("tdb/insert-free-page: tbc recycle page: %d.\n", pgno);
code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn);
if (code < 0) {
tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
taosArrayDestroy(pPager->frps);
pPager->frps = NULL;
return code;
}
while (TARRAY_SIZE(pPager->frps) > 0) {
pgno = *(SPgno *)taosArrayPop(pPager->frps);
code = tdbTbInsert(pPager->pEnv->pFreeDb, &pgno, sizeof(pgno), NULL, 0, pTxn);
if (code < 0) {
tdbError("tdb/insert-free-page: tb insert failed with ret: %d.", code);
taosArrayDestroy(pPager->frps);
pPager->frps = NULL;
return code;
}
}
taosArrayDestroy(pPager->frps);
pPager->frps = NULL;
pPage->pPager = NULL;
#endif
return code;
}
@ -801,10 +766,6 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
return code;
}
if (pPager->frps) {
return code;
}
code = tdbTbcOpen(pPager->pEnv->pFreeDb, &pCur, pTxn);
if (code < 0) {
return code;
@ -843,11 +804,7 @@ static int tdbPagerRemoveFreePage(SPager *pPager, SPgno *pPgno, TXN *pTxn) {
static int tdbPagerAllocFreePage(SPager *pPager, SPgno *ppgno, TXN *pTxn) {
// Allocate a page from the free list
#if ENABLE_PAGE_RECYLING
return tdbPagerRemoveFreePage(pPager, ppgno, pTxn);
#else
return 0;
#endif
}
static int tdbPagerAllocNewPage(SPager *pPager, SPgno *ppgno) {

1
source/libs/tdb/src/inc/tdbInt.h
View File

@ -402,7 +402,6 @@ struct SPager {
// u8 inTran;
TXN *pActiveTxn;
SArray *ofps;
SArray *frps;
SPager *pNext; // used by TDB
SPager *pHashNext; // used by TDB
#ifdef USE_MAINDB

4
source/os/src/osFile.c
View File

@ -52,7 +52,6 @@ typedef int32_t FileFd;
#ifdef WINDOWS
typedef struct TdFile {
TdThreadRwlock rwlock;
int refId;
HANDLE hFile;
FILE *fp;
int32_t tdFileOptions;
@ -60,7 +59,6 @@ typedef struct TdFile {
#else
typedef struct TdFile {
TdThreadRwlock rwlock;
int refId;
FileFd fd;
FILE *fp;
} TdFile;
@ -1196,7 +1194,6 @@ TdFilePtr taosOpenFile(const char *path, int32_t tdFileOptions) {
(void)taosThreadRwlockInit(&(pFile->rwlock), NULL);
#endif
pFile->fp = fp;
pFile->refId = 0;
#ifdef WINDOWS
pFile->hFile = hFile;
@ -1252,7 +1249,6 @@ int32_t taosCloseFile(TdFilePtr *ppFile) {
(*ppFile)->fd = -1;
#endif
}
(*ppFile)->refId = 0;
#if FILE_WITH_LOCK
(void)taosThreadRwlockUnlock(&((*ppFile)->rwlock));
(void)taosThreadRwlockDestroy(&((*ppFile)->rwlock));