diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 9b96ebe7cb73..2d6843ececf8 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -218,5 +218,9 @@ else () endif() MESSAGE(STATUS "DEPS_DIR: " ${TD_DEPS_DIR}) +if(${TD_ACORE}) + add_definitions(-DTD_ACORE) +endif(${TD_ACORE}) + MESSAGE("C Compiler: ${CMAKE_C_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_C_COMPILER_VERSION})") MESSAGE("CXX Compiler: ${CMAKE_CXX_COMPILER} (${CMAKE_C_COMPILER_ID}, ${CMAKE_CXX_COMPILER_VERSION})") diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 5125c1caefce..8f0440c90991 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -124,6 +124,7 @@ extern bool tsEnableWhiteList; extern int64_t tsDndStart; extern int64_t tsDndStartOsUptime; extern int64_t tsDndUpTime; +extern int64_t tsDndStarted; // dnode misc extern uint32_t tsEncryptionKeyChksum; @@ -243,6 +244,9 @@ extern int64_t tsmaDataDeleteMark; // wal extern int64_t tsWalFsyncDataSizeLimit; +// misc +extern bool tsAcoreOS; + // internal extern int32_t tsTransPullupInterval; extern int32_t tsCompactPullupInterval; @@ -272,9 +276,9 @@ extern bool tsExperimental; // #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, - const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc); -int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, - SArray *pArgs); + const char *envFile, char *apolloUrl, SArray *pArgs, ELogMode mode); +int32_t taosReadDataFolder(const char *cfgDir, const char **envCmd, + const char *envFile, char *apolloUrl, SArray *pArgs); int32_t taosInitCfg(const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc); void taosCleanupCfg(); diff --git a/include/libs/transport/trpc.h b/include/libs/transport/trpc.h index cfa3f44f7fa7..863c10e4638b 100644 --- a/include/libs/transport/trpc.h +++ b/include/libs/transport/trpc.h @@ -19,6 +19,7 @@ extern "C" { #endif +#ifndef TD_ACORE #include #include #include "taosdef.h" @@ -190,6 +191,193 @@ int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); int32_t rpcCvtErrCode(int32_t code); +#else +#include +#include +#include "taosdef.h" +#include "tmsg.h" +#include "ttrace.h" + +#define TAOS_CONN_SERVER 0 +#define TAOS_CONN_CLIENT 1 +#define IsReq(pMsg) (pMsg->msgType & 1U) + +extern int32_t tsRpcHeadSize; + +typedef struct { + uint32_t clientIp; + uint16_t clientPort; + int64_t applyIndex; + uint64_t applyTerm; + char user[TSDB_USER_LEN]; +} SRpcConnInfo; + +typedef enum { + TD_ACORE_CLIENT = 1, + TD_ACORE_DSVR_CLIENT = 2, + TD_ACORE_DSVR_STA_CLIENT = 4, + TD_ACORE_DSVR_SYNC_CLIENT = 8, + TD_ACORE_DSVR = 16 +} RPC_TYPE; + +typedef struct SRpcHandleInfo { + // rpc info + void *handle; // rpc handle returned to app + int64_t refId; // refid, used by server + int8_t noResp; // has response or not(default 0, 0: resp, 1: no resp) + int8_t persistHandle; // persist handle or not + int8_t hasEpSet; + int32_t cliVer; + + // app info + void *ahandle; // app handle set by client + void *wrapper; // wrapper handle + void *node; // node mgmt handle + + // resp info + void *rsp; + int32_t rspLen; + + STraceId traceId; + + SRpcConnInfo conn; + int8_t forbiddenIp; + int8_t notFreeAhandle; + int8_t compressed; + int16_t connType; + int64_t seq; + int64_t qId; + int32_t msgType; + void *reqWithSem; + int32_t refIdMgt; +} SRpcHandleInfo; + +typedef struct SRpcMsg { + tmsg_t msgType; + void *pCont; + int32_t contLen; + int32_t code; + int32_t type; + void *parent; + SRpcHandleInfo info; + +} SRpcMsg; + +typedef void (*RpcCfp)(void *parent, SRpcMsg *, SEpSet *epset); +typedef bool (*RpcRfp)(int32_t code, tmsg_t msgType); +typedef bool (*RpcTfp)(int32_t code, tmsg_t msgType); +typedef bool (*RpcFFfp)(tmsg_t msgType); +typedef bool (*RpcNoDelayfp)(tmsg_t msgType); +typedef void (*RpcDfp)(void *ahandle); + +typedef struct SRpcInit { + char localFqdn[TSDB_FQDN_LEN]; + uint16_t localPort; // local port + char *label; // for debug purpose + int32_t numOfThreads; // number of threads to handle connections + int32_t sessions; // number of sessions allowed + int8_t connType; // TAOS_CONN_UDP, TAOS_CONN_TCPC, TAOS_CONN_TCPS + int32_t idleTime; // milliseconds, 0 means idle timer is disabled + int32_t compatibilityVer; + + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int64_t retryMaxTimeout; + + int32_t failFastThreshold; + int32_t failFastInterval; + + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size + int8_t encryption; // encrypt or not + + // the following is for client app ecurity only + char *user; // user name + + // call back to process incoming msg + RpcCfp cfp; + + // retry not not for particular msg + RpcRfp rfp; + + // set up timeout for particular msg + RpcTfp tfp; + + // destroy client ahandle; + RpcDfp dfp; + // fail fast fp + RpcFFfp ffp; + + RpcNoDelayfp noDelayFp; + + int32_t connLimitNum; + int32_t connLimitLock; + int32_t timeToGetConn; + int8_t supportBatch; // 0: no batch, 1. batch + int32_t batchSize; + int8_t notWaitAvaliableConn; // 1: wait to get, 0: no wait + int32_t shareConnLimit; + int8_t shareConn; // 0: no share, 1. share + int8_t startReadTimer; + int64_t readTimeout; // s + void *parent; + +} SRpcInit; + +typedef struct { + void *val; + int32_t (*clone)(void *src, void **dst); +} SRpcCtxVal; + +typedef struct { + int32_t msgType; + void *val; + int32_t (*clone)(void *src, void **dst); +} SRpcBrokenlinkVal; + +typedef struct { + SHashObj *args; + SRpcBrokenlinkVal brokenVal; + void (*freeFunc)(const void *arg); + int64_t st; +} SRpcCtx; + +int32_t rpcInit(); +void rpcCleanup(); + +void *rpcOpen(const SRpcInit *pRpc); +void rpcClose(void *); +void rpcCloseImpl(void *); +void *rpcMallocCont(int64_t contLen); +void rpcFreeCont(void *pCont); +void *rpcReallocCont(void *ptr, int64_t contLen); + +// Because taosd supports multi-process mode +// These functions should not be used on the server side +// Please use tmsg functions, which are defined in tmsgcb.h +int32_t rpcSendRequest(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid); +int32_t rpcSendResponse(SRpcMsg *pMsg); +int32_t rpcRegisterBrokenLinkArg(SRpcMsg *msg); +int32_t rpcReleaseHandle(void *handle, int8_t type); // just release conn to rpc instance, no close sock + +// These functions will not be called in the child process +int32_t rpcSendRequestWithCtx(void *thandle, const SEpSet *pEpSet, SRpcMsg *pMsg, int64_t *rid, SRpcCtx *ctx); +int32_t rpcSendRecv(void *shandle, SEpSet *pEpSet, SRpcMsg *pReq, SRpcMsg *pRsp); +int32_t rpcSendRecvWithTimeout(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp, int8_t *epUpdated, + int32_t timeoutMs); + +int32_t rpcFreeConnById(void *shandle, int64_t connId); + +int32_t rpcSetDefaultAddr(void *thandle, const char *ip, const char *fqdn); +int32_t rpcAllocHandle(int64_t *refId); +int32_t rpcSetIpWhite(void *thandl, void *arg); + +int32_t rpcUtilSIpRangeToStr(SIpV4Range *pRange, char *buf); + +int32_t rpcUtilSWhiteListToStr(SIpWhiteList *pWhiteList, char **ppBuf); +int32_t rpcCvtErrCode(int32_t code); + +#endif #ifdef __cplusplus } #endif diff --git a/include/os/osSocket.h b/include/os/osSocket.h index 49acf285ee62..965f15a0ddf4 100644 --- a/include/os/osSocket.h +++ b/include/os/osSocket.h @@ -18,17 +18,17 @@ // If the error is in a third-party library, place this header file under the third-party library header file. // When you want to use this feature, you should find or add the same function in the following section. -#ifndef ALLOW_FORBID_FUNC -#define socket SOCKET_FUNC_TAOS_FORBID -#define bind BIND_FUNC_TAOS_FORBID -#define listen LISTEN_FUNC_TAOS_FORBID -#define accept ACCEPT_FUNC_TAOS_FORBID -#define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID -#define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID -#define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID -#define inet_addr INET_ADDR_FUNC_TAOS_FORBID -#define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID -#endif +// #ifndef ALLOW_FORBID_FUNC +// #define socket SOCKET_FUNC_TAOS_FORBID +// #define bind BIND_FUNC_TAOS_FORBID +// #define listen LISTEN_FUNC_TAOS_FORBID +// #define accept ACCEPT_FUNC_TAOS_FORBID +// #define epoll_create EPOLL_CREATE_FUNC_TAOS_FORBID +// #define epoll_ctl EPOLL_CTL_FUNC_TAOS_FORBID +// #define epoll_wait EPOLL_WAIT_FUNC_TAOS_FORBID +// #define inet_addr INET_ADDR_FUNC_TAOS_FORBID +// #define inet_ntoa INET_NTOA_FUNC_TAOS_FORBID +// #endif #if defined(WINDOWS) #if BYTE_ORDER == LITTLE_ENDIAN diff --git a/include/util/tlog.h b/include/util/tlog.h index 78ca6ffe15c6..4dd3e8561a69 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -34,6 +34,12 @@ typedef enum { DEBUG_FILE = 128 } ELogLevel; +typedef enum { + LOG_MODE_TAOSC = 1, + LOG_MODE_TAOSD = 2, + LOG_MODE_BOTH = 3, // LOG_MODE_TAOSC | LOG_MODE_TAOSD +} ELogMode; + typedef void (*LogFp)(int64_t ts, ELogLevel level, const char *content); extern bool tsLogEmbedded; @@ -73,7 +79,7 @@ extern int32_t simDebugFlag; extern int32_t tqClientDebugFlag; int32_t taosInitLogOutput(const char **ppLogName); -int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc); +int32_t taosInitLog(const char *logName, int32_t maxFiles, ELogMode mode); void taosCloseLog(); void taosResetLog(); void taosDumpData(uint8_t *msg, int32_t len); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 072cbee2f4a3..2f3160833378 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -960,13 +960,13 @@ void taos_init_imp(void) { const char *logName = CUS_PROMPT "slog"; ENV_ERR_RET(taosInitLogOutput(&logName), "failed to init log output"); - if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, 1) != 0) { + if (taosCreateLog(logName, 10, configDir, NULL, NULL, NULL, NULL, tsAcoreOS ? LOG_MODE_BOTH : LOG_MODE_TAOSC) != 0) { (void)printf(" WARING: Create %s failed:%s. configDir=%s\n", logName, strerror(errno), configDir); tscInitRes = -1; return; } - ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1), "failed to init cfg"); + ENV_ERR_RET(taosInitCfg(configDir, NULL, NULL, NULL, NULL, tsAcoreOS ? 0 : 1), "failed to init cfg"); initQueryModuleMsgHandle(); ENV_ERR_RET(taosConvInit(), "failed to init conv"); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 1f41a9b0b0fc..77c84ad5fc48 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -93,7 +93,7 @@ void taos_cleanup(void) { tscInfo("all local resources released"); taosCleanupCfg(); - taosCloseLog(); + if (!tsAcoreOS) taosCloseLog(); } static setConfRet taos_set_config_imp(const char *config) { diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 5076006f2956..f5cb6b839408 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -100,6 +100,7 @@ int32_t tsArbSetAssignedTimeoutSec = 30; int64_t tsDndStart = 0; int64_t tsDndStartOsUptime = 0; int64_t tsDndUpTime = 0; +int64_t tsDndStarted = 0; // dnode misc uint32_t tsEncryptionKeyChksum = 0; @@ -278,6 +279,9 @@ int32_t tsTtlFlushThreshold = 100; /* maximum number of dirty items in memory. */ int32_t tsTtlBatchDropNum = 10000; // number of tables dropped per batch +// misc +bool tsAcoreOS = false; + // internal int32_t tsTransPullupInterval = 2; int32_t tsCompactPullupInterval = 10; @@ -1731,24 +1735,26 @@ int32_t taosSetReleaseCfg(SConfig *pCfg); static int32_t taosSetAllDebugFlag(SConfig *pCfg, int32_t flag); +static int8_t tsLogCreated = 0; + int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, - const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { + const char *envFile, char *apolloUrl, SArray *pArgs, ELogMode mode) { int32_t code = TSDB_CODE_SUCCESS; int32_t lino = 0; SConfig *pCfg = NULL; + if (atomic_val_compare_exchange_8(&tsLogCreated, 0, 1) != 0) return 0; + if (tsCfg == NULL) { TAOS_CHECK_GOTO(osDefaultInit(), &lino, _exit); } TAOS_CHECK_GOTO(cfgInit(&pCfg), &lino, _exit); - if (tsc) { - tsLogEmbedded = 0; - TAOS_CHECK_GOTO(taosAddClientLogCfg(pCfg), &lino, _exit); - } else { - tsLogEmbedded = 1; - TAOS_CHECK_GOTO(taosAddClientLogCfg(pCfg), &lino, _exit); + tsLogEmbedded = (mode & LOG_MODE_TAOSC) ? 0 : 1; + TAOS_CHECK_GOTO(taosAddClientLogCfg(pCfg), &lino, _exit); + + if (mode & LOG_MODE_TAOSD) { TAOS_CHECK_GOTO(taosAddServerLogCfg(pCfg), &lino, _exit); } @@ -1762,10 +1768,8 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi goto _exit; } - if (tsc) { - TAOS_CHECK_GOTO(taosSetClientLogCfg(pCfg), &lino, _exit); - } else { - TAOS_CHECK_GOTO(taosSetClientLogCfg(pCfg), &lino, _exit); + TAOS_CHECK_GOTO(taosSetClientLogCfg(pCfg), &lino, _exit); + if (mode & LOG_MODE_TAOSD) { TAOS_CHECK_GOTO(taosSetServerLogCfg(pCfg), &lino, _exit); } @@ -1778,7 +1782,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi goto _exit; } - if ((code = taosInitLog(logname, logFileNum, tsc)) != 0) { + if ((code = taosInitLog(logname, logFileNum, mode)) != 0) { (void)printf("failed to init log file since %s\n", tstrerror(code)); goto _exit; } diff --git a/source/dnode/mgmt/CMakeLists.txt b/source/dnode/mgmt/CMakeLists.txt index 5d356e06b1be..aec377778dbf 100644 --- a/source/dnode/mgmt/CMakeLists.txt +++ b/source/dnode/mgmt/CMakeLists.txt @@ -8,7 +8,12 @@ add_subdirectory(mgmt_dnode) add_subdirectory(test) aux_source_directory(exe EXEC_SRC) -add_executable(taosd ${EXEC_SRC}) +if(${TD_ACORE}) + add_library(taosd ${EXEC_SRC}) +else() + add_executable(taosd ${EXEC_SRC}) +endif() + target_include_directories( taosd PRIVATE "${CMAKE_CURRENT_SOURCE_DIR}/node_mgmt/inc" @@ -22,6 +27,11 @@ IF(TD_ENTERPRISE) ENDIF() ENDIF() + +IF(${TD_ACORE}) + add_definitions(-DTD_ACORE) +ENDIF(${TD_ACORE}) + IF(TD_LINUX_64 AND JEMALLOC_ENABLED) ADD_DEFINITIONS(-DTD_JEMALLOC_ENABLED -I${CMAKE_BINARY_DIR}/build/include -L${CMAKE_BINARY_DIR}/build/lib -Wl,-rpath,${CMAKE_BINARY_DIR}/build/lib -ljemalloc) SET(LINK_JEMALLOC "-L${CMAKE_BINARY_DIR}/build/lib -ljemalloc") diff --git a/source/dnode/mgmt/exe/dmMain.c b/source/dnode/mgmt/exe/dmMain.c index 4aa76d92845f..50689457b6e9 100644 --- a/source/dnode/mgmt/exe/dmMain.c +++ b/source/dnode/mgmt/exe/dmMain.c @@ -102,13 +102,17 @@ static void dmStopDnode(int signum, void *sigInfo, void *context) { dInfo("shut down signal is %d", signum); #ifndef WINDOWS - dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, - taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); + if (sigInfo) { + dInfo("sender PID:%d cmdline:%s", ((siginfo_t *)sigInfo)->si_pid, + taosGetCmdlineByPID(((siginfo_t *)sigInfo)->si_pid)); + } #endif dmStop(); } +void dmStopDaemon() { dmStopDnode(SIGTERM, NULL, NULL); } + void dmLogCrash(int signum, void *sigInfo, void *context) { // taosIgnSignal(SIGTERM); // taosIgnSignal(SIGHUP); @@ -374,14 +378,18 @@ static int32_t dmInitLog() { TAOS_CHECK_RETURN(taosInitLogOutput(&logName)); - return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, 0); + return taosCreateLog(logName, 1, configDir, global.envCmd, global.envFile, global.apolloUrl, global.pArgs, LOG_MODE_TAOSD); } static void taosCleanupArgs() { if (global.envCmd != NULL) taosMemoryFreeClear(global.envCmd); } +#ifdef TD_ACORE +int dmStartDaemon(int argc, char const *argv[]) { +#else int main(int argc, char const *argv[]) { +#endif int32_t code = 0; #ifdef TD_JEMALLOC_ENABLED bool jeBackgroundThread = true; diff --git a/source/dnode/mgmt/node_mgmt/src/dmNodes.c b/source/dnode/mgmt/node_mgmt/src/dmNodes.c index 3cb9030f6071..c33a9e137bdb 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmNodes.c +++ b/source/dnode/mgmt/node_mgmt/src/dmNodes.c @@ -155,6 +155,12 @@ int32_t dmRunDnode(SDnode *pDnode) { return code; } + int64_t dndStarted = taosGetTimestampMs(); + if (dndStarted == 0) { + ++dndStarted; + } + atomic_store_64(&tsDndStarted, dndStarted); + while (1) { if (pDnode->stop) { dInfo("The daemon is about to stop"); diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 13c8c73f445b..26721f020aa9 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -37,7 +37,7 @@ void Testbase::InitLog(const char* path) { taosGetSystemInfo(); tsQueueMemoryAllowed = tsTotalMemoryKB * 0.1; - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log file\n"); } } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index fd02367f6dc5..f78cd0d70343 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -14,12 +14,12 @@ */ #define _DEFAULT_SOURCE +#include "mndProfile.h" #include "audit.h" #include "mndDb.h" #include "mndDnode.h" #include "mndMnode.h" #include "mndPrivilege.h" -#include "mndProfile.h" #include "mndQnode.h" #include "mndShow.h" #include "mndSma.h" diff --git a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp index 97b7ccd5f768..a68517ddacae 100644 --- a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp +++ b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp @@ -35,7 +35,7 @@ class MndTestSdb : public ::testing::Test { taosRemoveDir(path); taosMkDir(path); tstrncpy(tsLogDir, path, PATH_MAX); - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log file\n"); } } diff --git a/source/dnode/mnode/impl/test/stream/stream.cpp b/source/dnode/mnode/impl/test/stream/stream.cpp index 45bc4c2ce291..cc5d5cd7904f 100644 --- a/source/dnode/mnode/impl/test/stream/stream.cpp +++ b/source/dnode/mnode/impl/test/stream/stream.cpp @@ -264,7 +264,7 @@ TEST_F(StreamTest, plan_Test) { SNode * pAst = NULL; SQueryPlan *pPlan = NULL; - if (taosCreateLog("taoslog", 10, "/etc/taos", NULL, NULL, NULL, NULL, 1) != 0) { + if (taosCreateLog("taoslog", 10, "/etc/taos", NULL, NULL, NULL, NULL, LOG_MODE_TAOSC) != 0) { // ignore create log failed, only print (void) printf(" WARING: Create failed:%s. configDir\n", strerror(errno)); } diff --git a/source/dnode/mnode/impl/test/trans/trans2.cpp b/source/dnode/mnode/impl/test/trans/trans2.cpp index b73f07c77886..186a5319cf0e 100644 --- a/source/dnode/mnode/impl/test/trans/trans2.cpp +++ b/source/dnode/mnode/impl/test/trans/trans2.cpp @@ -52,7 +52,7 @@ class MndTestTrans2 : public ::testing::Test { taosRemoveDir(logpath); taosMkDir(logpath); tstrncpy(tsLogDir, logpath, PATH_MAX); - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log file\n"); } } diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 25c82b84526c..607380b64adb 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -162,7 +162,7 @@ void ctgTestInitLogFile() { (void)ctgdEnableDebug("cache", true); (void)ctgdEnableDebug("lock", true); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, LOG_MODE_TAOSC) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); ASSERT(0); } diff --git a/source/libs/executor/test/joinTests.cpp b/source/libs/executor/test/joinTests.cpp index 6dd42c3a93a6..afdb617fd0e8 100755 --- a/source/libs/executor/test/joinTests.cpp +++ b/source/libs/executor/test/joinTests.cpp @@ -2884,7 +2884,7 @@ void jtInitLogFile() { qDebugFlag = 159; TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { JT_PRINTF("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/executor/test/queryPlanTests.cpp b/source/libs/executor/test/queryPlanTests.cpp index 69097ce755d2..fe822385ee07 100755 --- a/source/libs/executor/test/queryPlanTests.cpp +++ b/source/libs/executor/test/queryPlanTests.cpp @@ -3005,7 +3005,7 @@ void qptInitLogFile() { qDebugFlag = 159; TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_BOTH ) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 4efa8764e56a..41250e3124a2 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -65,7 +65,7 @@ void udfUdfdExit(uv_process_t *process, int64_t exitStatus, int32_t termSignal) TAOS_UDF_CHECK_PTR_RVOID(process); fnInfo("udfd process exited with status %" PRId64 ", signal %d", exitStatus, termSignal); SUdfdData *pData = process->data; - if(pData == NULL) { + if (pData == NULL) { fnError("udfd process data is NULL"); return; } @@ -410,7 +410,7 @@ typedef void *QUEUE[2]; #define QUEUE_NEXT_PREV(q) (QUEUE_PREV(QUEUE_NEXT(q))) /* Public macros. */ -#define QUEUE_DATA(ptr, type, field) ((type *)((char *)(ptr) - offsetof(type, field))) +#define QUEUE_DATA(ptr, type, field) ((type *)((char *)(ptr)-offsetof(type, field))) /* Important note: mutating the list while QUEUE_FOREACH is * iterating over its elements results in undefined behavior. @@ -1934,7 +1934,8 @@ void udfcAsyncTaskCb(uv_async_t *async) { } void cleanUpUvTasks(SUdfcProxy *udfc) { - fnDebug("clean up uv tasks") QUEUE wq; + fnDebug("clean up uv tasks"); + QUEUE wq; uv_mutex_lock(&udfc->taskQueueMutex); QUEUE_MOVE(&udfc->taskQueue, &wq); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index bbfd43d5f7f9..07456085a8e0 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -1566,7 +1566,7 @@ static void udfdPrintVersion() { static int32_t udfdInitLog() { const char *logName = "udfdlog"; TAOS_CHECK_RETURN(taosInitLogOutput(&logName)); - return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, LOG_MODE_TAOSD); } void udfdCtrlAllocBufCb(uv_handle_t *handle, size_t suggested_size, uv_buf_t *buf) { diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index f28b44d1b8d5..780bdb3585c8 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -34,7 +34,7 @@ static int32_t parseArgs(int32_t argc, char *argv[]) { static int32_t initLog() { char logName[12] = {0}; snprintf(logName, sizeof(logName), "%slog", "udfc"); - return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, 0); + return taosCreateLog(logName, 1, configDir, NULL, NULL, NULL, NULL, LOG_MODE_TAOSD); } int scalarFuncTest() { diff --git a/source/libs/index/test/fstUT.cc b/source/libs/index/test/fstUT.cc index 04bc6c9a506b..fcbae0ccf56e 100644 --- a/source/libs/index/test/fstUT.cc +++ b/source/libs/index/test/fstUT.cc @@ -28,7 +28,7 @@ static void EnvInit() { taosMkDir(path.c_str()); // init log file tstrncpy(tsLogDir, path.c_str(), PATH_MAX); - if (taosInitLog("tindex.idx", 1, false) != 0) { + if (taosInitLog("tindex.idx", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log"); } // init index file diff --git a/source/libs/index/test/indexBench.cc b/source/libs/index/test/indexBench.cc index 369ed54d0bf0..b574acb33450 100644 --- a/source/libs/index/test/indexBench.cc +++ b/source/libs/index/test/indexBench.cc @@ -40,7 +40,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 090ff02ffb4e..f2d785eb78d0 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -283,7 +283,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/index/test/index_executor_tests.cpp b/source/libs/index/test/index_executor_tests.cpp index f65028e56449..7ad9dee4a151 100644 --- a/source/libs/index/test/index_executor_tests.cpp +++ b/source/libs/index/test/index_executor_tests.cpp @@ -57,7 +57,7 @@ void sifInitLogFile() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/index/test/jsonUT.cc b/source/libs/index/test/jsonUT.cc index 493101eb27ab..cdd1e6e478db 100644 --- a/source/libs/index/test/jsonUT.cc +++ b/source/libs/index/test/jsonUT.cc @@ -40,7 +40,7 @@ static void initLog() { taosRemoveDir(tsLogDir); taosMkDir(tsLogDir); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/parser/test/parTestMain.cpp b/source/libs/parser/test/parTestMain.cpp index 3986c96f076b..269330be2f1e 100644 --- a/source/libs/parser/test/parTestMain.cpp +++ b/source/libs/parser/test/parTestMain.cpp @@ -72,7 +72,7 @@ class ParserEnv : public testing::Environment { taosRemoveDir(path); ASSERT_EQ(TSDB_CODE_SUCCESS, taosMkDir(path)); tstrncpy(tsLogDir, path, PATH_MAX); - if (taosInitLog("taoslog", 1, false) != 0) { + if (taosInitLog("taoslog", 1, LOG_MODE_TAOSD) != 0) { std::cout << "failed to init log file" << std::endl; } } diff --git a/source/libs/planner/test/planTestMain.cpp b/source/libs/planner/test/planTestMain.cpp index 2a1e169d5f06..040b36470a42 100644 --- a/source/libs/planner/test/planTestMain.cpp +++ b/source/libs/planner/test/planTestMain.cpp @@ -67,7 +67,7 @@ class PlannerEnv : public testing::Environment { taosRemoveDir(path); ASSERT_EQ(TSDB_CODE_SUCCESS,taosMkDir(path)); tstrncpy(tsLogDir, path, PATH_MAX); - if (taosInitLog("taoslog", 1, false) != 0) { + if (taosInitLog("taoslog", 1, LOG_MODE_TAOSD) != 0) { std::cout << "failed to init log file" << std::endl; } } diff --git a/source/libs/qworker/test/qworkerTests.cpp b/source/libs/qworker/test/qworkerTests.cpp index 42b538b14fe3..5f462ed9e960 100644 --- a/source/libs/qworker/test/qworkerTests.cpp +++ b/source/libs/qworker/test/qworkerTests.cpp @@ -105,7 +105,7 @@ void qwtInitLogFile() { qDebugFlag = 159; strcpy(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 8bbadd0e228e..bede0cdafc60 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -57,7 +57,7 @@ void flttInitLogFile() { qDebugFlag = 159; tstrncpy(tsLogDir, TD_LOG_DIR_PATH, PATH_MAX); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index 4cab6445828b..c28e6c4560eb 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -83,7 +83,7 @@ void scltInitLogFile() { qDebugFlag = 159; tstrncpy(tsLogDir, TD_LOG_DIR_PATH, PATH_MAX); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/scheduler/test/schedulerTests.cpp b/source/libs/scheduler/test/schedulerTests.cpp index 6e13e37e88e8..19d9dab1ae7b 100644 --- a/source/libs/scheduler/test/schedulerTests.cpp +++ b/source/libs/scheduler/test/schedulerTests.cpp @@ -79,7 +79,7 @@ void schtInitLogFile() { qDebugFlag = 159; TAOS_STRCPY(tsLogDir, TD_LOG_DIR_PATH); - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, false) < 0) { + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum, LOG_MODE_TAOSD) < 0) { (void)printf("failed to open log file in directory:%s\n", tsLogDir); } } diff --git a/source/libs/sync/test/syncTest.cpp b/source/libs/sync/test/syncTest.cpp index 3a9a29f023cd..6c8449fbf6cb 100644 --- a/source/libs/sync/test/syncTest.cpp +++ b/source/libs/sync/test/syncTest.cpp @@ -48,7 +48,7 @@ void test4() { } int main(int argc, char** argv) { - taosInitLog("/tmp/syncTest.log", 100, false); + taosInitLog("/tmp/syncTest.log", 100, LOG_MODE_TAOSD); tsAsyncLog = 0; sDebugFlag = DEBUG_SCREEN + DEBUG_FILE + DEBUG_TRACE + DEBUG_INFO + DEBUG_ERROR; diff --git a/source/libs/sync/test/syncTestTool.cpp b/source/libs/sync/test/syncTestTool.cpp index 6671cebcafaf..63016467ad04 100644 --- a/source/libs/sync/test/syncTestTool.cpp +++ b/source/libs/sync/test/syncTestTool.cpp @@ -351,7 +351,7 @@ int main(int argc, char** argv) { char logFile[256]; snprintf(logFile, sizeof(logFile), "/tmp/%s-replicaNum%d-myIndex%d.log", gDir, replicaNum, myIndex); - taosInitLog(logFile, 100, false); + taosInitLog(logFile, 100, LOG_MODE_TAOSD); sTrace("logFile : %s", logFile); gSnapshotLastApplyIndex = lastApplyIndex; diff --git a/source/libs/transport/CMakeLists.txt b/source/libs/transport/CMakeLists.txt index 6ad130017a0b..b9d6aa280bb6 100644 --- a/source/libs/transport/CMakeLists.txt +++ b/source/libs/transport/CMakeLists.txt @@ -25,6 +25,10 @@ if(${BUILD_WITH_UV_TRANS}) endif(${BUILD_WITH_UV}) endif(${BUILD_WITH_UV_TRANS}) +if(${TD_ACORE}) + add_definitions(-DTD_ACORE) +endif(${TD_ACORE}) + if(${BUILD_TEST}) add_subdirectory(test) endif(${BUILD_TEST}) diff --git a/source/libs/transport/inc/transComm.h b/source/libs/transport/inc/transComm.h index 6ec5f74f6272..38b701132814 100644 --- a/source/libs/transport/inc/transComm.h +++ b/source/libs/transport/inc/transComm.h @@ -18,6 +18,14 @@ extern "C" { #endif +#ifdef TD_ACORE +#include "tmsg.h" +#include "transLog.h" +#include "transportInt.h" +#include "trpc.h" +#include "ttrace.h" + +#else #include #include "theap.h" #include "tmsg.h" @@ -25,6 +33,7 @@ extern "C" { #include "transportInt.h" #include "trpc.h" #include "ttrace.h" +#endif typedef bool (*FilteFunc)(void* arg); @@ -110,6 +119,11 @@ typedef SRpcCtxVal STransCtxVal; typedef SRpcInfo STrans; typedef SRpcConnInfo STransHandleInfo; +typedef struct { + tsem_t* sem; + STransMsg pMsg; +} STransReqWithSem; + // ref mgt handle typedef struct SExHandle { void* handle; @@ -231,7 +245,17 @@ typedef enum { ConnNormal, ConnAcquire, ConnRelease, ConnBroken, ConnInPool } Co #define transIsReq(type) (type & 1U) #define transLabel(trans) ((STrans*)trans)->label +#ifdef TD_ACORE +typedef struct SConnBuffer { + char* buf; + int len; + int cap; + int left; + int total; + int invalid; +} SConnBuffer; +#else typedef struct SConnBuffer { char* buf; int len; @@ -297,44 +321,81 @@ bool transAsyncPoolIsEmpty(SAsyncPool* pool); int32_t transInitBuffer(SConnBuffer* buf); int32_t transClearBuffer(SConnBuffer* buf); void transDestroyBuffer(SConnBuffer* buf); -int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); -bool transReadComplete(SConnBuffer* connBuf); int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf); +int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf); +bool transReadComplete(SConnBuffer* connBuf); int32_t transSetConnOption(uv_tcp_t* stream, int keepalive); +#endif -void transRefSrvHandle(void* handle); -void transUnrefSrvHandle(void* handle); +int32_t transInitBuffer(SConnBuffer* buf); +int32_t transClearBuffer(SConnBuffer* buf); +void transDestroyBuffer(SConnBuffer* buf); +int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf); -void transRefCliHandle(void* handle); -int32_t transUnrefCliHandle(void* handle); + +#ifdef TD_ACORE +int32_t transSendRequest2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); +int32_t transSendRecv2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp); +int32_t transSendRecvWithTimeout2(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs); +int32_t transSendRequestWithId2(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId); +int32_t transFreeConnById2(void* pInit, int64_t transpointId); +int32_t transSendResponse2(STransMsg* msg); +int32_t transRegisterMsg2(const STransMsg* msg); +int32_t transSetDefaultAddr2(void* pInit, const char* ip, const char* fqdn); +int32_t transSetIpWhiteList2(void* pInit, void* arg, FilteFunc* func); +void transRefSrvHandle2(void* handle); +void transUnrefSrvHandle2(void* handle); + +void transRefCliHandle2(void* handle); +void transUnrefCliHandle2(void* handle); int32_t transGetRefCount(void* handle); -int32_t transReleaseCliHandle(void* handle); -int32_t transReleaseSrvHandle(void* handle); +int32_t transReleaseCliHandle2(void* handle, int32_t status); +int32_t transReleaseSrvHandle2(void* handle, int32_t status); +#else int32_t transSendRequest(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* pCtx); int32_t transSendRecv(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp); int32_t transSendRecvWithTimeout(void* pInit, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, int32_t timeoutMs); + int32_t transSendRequestWithId(void* pInit, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId); int32_t transFreeConnById(void* pInit, int64_t transpointId); -int32_t transSendResponse(const STransMsg* msg); +int32_t transSendResponse(STransMsg* msg); int32_t transRegisterMsg(const STransMsg* msg); int32_t transSetDefaultAddr(void* pInit, const char* ip, const char* fqdn); int32_t transSetIpWhiteList(void* pInit, void* arg, FilteFunc* func); +void transRefSrvHandle(void* handle); +void transUnrefSrvHandle(void* handle); -void transSockInfo2Str(struct sockaddr* sockname, char* dst); +void transRefCliHandle(void* handle); +void transUnrefCliHandle(void* handle); +int32_t transGetRefCount(void* handle); -int32_t transAllocHandle(int64_t* refId); +int32_t transReleaseCliHandle(void* handle); +int32_t transReleaseSrvHandle(void* handle); + +#endif + +#ifndef TD_ACORE +void transSockInfo2Str(struct sockaddr* sockname, char* dst); +int32_t transAllocHandle(int64_t* refId); void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); - void transCloseClient(void* arg); void transCloseServer(void* arg); +#else +int32_t transAllocHandle2(int64_t* refId); +void* transInitClient2(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); +void* transInitServer2(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); +void transCloseClient2(void* arg); +void transCloseServer2(void* arg); +#endif void transCtxInit(STransCtx* ctx); void transCtxCleanup(STransCtx* ctx); @@ -415,11 +476,13 @@ void transQueueDestroy(STransQueue* queue); /* * delay queue based on uv loop and uv timer, and only used in retry */ + +#ifndef TD_ACORE + typedef struct STaskArg { void* param1; void* param2; } STaskArg; - typedef struct SDelayTask { void (*func)(void* arg); void* arg; @@ -438,6 +501,22 @@ void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)); SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs); void transDQCancel(SDelayQueue* queue, SDelayTask* task); +typedef struct { + queue node; // queue for write + queue q; // queue for reqs + uv_write_t wreq; + void* arg; +} SWReqsWrapper; + +int32_t initWQ(queue* wq); +void destroyWQ(queue* wq); +uv_write_t* allocWReqFromWQ(queue* wq, void* arg); + +void freeWReqToWQ(queue* wq, SWReqsWrapper* w); + +int32_t transSetReadOption(uv_handle_t* handle); + +#endif bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b); bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b); @@ -486,6 +565,31 @@ int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf); int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf); int32_t transUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf); +#ifdef TD_ACORE + +#define ASYNC_CHECK_HANDLE(idMgt, id, exh1) \ + do { \ + if (id > 0) { \ + SExHandle* exh2 = transAcquireExHandle(idMgt, id); \ + if (exh2 == NULL || exh1 != exh2 || (exh2 != NULL && exh2->refId != id)) { \ + tDebug("handle not match, exh1:%p, exh2:%p, refId:%" PRId64 "", exh1, exh2, id); \ + code = TSDB_CODE_INVALID_MSG; \ + goto _return1; \ + } \ + } else { \ + tDebug("invalid handle to release"); \ + goto _return2; \ + } \ + } while (0) +int32_t transUpdateCb(RPC_TYPE type, STrans* pTransport); + +int32_t transGetCb(RPC_TYPE type, STrans** ppTransport); + +int32_t transSendReq(STrans* pTransport, SRpcMsg* pMsg, void* pEpSet); + +int32_t transSendResp(const SRpcMsg* pMsg); +#endif + enum { REQ_STATUS_INIT = 0, REQ_STATUS_PROCESSING }; #if defined(WINDOWS) || defined(DARWIN) @@ -499,20 +603,6 @@ enum { REQ_STATUS_INIT = 0, REQ_STATUS_PROCESSING }; #define HEAP_MISS_HIT_LIMIT 100000 #define READ_TIMEOUT 100000 -typedef struct { - queue node; // queue for write - queue q; // queue for reqs - uv_write_t wreq; - void* arg; -} SWReqsWrapper; - -int32_t initWQ(queue* wq); -void destroyWQ(queue* wq); -uv_write_t* allocWReqFromWQ(queue* wq, void* arg); - -void freeWReqToWQ(queue* wq, SWReqsWrapper* w); - -int32_t transSetReadOption(uv_handle_t* handle); #ifdef __cplusplus } #endif diff --git a/source/libs/transport/inc/transportInt.h b/source/libs/transport/inc/transportInt.h index 39afb2934251..c4c7fe4ee4cf 100644 --- a/source/libs/transport/inc/transportInt.h +++ b/source/libs/transport/inc/transportInt.h @@ -16,6 +16,7 @@ #ifndef _TD_TRANSPORT_INT_H_ #define _TD_TRANSPORT_INT_H_ +#ifndef TD_ACORE #include #include "lz4.h" #include "os.h" @@ -27,11 +28,25 @@ #include "tref.h" #include "trpc.h" #include "tutil.h" +#else +#include "lz4.h" +#include "os.h" +#include "taoserror.h" +#include "tglobal.h" +#include "thash.h" +#include "tmsg.h" +#include "transLog.h" +#include "tref.h" +#include "trpc.h" +#include "tutil.h" +#endif #ifdef __cplusplus extern "C" { #endif + +#ifndef TD_ACORE void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit); @@ -81,7 +96,72 @@ typedef struct { int8_t startReadTimer; int64_t readTimeout; TdThreadMutex mutex; + int16_t type; + + TdThreadMutex sidMutx; + SHashObj* sidTable; + + TdThreadMutex seqMutex; + int64_t seq; + SHashObj* seqTable; +} SRpcInfo; +#else +void* taosInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); +void* taosInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle); + +void taosCloseServer(void* arg); +void taosCloseClient(void* arg); + +typedef struct { + int sessions; // number of sessions allowed + int numOfThreads; // number of threads to process incoming messages + int idleTime; // milliseconds; + uint16_t localPort; + int8_t connType; + char label[TSDB_LABEL_LEN]; + char user[TSDB_UNI_LEN]; // meter ID + int32_t compatibilityVer; + int32_t compressSize; // -1: no compress, 0 : all data compressed, size: compress data if larger than size + int8_t encryption; // encrypt or not + + int32_t retryMinInterval; // retry init interval + int32_t retryStepFactor; // retry interval factor + int32_t retryMaxInterval; // retry max interval + int32_t retryMaxTimeout; + + int32_t failFastThreshold; + int32_t failFastInterval; + + int8_t notWaitAvaliableConn; // 1: no delay, 0: delay + + void (*cfp)(void* parent, SRpcMsg*, SEpSet*); + bool (*retry)(int32_t code, tmsg_t msgType); + bool (*startTimer)(int32_t code, tmsg_t msgType); + void (*destroyFp)(void* ahandle); + bool (*failFastFp)(tmsg_t msgType); + bool (*noDelayFp)(tmsg_t msgType); + + int32_t connLimitNum; + int8_t connLimitLock; // 0: no lock. 1. lock + int8_t supportBatch; // 0: no batch, 1: support batch + int32_t batchSize; + int32_t timeToGetConn; + int index; + void* parent; + void* tcphandle; // returned handle from TCP initialization + int64_t refId; + TdThreadMutex mutex; + int16_t type; + + TdThreadMutex sidMutx; + SHashObj* sidTable; + + TdThreadMutex seqMutex; + int64_t seq; + SHashObj* seqTable; + int8_t shareConnLimit; } SRpcInfo; +#endif #ifdef __cplusplus } diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index bdcbfeb1cd50..b6a1ac402be0 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE // clang-format off +#ifndef TD_ACORE #include #include "zlib.h" #include "thttp.h" @@ -868,3 +869,19 @@ void taosDestroyHttpChan(int64_t chanId) { TAOS_UNUSED(taosReleaseRef(httpRefMgt, chanId)); TAOS_UNUSED(taosRemoveRef(httpRefMgt, chanId)); } +#else +#include "thttp.h" +#include "taoserror.h" +#include "transComm.h" +#include "zlib.h" +void transHttpEnvDestroy() { return; } +int32_t taosSendHttpReport(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag) { + return 0; +} +int32_t taosSendHttpReportWithQID(const char* server, const char* uri, uint16_t port, char* pCont, int32_t contLen, + EHttpCompFlag flag, const char* qid) { + return 0; +} + +#endif \ No newline at end of file diff --git a/source/libs/transport/src/trans.c b/source/libs/transport/src/trans.c index de129773a02b..b78909ab1460 100644 --- a/source/libs/transport/src/trans.c +++ b/source/libs/transport/src/trans.c @@ -15,9 +15,9 @@ #include "transComm.h" +#ifndef TD_ACORE void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* pInit) = { transInitServer, transInitClient}; - void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; void (*taosRefHandle[])(void* handle) = {transRefSrvHandle, transRefCliHandle}; @@ -249,3 +249,300 @@ void rpcCleanup(void) { return; } + +#else + +#ifdef TD_ACORE +void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { + transInitServer2, transInitClient2}; + +void (*taosCloseHandle[])(void* arg) = {transCloseServer2, transCloseClient2}; +int (*transReleaseHandle[])(void* handle, int32_t status) = {transReleaseSrvHandle2, transReleaseCliHandle2}; +#else + +void* (*taosInitHandle[])(uint32_t ip, uint32_t port, char* label, int32_t numOfThreads, void* fp, void* shandle) = { + transInitServer, transInitClient}; +void (*taosCloseHandle[])(void* arg) = {transCloseServer, transCloseClient}; + +int (*transReleaseHandle[])(void* handle) = {transReleaseSrvHandle, transReleaseCliHandle}; + +#endif + +static int32_t transValidLocalFqdn(const char* localFqdn, uint32_t* ip) { return 0; } +typedef struct { + char* lablset; + RPC_TYPE type; + +} SLableSet; +static SLableSet labelSet[] = { + {"TSC", TD_ACORE_CLIENT | TD_ACORE_DSVR}, + {"DNODE-CLI", TD_ACORE_DSVR_CLIENT | TD_ACORE_DSVR}, + {"DNODE-STA-CLI", TD_ACORE_DSVR_STA_CLIENT | TD_ACORE_DSVR}, + {"DNODE-SYNC-CLI", TD_ACORE_DSVR_SYNC_CLIENT | TD_ACORE_DSVR}, + {"DND-S", TD_ACORE_DSVR}, +}; + +void* rpcOpen(const SRpcInit* pInit) { + int32_t code = rpcInit(); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _end); + } + + SRpcInfo* pRpc = taosMemoryCalloc(1, sizeof(SRpcInfo)); + if (pRpc == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + if (pInit->label) { + int len = strlen(pInit->label) > sizeof(pRpc->label) ? sizeof(pRpc->label) : strlen(pInit->label); + memcpy(pRpc->label, pInit->label, len); + } + + pRpc->compressSize = pInit->compressSize; + if (pRpc->compressSize < 0) { + pRpc->compressSize = -1; + } + + pRpc->encryption = pInit->encryption; + pRpc->compatibilityVer = pInit->compatibilityVer; + + pRpc->retryMinInterval = pInit->retryMinInterval; // retry init interval + pRpc->retryStepFactor = pInit->retryStepFactor; + pRpc->retryMaxInterval = pInit->retryMaxInterval; + pRpc->retryMaxTimeout = pInit->retryMaxTimeout; + + pRpc->failFastThreshold = pInit->failFastThreshold; + pRpc->failFastInterval = pInit->failFastInterval; + + // register callback handle + pRpc->cfp = pInit->cfp; + pRpc->retry = pInit->rfp; + pRpc->startTimer = pInit->tfp; + pRpc->destroyFp = pInit->dfp; + pRpc->failFastFp = pInit->ffp; + pRpc->noDelayFp = pInit->noDelayFp; + pRpc->connLimitNum = pInit->connLimitNum; + if (pRpc->connLimitNum == 0) { + pRpc->connLimitNum = 20; + } + + pRpc->connLimitLock = pInit->connLimitLock; + pRpc->supportBatch = pInit->supportBatch; + pRpc->batchSize = pInit->batchSize; + + pRpc->numOfThreads = pInit->numOfThreads > TSDB_MAX_RPC_THREADS ? TSDB_MAX_RPC_THREADS : pInit->numOfThreads; + if (pRpc->numOfThreads <= 0) { + pRpc->numOfThreads = 1; + } + + uint32_t ip = 0; + if (pInit->connType == TAOS_CONN_SERVER) { + if ((code = transValidLocalFqdn(pInit->localFqdn, &ip)) != 0) { + tError("invalid fqdn:%s, errmsg:%s", pInit->localFqdn, tstrerror(code)); + TAOS_CHECK_GOTO(code, NULL, _end); + } + } + + pRpc->connType = pInit->connType; + pRpc->idleTime = pInit->idleTime; + pRpc->parent = pInit->parent; + if (pInit->user) { + tstrncpy(pRpc->user, pInit->user, sizeof(pRpc->user)); + } + pRpc->timeToGetConn = pInit->timeToGetConn; + if (pRpc->timeToGetConn == 0) { + pRpc->timeToGetConn = 10 * 1000; + } + pRpc->notWaitAvaliableConn = pInit->notWaitAvaliableConn; + + pRpc->tcphandle = + (*taosInitHandle[pRpc->connType])(ip, pInit->localPort, pRpc->label, pRpc->numOfThreads, NULL, pRpc); + + if (pRpc->tcphandle == NULL) { + // tError("failed to init rpc handle"); + // TAOS_CHECK_GOTO(terrno, NULL, _end); + } + for (int8_t i = 0; i < sizeof(labelSet) / sizeof(labelSet[0]); i++) { + if (strcmp(labelSet[i].lablset, pRpc->label) == 0) { + pRpc->type = labelSet[i].type; + break; + } + } + + taosThreadMutexInit(&pRpc->sidMutx, NULL); + pRpc->sidTable = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + + taosThreadMutexInit(&pRpc->seqMutex, NULL); + pRpc->seqTable = taosHashInit(4096, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_ENTRY_LOCK); + pRpc->seq = 1; + transUpdateCb(pRpc->type, pRpc); + + int64_t refId = transAddExHandle(transGetInstMgt(), pRpc); + void* tmp = transAcquireExHandle(transGetInstMgt(), refId); + pRpc->refId = refId; + return (void*)refId; +_end: + taosMemoryFree(pRpc); + terrno = code; + + return NULL; +} +void rpcClose(void* arg) { + tInfo("start to close rpc"); + return; +} +void rpcCloseImpl(void* arg) { + if (arg == NULL) return; + return; + // SRpcInfo* pRpc = (SRpcInfo*)arg; + // if (pRpc->tcphandle != NULL) { + // (*taosCloseHandle[pRpc->connType])(pRpc->tcphandle); + // } + // taosMemoryFree(pRpc); +} + +void* rpcMallocCont(int64_t contLen) { + int64_t size = contLen + TRANS_MSG_OVERHEAD; + char* start = taosMemoryCalloc(1, size); + if (start == NULL) { + tError("failed to malloc msg, size:%" PRId64, size); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } else { + tTrace("malloc mem:%p size:%" PRId64, start, size); + } + + return start + sizeof(STransMsgHead); +} + +void rpcFreeCont(void* cont) { transFreeMsg(cont); } + +void* rpcReallocCont(void* ptr, int64_t contLen) { + if (ptr == NULL) return rpcMallocCont(contLen); + + char* st = (char*)ptr - TRANS_MSG_OVERHEAD; + int64_t sz = contLen + TRANS_MSG_OVERHEAD; + char* nst = taosMemoryRealloc(st, sz); + if (nst == NULL) { + taosMemoryFree(st); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } else { + st = nst; + } + + return st + TRANS_MSG_OVERHEAD; +} + +int32_t rpcSendRequest(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid) { +#ifdef TD_ACORE + return transSendRequest2(shandle, pEpSet, pMsg, NULL); +#else + return transSendRequest(shandle, pEpSet, pMsg, NULL); +#endif +} +int32_t rpcSendRequestWithCtx(void* shandle, const SEpSet* pEpSet, SRpcMsg* pMsg, int64_t* pRid, SRpcCtx* pCtx) { +#ifdef TD_ACORE + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { + return transSendRequest2(shandle, pEpSet, pMsg, pCtx); + } else { + return transSendRequestWithId2(shandle, pEpSet, pMsg, pRid); + } +#else + if (pCtx != NULL || pMsg->info.handle != 0 || pMsg->info.noResp != 0 || pRid == NULL) { + return transSendRequest(shandle, pEpSet, pMsg, pCtx); + } else { + return transSendRequestWithId(shandle, pEpSet, pMsg, pRid); + } +#endif +} + +int32_t rpcSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { +#ifdef TD_ACORE + return transSendRequestWithId2(shandle, pEpSet, pReq, transpointId); +#else + return transSendRequestWithId(shandle, pEpSet, pReq, transpointId); +#endif +} + +int32_t rpcSendRecv(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp) { +#ifdef TD_ACORE + return transSendRecv2(shandle, pEpSet, pMsg, pRsp); +#else + return transSendRecv(shandle, pEpSet, pMsg, pRsp); +#endif +} +int32_t rpcSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, SRpcMsg* pMsg, SRpcMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { +#ifdef TD_ACORE + return transSendRecvWithTimeout2(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +#else + return transSendRecvWithTimeout(shandle, pEpSet, pMsg, pRsp, epUpdated, timeoutMs); +#endif +} +int32_t rpcFreeConnById(void* shandle, int64_t connId) { +#ifdef TD_ACORE + return transFreeConnById2(shandle, connId); +#else + return transFreeConnById(shandle, connId); +#endif +} + +int32_t rpcSendResponse(SRpcMsg* pMsg) { +#ifdef TD_ACORE + return transSendResponse2(pMsg); +#else + return transSendResponse(pMsg); +#endif +} + +int32_t rpcRegisterBrokenLinkArg(SRpcMsg* msg) { +#ifdef TD_ACORE + return transRegisterMsg2(msg); +#else + return transRegisterMsg(msg); +#endif +} +int32_t rpcReleaseHandle(void* handle, int8_t type) { return (*transReleaseHandle[type])(handle, 0); } + +// client only +int32_t rpcSetDefaultAddr(void* thandle, const char* ip, const char* fqdn) { + // later +#ifdef TD_ACORE + return transSetDefaultAddr2(thandle, ip, fqdn); +#else + return transSetDefaultAddr(thandle, ip, fqdn); +#endif +} +// server only +int32_t rpcSetIpWhite(void* thandle, void* arg) { +#ifdef TD_ACORE + return transSetIpWhiteList2(thandle, arg, NULL); +#else + return transSetIpWhiteList(thandle, arg, NULL); +#endif +} + +int32_t rpcAllocHandle(int64_t* refId) { return transAllocHandle2(refId); } + +int32_t rpcUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return transUtilSIpRangeToStr(pRange, buf); } +int32_t rpcUtilSWhiteListToStr(SIpWhiteList* pWhiteList, char** ppBuf) { + return transUtilSWhiteListToStr(pWhiteList, ppBuf); +} + +int32_t rpcCvtErrCode(int32_t code) { + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + return TSDB_CODE_RPC_NETWORK_ERROR; + } + return code; +} + +int32_t rpcInit() { return transInit(); } + +void rpcCleanup(void) { + transCleanup(); + transHttpEnvDestroy(); + + return; +} + +#endif \ No newline at end of file diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index a90c5467965d..b7a7ec91d50e 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -16,9 +16,11 @@ #include "taoserror.h" #include "transComm.h" #include "tmisce.h" +#include "tversion.h" #include "transLog.h" // clang-format on +#ifndef TD_ACORE typedef struct { int32_t numOfConn; queue msgQ; @@ -3068,6 +3070,7 @@ void transCloseClient(void* arg) { taosMemoryFree(cli->pThreadObj); taosMemoryFree(cli); } + void transRefCliHandle(void* handle) { int32_t ref = 0; if (handle == NULL) { @@ -3844,235 +3847,338 @@ static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn* pConn, char* key) { return 1; } - return 0; -} + static FORCE_INLINE bool filterToDebug(void* e, void* arg) { + SCliReq* pReq = QUEUE_DATA(e, SCliReq, q); + STraceId* trace = &pReq->msg.info.traceId; + tGWarn("%s is sent to, and no resp from server", TMSG_INFO(pReq->msg.msgType)); + return false; + } + static FORCE_INLINE void logConnMissHit(SCliConn * pConn) { + // queue set; + // QUEUE_INIT(&set); + SCliThrd* pThrd = pConn->hostThrd; + STrans* pInst = pThrd->pInst; + pConn->heapMissHit++; + tDebug("conn %p has %d reqs, %d sentout and %d status in process, total limit:%d, switch to other conn", pConn, + transQueueSize(&pConn->reqsToSend), transQueueSize(&pConn->reqsSentOut), taosHashGetSize(pConn->pQTable), + pInst->shareConnLimit); + // if (transQueueSize(&pConn->reqsSentOut) >= pInst->shareConnLimit) { + // transQueueRemoveByFilter(&pConn->reqsSentOut, filterToDebug, NULL, &set, 1); + // } + } + static SCliConn* getConnFromHeapCache(SHashObj * pConnHeapCache, char* key) { + int code = 0; + SHeap* pHeap = NULL; + SCliConn* pConn = NULL; + code = getOrCreateHeap(pConnHeapCache, key, &pHeap); + if (code != 0) { + tTrace("failed to get conn heap from cache for key:%s", key); + return NULL; + } + code = transHeapGet(pHeap, &pConn); + if (code != 0) { + tTrace("failed to get conn from heap cache for key:%s", key); + return NULL; + } else { + tTrace("conn %p get conn from heap cache for key:%s", pConn, key); + if (shouldSWitchToOtherConn(pConn, key)) { + SCliConn* pNewConn = NULL; + code = balanceConnHeapCache(pConnHeapCache, pConn, &pNewConn); + if (code == 1) { + tTrace("conn %p start to handle reqs", pNewConn); + return pNewConn; + } + return NULL; + } + } -static FORCE_INLINE bool filterToDebug(void* e, void* arg) { - SCliReq* pReq = QUEUE_DATA(e, SCliReq, q); - STraceId* trace = &pReq->msg.info.traceId; - tGWarn("%s is sent to, and no resp from server", TMSG_INFO(pReq->msg.msgType)); - return false; -} -static FORCE_INLINE void logConnMissHit(SCliConn* pConn) { - // queue set; - // QUEUE_INIT(&set); - SCliThrd* pThrd = pConn->hostThrd; - STrans* pInst = pThrd->pInst; - pConn->heapMissHit++; - tDebug("conn %p has %d reqs, %d sentout and %d status in process, total limit:%d, switch to other conn", pConn, - transQueueSize(&pConn->reqsToSend), transQueueSize(&pConn->reqsSentOut), taosHashGetSize(pConn->pQTable), - pInst->shareConnLimit); - // if (transQueueSize(&pConn->reqsSentOut) >= pInst->shareConnLimit) { - // transQueueRemoveByFilter(&pConn->reqsSentOut, filterToDebug, NULL, &set, 1); - // } -} -static SCliConn* getConnFromHeapCache(SHashObj* pConnHeapCache, char* key) { - int code = 0; - SHeap* pHeap = NULL; - SCliConn* pConn = NULL; - code = getOrCreateHeap(pConnHeapCache, key, &pHeap); - if (code != 0) { - tTrace("failed to get conn heap from cache for key:%s", key); - return NULL; + return pConn; } - code = transHeapGet(pHeap, &pConn); - if (code != 0) { - tTrace("failed to get conn from heap cache for key:%s", key); - return NULL; - } else { - tTrace("conn %p get conn from heap cache for key:%s", pConn, key); - if (shouldSWitchToOtherConn(pConn, key)) { - SCliConn* pNewConn = NULL; - code = balanceConnHeapCache(pConnHeapCache, pConn, &pNewConn); - if (code == 1) { - tTrace("conn %p start to handle reqs", pNewConn); - return pNewConn; + static int32_t addConnToHeapCache(SHashObj * pConnHeapCacahe, SCliConn * pConn) { + SHeap* p = NULL; + int32_t code = 0; + + if (pConn->heap != NULL) { + p = pConn->heap; + tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d, add direct", pConn, pConn->dstAddr, + pConn->inHeap, pConn->reqRefCnt); + } else { + code = getOrCreateHeap(pConnHeapCacahe, pConn->dstAddr, &p); + if (code != 0) { + return code; + } + if (pConn->connnected == 0) { + int64_t now = taosGetTimestampMs(); + if (now - p->lastConnFailTs < 3000) { + return TSDB_CODE_RPC_NETWORK_UNAVAIL; + } } - return NULL; } + + code = transHeapInsert(p, pConn); + tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d", pConn, pConn->dstAddr, pConn->inHeap, + pConn->reqRefCnt); + return code; } - return pConn; -} -static int32_t addConnToHeapCache(SHashObj* pConnHeapCacahe, SCliConn* pConn) { - SHeap* p = NULL; - int32_t code = 0; + static int32_t delConnFromHeapCache(SHashObj * pConnHeapCache, SCliConn * pConn) { + if (pConn->heap != NULL) { + tTrace("conn %p try to delete from heap cache direct", pConn); + return transHeapDelete(pConn->heap, pConn); + } - if (pConn->heap != NULL) { - p = pConn->heap; - tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d, add direct", pConn, pConn->dstAddr, - pConn->inHeap, pConn->reqRefCnt); - } else { - code = getOrCreateHeap(pConnHeapCacahe, pConn->dstAddr, &p); + SHeap* p = taosHashGet(pConnHeapCache, pConn->dstAddr, strlen(pConn->dstAddr)); + if (p == NULL) { + tTrace("failed to get heap cache for key:%s, no need to del", pConn->dstAddr); + return 0; + } + int32_t code = transHeapDelete(p, pConn); if (code != 0) { - return code; + tTrace("conn %p failed delete from heap cache since %s", pConn, tstrerror(code)); } - if (pConn->connnected == 0) { - int64_t now = taosGetTimestampMs(); - if (now - p->lastConnFailTs < 3000) { - return TSDB_CODE_RPC_NETWORK_UNAVAIL; + return code; + } + + static int8_t balanceConnHeapCache(SHashObj * pConnHeapCache, SCliConn * pConn, SCliConn * *pNewConn) { + SCliThrd* pThrd = pConn->hostThrd; + STrans* pInst = pThrd->pInst; + SCliConn* pTopConn = NULL; + if (pConn->heap != NULL && pConn->inHeap != 0) { + TAOS_UNUSED(transHeapBalance(pConn->heap, pConn)); + if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) { + int32_t curReqs = REQS_ON_CONN(pConn); + int32_t topReqs = REQS_ON_CONN(pTopConn); + if (curReqs > topReqs && topReqs < pInst->shareConnLimit) { + *pNewConn = pTopConn; + return 1; + } } } + return 0; } + // conn heap + int32_t compareHeapNode(const HeapNode* a, const HeapNode* b) { + SCliConn* args1 = container_of(a, SCliConn, node); + SCliConn* args2 = container_of(b, SCliConn, node); - code = transHeapInsert(p, pConn); - tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d", pConn, pConn->dstAddr, pConn->inHeap, - pConn->reqRefCnt); - return code; -} - -static int32_t delConnFromHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn) { - if (pConn->heap != NULL) { - tTrace("conn %p try to delete from heap cache direct", pConn); - return transHeapDelete(pConn->heap, pConn); + int32_t totalReq1 = REQS_ON_CONN(args1); + int32_t totalReq2 = REQS_ON_CONN(args2); + if (totalReq1 > totalReq2) { + return 0; + } + return 1; } + int32_t transHeapInit(SHeap * heap, int32_t(*cmpFunc)(const HeapNode* a, const HeapNode* b)) { + heap->heap = heapCreate(cmpFunc); + if (heap->heap == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } - SHeap* p = taosHashGet(pConnHeapCache, pConn->dstAddr, strlen(pConn->dstAddr)); - if (p == NULL) { - tTrace("failed to get heap cache for key:%s, no need to del", pConn->dstAddr); + heap->cmpFunc = cmpFunc; return 0; } - int32_t code = transHeapDelete(p, pConn); - if (code != 0) { - tTrace("conn %p failed delete from heap cache since %s", pConn, tstrerror(code)); + void transHeapDestroy(SHeap * heap) { + if (heap != NULL) { + heapDestroy(heap->heap); + } } - return code; -} - -static int8_t balanceConnHeapCache(SHashObj* pConnHeapCache, SCliConn* pConn, SCliConn** pNewConn) { - SCliThrd* pThrd = pConn->hostThrd; - STrans* pInst = pThrd->pInst; - SCliConn* pTopConn = NULL; - if (pConn->heap != NULL && pConn->inHeap != 0) { - TAOS_UNUSED(transHeapBalance(pConn->heap, pConn)); - if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) { - int32_t curReqs = REQS_ON_CONN(pConn); - int32_t topReqs = REQS_ON_CONN(pTopConn); - if (curReqs > topReqs && topReqs < pInst->shareConnLimit) { - *pNewConn = pTopConn; - return 1; - } + int32_t transHeapGet(SHeap * heap, SCliConn * *p) { + if (heapSize(heap->heap) == 0) { + *p = NULL; + return -1; } + HeapNode* minNode = heapMin(heap->heap); + if (minNode == NULL) { + *p = NULL; + return -1; + } + *p = container_of(minNode, SCliConn, node); + return 0; } - return 0; -} -// conn heap -int32_t compareHeapNode(const HeapNode* a, const HeapNode* b) { - SCliConn* args1 = container_of(a, SCliConn, node); - SCliConn* args2 = container_of(b, SCliConn, node); + int32_t transHeapInsert(SHeap * heap, SCliConn * p) { + // impl later + p->reqRefCnt++; + if (p->inHeap == 1) { + tTrace("failed to insert conn %p since already in heap", p); + return TSDB_CODE_DUP_KEY; + } - int32_t totalReq1 = REQS_ON_CONN(args1); - int32_t totalReq2 = REQS_ON_CONN(args2); - if (totalReq1 > totalReq2) { + heapInsert(heap->heap, &p->node); + p->inHeap = 1; + p->lastAddHeapTime = taosGetTimestampMs(); + p->heap = heap; return 0; } - return 1; -} -int32_t transHeapInit(SHeap* heap, int32_t (*cmpFunc)(const HeapNode* a, const HeapNode* b)) { - heap->heap = heapCreate(cmpFunc); - if (heap->heap == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + int32_t transHeapDelete(SHeap * heap, SCliConn * p) { + // impl later + if (p->connnected == 0) { + TAOS_UNUSED(transHeapUpdateFailTs(heap, p)); + } - heap->cmpFunc = cmpFunc; - return 0; -} -void transHeapDestroy(SHeap* heap) { - if (heap != NULL) { - heapDestroy(heap->heap); + if (p->inHeap == 0) { + tTrace("failed to del conn %p since not in heap", p); + return 0; + } else { + int64_t now = taosGetTimestampMs(); + if (p->forceDelFromHeap == 0 && now - p->lastAddHeapTime < 10000) { + tTrace("conn %p not added/delete to heap frequently", p); + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } + } + + p->inHeap = 0; + p->reqRefCnt--; + if (p->reqRefCnt == 0) { + heapRemove(heap->heap, &p->node); + tTrace("conn %p delete from heap", p); + } else if (p->reqRefCnt < 0) { + tTrace("conn %p has %d reqs, not delete from heap,assert", p, p->reqRefCnt); + } else { + tTrace("conn %p has %d reqs, not delete from heap", p, p->reqRefCnt); + } + return 0; } -} -int32_t transHeapGet(SHeap* heap, SCliConn** p) { - if (heapSize(heap->heap) == 0) { - *p = NULL; - return -1; + + int32_t transHeapUpdateFailTs(SHeap * heap, SCliConn * p) { + heap->lastConnFailTs = taosGetTimestampMs(); + return 0; } - HeapNode* minNode = heapMin(heap->heap); - if (minNode == NULL) { - *p = NULL; - return -1; + int32_t transHeapMayBalance(SHeap * heap, SCliConn * p) { + if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + return 0; + } + SCliThrd* pThrd = p->hostThrd; + STrans* pInst = pThrd->pInst; + int32_t balanceLimit = pInst->shareConnLimit >= 4 ? pInst->shareConnLimit / 2 : 2; + + SCliConn* topConn = NULL; + int32_t code = transHeapGet(heap, &topConn); + if (code != 0) { + return code; + } + + if (topConn == p) return code; + + int32_t reqsOnTop = REQS_ON_CONN(topConn); + int32_t reqsOnCur = REQS_ON_CONN(p); + + if (reqsOnTop >= balanceLimit && reqsOnCur < balanceLimit) { + TAOS_UNUSED(transHeapBalance(heap, p)); + } + return code; } - *p = container_of(minNode, SCliConn, node); - return 0; -} -int32_t transHeapInsert(SHeap* heap, SCliConn* p) { - // impl later - p->reqRefCnt++; - if (p->inHeap == 1) { - tTrace("failed to insert conn %p since already in heap", p); - return TSDB_CODE_DUP_KEY; + + int32_t transHeapBalance(SHeap * heap, SCliConn * p) { + if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + return 0; + } + heapRemove(heap->heap, &p->node); + heapInsert(heap->heap, &p->node); + return 0; } +#else - heapInsert(heap->heap, &p->node); - p->inHeap = 1; - p->lastAddHeapTime = taosGetTimestampMs(); - p->heap = heap; - return 0; -} -int32_t transHeapDelete(SHeap* heap, SCliConn* p) { - // impl later - if (p->connnected == 0) { - TAOS_UNUSED(transHeapUpdateFailTs(heap, p)); +void transRefCliHandle(void* handle) { return; } +void transUnrefCliHandle(void* handle) { return; } +int32_t transReleaseCliHandle(void* handle) { return 0; } +int32_t transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransCtx* ctx) { + int32_t code = 0; + int32_t cliVer = 0; + + code = taosVersionStrToInt(td_version, &cliVer); + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + return TSDB_CODE_RPC_MODULE_QUIT; } - if (p->inHeap == 0) { - tTrace("failed to del conn %p since not in heap", p); - return 0; + TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + pReq->type = pTransInst->type; + pReq->info.connType = pReq->type; + pReq->info.cliVer = cliVer; + pReq->info.msgType = pReq->msgType; + // pReq->info.ahandle = pReq->ahandle; + if (pReq->info.handle != NULL) { + pReq->info.qId = (int64_t)pReq->info.handle; } else { - int64_t now = taosGetTimestampMs(); - if (p->forceDelFromHeap == 0 && now - p->lastAddHeapTime < 10000) { - tTrace("conn %p not added/delete to heap frequently", p); - return TSDB_CODE_RPC_ASYNC_IN_PROCESS; - } + pReq->info.handle = (void*)-1; } - p->inHeap = 0; - p->reqRefCnt--; - if (p->reqRefCnt == 0) { - heapRemove(heap->heap, &p->node); - tTrace("conn %p delete from heap", p); - } else if (p->reqRefCnt < 0) { - tTrace("conn %p has %d reqs, not delete from heap,assert", p, p->reqRefCnt); - } else { - tTrace("conn %p has %d reqs, not delete from heap", p, p->reqRefCnt); + if (pReq->info.qId != 0) { + taosThreadMutexLock(&pTransInst->sidMutx); + /******/ + if (ctx != NULL) { + STransCtx* pUserCtx = taosHashGet(pTransInst->sidTable, &pReq->info.qId, sizeof(pReq->info.qId)); + if (pUserCtx != NULL) { + transCtxMerge(pUserCtx, ctx); + } else { + taosHashPut(pTransInst->sidTable, &pReq->info.qId, sizeof(pReq->info.qId), ctx, sizeof(STransCtx)); + } + } + taosThreadMutexUnlock(&pTransInst->sidMutx); } - return 0; + taosThreadMutexLock(&pTransInst->seqMutex); + pReq->info.seq = pTransInst->seq++; + pReq->parent = pTransInst; + taosHashPut(pTransInst->seqTable, &pReq->info.seq, sizeof(pReq->info.seq), &pReq->msgType, sizeof(pReq->msgType)); + taosThreadMutexUnlock(&pTransInst->seqMutex); + + sprintf(pReq->info.conn.user, "root"); + + code = transSendReq(pTransInst, pReq, NULL); + TAOS_UNUSED(transReleaseExHandle(transGetInstMgt(), (int64_t)shandle)); + return code; } -int32_t transHeapUpdateFailTs(SHeap* heap, SCliConn* p) { - heap->lastConnFailTs = taosGetTimestampMs(); +int32_t transSendRequestWithId(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, int64_t* transpointId) { + int32_t code; + // int32_t cliVer = 0; + // code = taosVersionStrToInt(version, &cliVer); + return transSendRequest(shandle, pEpSet, pReq, NULL); + return 0; } -int32_t transHeapMayBalance(SHeap* heap, SCliConn* p) { - if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { - return 0; + +int32_t transCreateSyncMsg(STransMsg* pTransMsg, int64_t* refId) { return 0; } +int32_t transSendRecvWithTimeout(void* shandle, SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp, int8_t* epUpdated, + int32_t timeoutMs) { + int32_t code = 0; + STrans* pTransInst = (STrans*)transAcquireExHandle(transGetInstMgt(), (int64_t)shandle); + if (pTransInst == NULL) { + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + return TSDB_CODE_RPC_MODULE_QUIT; } - SCliThrd* pThrd = p->hostThrd; - STrans* pInst = pThrd->pInst; - int32_t balanceLimit = pInst->shareConnLimit >= 4 ? pInst->shareConnLimit / 2 : 2; - SCliConn* topConn = NULL; - int32_t code = transHeapGet(heap, &topConn); + if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + STransReqWithSem* tmsg = taosMemoryCalloc(1, sizeof(STransReqWithSem)); + tmsg->sem = taosMemoryCalloc(1, sizeof(tsem_t)); + code = tsem_init(tmsg->sem, 0, 0); if (code != 0) { + taosMemoryFree(tmsg->sem); return code; } + pReq->info.reqWithSem = tmsg; + transSendRequest(shandle, NULL, pReq, NULL); + TAOS_UNUSED(tsem_wait(tmsg->sem)); + tsem_destroy(tmsg->sem); - if (topConn == p) return code; - - int32_t reqsOnTop = REQS_ON_CONN(topConn); - int32_t reqsOnCur = REQS_ON_CONN(p); + memcpy(pRsp, &tmsg->pMsg, sizeof(STransMsg)); + taosMemoryFree(tmsg); - if (reqsOnTop >= balanceLimit && reqsOnCur < balanceLimit) { - TAOS_UNUSED(transHeapBalance(heap, p)); - } - return code; + taosReleaseRef(transGetInstMgt(), (int64_t)shandle); + return 0; } +int32_t transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMsg* pRsp) { + return transSendRecvWithTimeout(shandle, (SEpSet*)pEpSet, pReq, pRsp, NULL, 0); +} +int32_t transSetDefaultAddr(void* shandle, const char* ip, const char* fqdn) { return 0; } +int32_t transAllocHandle(int64_t* refId) { return 0; } +int32_t transFreeConnById(void* shandle, int64_t transpointId) { return 0; } -int32_t transHeapBalance(SHeap* heap, SCliConn* p) { - if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { - return 0; - } - heapRemove(heap->heap, &p->node); - heapInsert(heap->heap, &p->node); - return 0; +void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + return NULL; } +void transCloseClient(void* arg) { return; } + +#endif \ No newline at end of file diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index c0edcd54e4b9..b8cfaef4ef62 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -14,7 +14,9 @@ */ #include "transComm.h" +#include "tqueue.h" +#ifndef TD_ACORE #define BUFFER_CAP 8 * 1024 static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; @@ -451,95 +453,6 @@ void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { return ret; } -int32_t transQueueInit(STransQueue* wq, void (*freeFunc)(void* arg)) { - QUEUE_INIT(&wq->node); - wq->freeFunc = (void (*)(void*))freeFunc; - wq->size = 0; - wq->inited = 1; - return 0; -} -void transQueuePush(STransQueue* q, void* arg) { - queue* node = arg; - QUEUE_PUSH(&q->node, node); - q->size++; -} -void* transQueuePop(STransQueue* q) { - if (q->size == 0) return NULL; - - queue* head = QUEUE_HEAD(&q->node); - QUEUE_REMOVE(head); - q->size--; - return head; -} -int32_t transQueueSize(STransQueue* q) { return q->size; } - -void* transQueueGet(STransQueue* q, int idx) { - if (q->size == 0) return NULL; - - while (idx-- > 0) { - queue* node = QUEUE_NEXT(&q->node); - if (node == &q->node) return NULL; - } - return NULL; -} - -void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size) { - queue* d = dst; - queue* node = QUEUE_NEXT(&q->node); - while (node != &q->node) { - queue* next = QUEUE_NEXT(node); - if (filter && filter(node, arg)) { - QUEUE_REMOVE(node); - q->size--; - QUEUE_PUSH(d, node); - if (--size == 0) { - break; - } - } - node = next; - } -} - -void* tranQueueHead(STransQueue* q) { - if (q->size == 0) return NULL; - - queue* head = QUEUE_HEAD(&q->node); - return head; -} - -void* transQueueRm(STransQueue* q, int i) { - // if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { - // return NULL; - // } - // if (i >= taosArrayGetSize(queue->q)) { - // return NULL; - // } - // void* ptr = taosArrayGetP(queue->q, i); - // taosArrayRemove(queue->q, i); - // return ptr; - return NULL; -} - -void transQueueRemove(STransQueue* q, void* e) { - if (q->size == 0) return; - queue* node = e; - QUEUE_REMOVE(node); - q->size--; -} - -bool transQueueEmpty(STransQueue* q) { return q->size == 0 ? true : false; } - -void transQueueClear(STransQueue* q) { - if (q->inited == 0) return; - while (!QUEUE_IS_EMPTY(&q->node)) { - queue* h = QUEUE_HEAD(&q->node); - QUEUE_REMOVE(h); - if (q->freeFunc != NULL) (q->freeFunc)(h); - q->size--; - } -} -void transQueueDestroy(STransQueue* q) { transQueueClear(q); } - static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) { SDelayTask* arg1 = container_of(a, SDelayTask, node); SDelayTask* arg2 = container_of(b, SDelayTask, node); @@ -684,52 +597,6 @@ SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, return task; } -void transPrintEpSet(SEpSet* pEpSet) { - if (pEpSet == NULL) { - tTrace("NULL epset"); - return; - } - char buf[512] = {0}; - int len = tsnprintf(buf, sizeof(buf), "epset:{"); - for (int i = 0; i < pEpSet->numOfEps; i++) { - if (i == pEpSet->numOfEps - 1) { - len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } else { - len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); - } - } - len += tsnprintf(buf + len, sizeof(buf) - len, "}"); - tTrace("%s, inUse:%d", buf, pEpSet->inUse); -} -bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b) { - if (a == NULL && b == NULL) { - return true; - } else if (a == NULL || b == NULL) { - return false; - } - - if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { - return false; - } - for (int i = 0; i < a->numOfEps; i++) { - if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { - return false; - } - } - return true; -} -bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b) { - if (a->numOfEps != b->numOfEps) { - return false; - } - for (int i = 0; i < a->numOfEps; i++) { - if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { - return false; - } - } - return true; -} - static void transInitEnv() { refMgt = transOpenRefMgt(50000, transDestroyExHandle); svrRefMgt = transOpenRefMgt(50000, transDestroyExHandle); @@ -905,12 +772,6 @@ int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { *ppBuf = pBuf; return len; } - -// int32_t transGenRandomError(int32_t status) { -// STUB_RAND_NETWORK_ERR(status) -// return status; -// } - int32_t initWQ(queue* wq) { int32_t code = 0; QUEUE_INIT(wq); @@ -957,6 +818,7 @@ uv_write_t* allocWReqFromWQ(queue* wq, void* arg) { QUEUE_INIT(&w->node); return &w->wreq; } + return NULL; } void freeWReqToWQ(queue* wq, SWReqsWrapper* w) { @@ -976,47 +838,1192 @@ int32_t transSetReadOption(uv_handle_t* handle) { return code; } -int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet) { - if (pEpset == NULL) { - return TSDB_CODE_INVALID_PARA; - } +#else +#define BUFFER_CAP 4096 + +typedef struct { + int32_t numOfThread; + STaosQueue** qhandle; + STaosQset** qset; + int64_t idx; + +} MultiThreadQhandle; +typedef struct TThread { + TdThread thread; + int idx; +} TThread; + +TdThreadMutex mutex[2]; +MultiThreadQhandle* multiQ[2] = {NULL, NULL}; +static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; - if (pReqEpSet == NULL) { - return TSDB_CODE_INVALID_PARA; - } +static int32_t refMgt; +static int32_t svrRefMgt; +static int32_t instMgt; +static int32_t transSyncMsgMgt; +TdThreadMutex mutex[2]; - int32_t size = sizeof(SReqEpSet) + sizeof(SEp) * pEpset->numOfEps; - SReqEpSet* pReq = (SReqEpSet*)taosMemoryCalloc(1, size); - if (pReq == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; +TdThreadMutex tableMutex; +SHashObj* hashTable = NULL; + +void transDestroySyncMsg(void* msg); + +int32_t transCompressMsg(char* msg, int32_t len) { + int32_t ret = 0; + int compHdr = sizeof(STransCompMsg); + STransMsgHead* pHead = transHeadFromCont(msg); + + char* buf = taosMemoryMalloc(len + compHdr + 8); // 8 extra bytes + if (buf == NULL) { + tWarn("failed to allocate memory for rpc msg compression, contLen:%d", len); + ret = len; + return ret; } - memcpy((char*)pReq, (char*)pEpset, size); - // clear previous - taosMemoryFree(*pReqEpSet); - if (transValidReqEpset(pReq) != TSDB_CODE_SUCCESS) { - taosMemoryFree(pReq); - return TSDB_CODE_INVALID_PARA; + int32_t clen = LZ4_compress_default(msg, buf, len, len + compHdr); + /* + * only the compressed size is less than the value of contLen - overhead, the compression is applied + * The first four bytes is set to 0, the second four bytes are utilized to keep the original length of message + */ + if (clen > 0 && clen < len - compHdr) { + STransCompMsg* pComp = (STransCompMsg*)msg; + pComp->reserved = 0; + pComp->contLen = htonl(len); + memcpy(msg + compHdr, buf, clen); + + tDebug("compress rpc msg, before:%d, after:%d", len, clen); + ret = clen + compHdr; + pHead->comp = 1; + } else { + ret = len; + pHead->comp = 0; } + taosMemoryFree(buf); + return ret; +} +int32_t transDecompressMsg(char** msg, int32_t* len) { + return 0; + // STransMsgHead* pHead = (STransMsgHead*)(*msg); + // if (pHead->comp == 0) return 0; - *pReqEpSet = pReq; - return TSDB_CODE_SUCCESS; + // char* pCont = transContFromHead(pHead); + + // STransCompMsg* pComp = (STransCompMsg*)pCont; + // int32_t oriLen = htonl(pComp->contLen); + + // char* buf = taosMemoryCalloc(1, oriLen + sizeof(STransMsgHead)); + // if (buf == NULL) { + // return terrno; + // } + + // STransMsgHead* pNewHead = (STransMsgHead*)buf; + // int32_t decompLen = LZ4_decompress_safe(pCont + sizeof(STransCompMsg), (char*)pNewHead->content, + // len - sizeof(STransMsgHead) - sizeof(STransCompMsg), oriLen); + // memcpy((char*)pNewHead, (char*)pHead, sizeof(STransMsgHead)); + + // pNewHead->msgLen = htonl(oriLen + sizeof(STransMsgHead)); + + // taosMemoryFree(pHead); + // *msg = buf; + // if (decompLen != oriLen) { + // return TSDB_CODE_INVALID_MSG; + // } + // return 0; } -int32_t transCreateUserEpsetFromReqEpset(const SReqEpSet* pReqEpSet, SEpSet* pEpSet) { - if (pReqEpSet == NULL) { - return TSDB_CODE_INVALID_PARA; +void transFreeMsg(void* msg) { + if (msg == NULL) { + return; } - memcpy((char*)pEpSet, (char*)pReqEpSet, sizeof(SReqEpSet) + sizeof(SEp) * pReqEpSet->numOfEps); - return TSDB_CODE_SUCCESS; + tTrace("rpc free cont:%p", (char*)msg - TRANS_MSG_OVERHEAD); + taosMemoryFree((char*)msg - sizeof(STransMsgHead)); } +// int transSockInfo2Str(struct sockaddr* sockname, char* dst) { +// return 0; +// struct sockaddr_in addr = *(struct sockaddr_in*)sockname; + +// char buf[20] = {0}; +// int r = uv_ip4_name(&addr, (char*)buf, sizeof(buf)); +// sprintf(dst, "%s:%d", buf, ntohs(addr.sin_port)); +// return r; +//} +// int32_t transInitBuffer(SConnBuffer* buf) { +// buf->buf = taosMemoryCalloc(1, BUFFER_CAP); +// if (buf->buf == NULL) { +// return terrno; +// } + +// buf->cap = BUFFER_CAP; +// buf->left = -1; +// buf->len = 0; +// buf->total = 0; +// buf->invalid = 0; +// return 0; +//} +// int32_t transDestroyBuffer(SConnBuffer* p) { +// taosMemoryFree(p->buf); +// p->buf = NULL; +// return 0; +//} + +// int32_t transClearBuffer(SConnBuffer* buf) { +// SConnBuffer* p = buf; +// if (p->cap > BUFFER_CAP) { +// p->cap = BUFFER_CAP; +// p->buf = taosMemoryRealloc(p->buf, BUFFER_CAP); +// if (p->buf == NULL) { +// return terrno; +// } +// } +// p->left = -1; +// p->len = 0; +// p->total = 0; +// p->invalid = 0; +// return 0; +//} + +// int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { +// static const int HEADSIZE = sizeof(STransMsgHead); +// return 0; +// int32_t code = 0; +// SConnBuffer* p = connBuf; +// if (p->left != 0 || p->total <= 0) { +// return TSDB_CODE_INVALID_MSG; +// } +// int total = p->total; +// if (total >= HEADSIZE && !p->invalid) { +// *buf = taosMemoryCalloc(1, total); +// if (*buf == NULL) { +// return terrno; +// } +// memcpy(*buf, p->buf, total); +// if ((code = transResetBuffer(connBuf, resetBuf)) < 0) { +// return code; +// } +// } else { +// total = -1; +// return TSDB_CODE_INVALID_MSG; +// } +// return total; +//} + +// int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { +// // SConnBuffer* p = connBuf; +// // if (p->total < p->len) { +// // int left = p->len - p->total; +// // memmove(p->buf, p->buf + p->total, left); +// // p->left = -1; +// // p->total = 0; +// // p->len = left; +// // } else if (p->total == p->len) { +// // p->left = -1; +// // p->total = 0; +// // p->len = 0; +// // if (p->cap > BUFFER_CAP) { +// // if (resetBuf) { +// // p->cap = BUFFER_CAP; +// // p->buf = taosMemoryRealloc(p->buf, p->cap); +// // if (p->buf == NULL) { +// // return terrno; +// // } +// // } +// // } +// // } else { +// // tError("failed to reset buffer, total:%d, len:%d, reason:%s", p->total, p->len, +// // tstrerror(TSDB_CODE_INVALID_MSG)); return TSDB_CODE_INVALID_MSG; +// // } +// return 0; +// } +// int32_t transAllocBuffer(SConnBuffer* connBuf, uv_buf_t* uvBuf) { +// // /* +// // * formate of data buffer: +// // * |<--------------------------data from socket------------------------------->| +// // * |<------STransMsgHead------->|<-------------------userdata--------------->|<-----auth data----->|<----user +// // * info--->| +// // */ +// // SConnBuffer* p = connBuf; +// // uvBuf->base = p->buf + p->len; +// // if (p->left == -1) { +// // uvBuf->len = p->cap - p->len; +// // } else { +// // if (p->left < p->cap - p->len) { +// // uvBuf->len = p->left; +// // } else { +// // p->cap = p->left + p->len; +// // p->buf = taosMemoryRealloc(p->buf, p->cap); +// // if (p->buf == NULL) { +// // uvBuf->base = NULL; +// // uvBuf->len = 0; +// // return terrno; +// // } +// // uvBuf->base = p->buf + p->len; +// // uvBuf->len = p->left; +// // } +// // } +// return 0; +// } +// check whether already read complete +// bool transReadComplete(SConnBuffer* connBuf) { +// SConnBuffer* p = connBuf; +// return false; +// if (p->len >= sizeof(STransMsgHead)) { +// if (p->left == -1) { +// STransMsgHead head; +// memcpy((char*)&head, connBuf->buf, sizeof(head)); +// int32_t msgLen = (int32_t)htonl(head.msgLen); +// p->total = msgLen; +// p->invalid = TRANS_NOVALID_PACKET(htonl(head.magicNum)) || head.version != TRANS_VER; +// } +// if (p->total >= p->len) { +// p->left = p->total - p->len; +// } else { +// p->left = 0; +// } +// } +// return (p->left == 0 || p->invalid) ? true : false; +//} + +// int32_t transSetConnOption(uv_tcp_t* stream, int keepalive) { +// #if defined(WINDOWS) || defined(DARWIN) +// #else +// return uv_tcp_keepalive(stream, 1, keepalive); +// #endif +// return uv_tcp_nodelay(stream, 1); +// // int ret = uv_tcp_keepalive(stream, 5, 60); +// } -int32_t transValidReqEpset(SReqEpSet* pReqEpSet) { - if (pReqEpSet == NULL) { - return TSDB_CODE_INVALID_PARA; +// int32_t transAsyncPoolCreate(uv_loop_t* loop, int sz, void* arg, AsyncCB cb, SAsyncPool** pPool) { +// SAsyncPool* pool = taosMemoryCalloc(1, sizeof(SAsyncPool)); +// if (pool == NULL) { +// return terrno; +// // return NULL; +// } +// int32_t code = 0; + +// pool->nAsync = sz; +// pool->asyncs = taosMemoryCalloc(1, sizeof(uv_async_t) * pool->nAsync); +// if (pool->asyncs == NULL) { +// taosMemoryFree(pool); +// return terrno; +// } + +// int i = 0, err = 0; +// for (i = 0; i < pool->nAsync; i++) { +// uv_async_t* async = &(pool->asyncs[i]); + +// SAsyncItem* item = taosMemoryCalloc(1, sizeof(SAsyncItem)); +// if (item == NULL) { +// code = terrno; +// break; +// } +// item->pThrd = arg; +// QUEUE_INIT(&item->qmsg); +// code = taosThreadMutexInit(&item->mtx, NULL); +// if (code) { +// taosMemoryFree(item); +// break; +// } + +// async->data = item; +// err = uv_async_init(loop, async, cb); +// if (err != 0) { +// tError("failed to init async, reason:%s", uv_err_name(err)); +// code = TSDB_CODE_THIRDPARTY_ERROR; +// break; +// } +// } + +// if (i != pool->nAsync) { +// transAsyncPoolDestroy(pool); +// pool = NULL; +// } + +// *pPool = pool; +// return 0; +// // return pool; +// } + +// void transAsyncPoolDestroy(SAsyncPool* pool) { +// if (pool == NULL) return; + +// for (int i = 0; i < pool->nAsync; i++) { +// uv_async_t* async = &(pool->asyncs[i]); +// SAsyncItem* item = async->data; +// if (item == NULL) continue; + +// TAOS_UNUSED(taosThreadMutexDestroy(&item->mtx)); +// taosMemoryFree(item); +// } +// taosMemoryFree(pool->asyncs); +// taosMemoryFree(pool); +// } +// bool transAsyncPoolIsEmpty(SAsyncPool* pool) { +// for (int i = 0; i < pool->nAsync; i++) { +// uv_async_t* async = &(pool->asyncs[i]); +// SAsyncItem* item = async->data; +// if (!QUEUE_IS_EMPTY(&item->qmsg)) return false; +// } +// return true; +// } +// int transAsyncSend(SAsyncPool* pool, queue* q) { +// if (atomic_load_8(&pool->stop) == 1) { +// return TSDB_CODE_RPC_ASYNC_MODULE_QUIT; +// } +// int idx = pool->index % pool->nAsync; + +// // no need mutex here +// if (pool->index++ > pool->nAsync * 2000) { +// pool->index = 0; +// } +// uv_async_t* async = &(pool->asyncs[idx]); +// SAsyncItem* item = async->data; + +// if (taosThreadMutexLock(&item->mtx) != 0) { +// tError("failed to lock mutex"); +// } + +// QUEUE_PUSH(&item->qmsg, q); +// TAOS_UNUSED(taosThreadMutexUnlock(&item->mtx)); +// int ret = uv_async_send(async); +// if (ret != 0) { +// tError("failed to send async,reason:%s", uv_err_name(ret)); +// return TSDB_CODE_THIRDPARTY_ERROR; +// } +// return 0; +// } + +void transCtxInit(STransCtx* ctx) { + // init transCtx + ctx->args = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); +} +void transCtxCleanup(STransCtx* ctx) { + if (ctx == NULL || ctx->args == NULL) { + return; } - if (pReqEpSet->numOfEps == 0 || pReqEpSet->numOfEps > TSDB_MAX_EP_NUM || pReqEpSet->inUse >= TSDB_MAX_EP_NUM) { - return TSDB_CODE_INVALID_PARA; + + STransCtxVal* iter = taosHashIterate(ctx->args, NULL); + while (iter) { + ctx->freeFunc(iter->val); + iter = taosHashIterate(ctx->args, iter); } - return TSDB_CODE_SUCCESS; + if (ctx->freeFunc) ctx->freeFunc(ctx->brokenVal.val); + taosHashCleanup(ctx->args); + ctx->args = NULL; +} + +void transCtxMerge(STransCtx* dst, STransCtx* src) { + if (src->args == NULL || src->freeFunc == NULL) { + return; + } + if (dst->args == NULL) { + dst->args = src->args; + dst->brokenVal = src->brokenVal; + dst->freeFunc = src->freeFunc; + src->args = NULL; + return; + } + void* key = NULL; + size_t klen = 0; + void* iter = taosHashIterate(src->args, NULL); + while (iter) { + STransCtxVal* sVal = (STransCtxVal*)iter; + key = taosHashGetKey(sVal, &klen); + + int32_t code = taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); + if (code != 0) { + tError("failed to put val to hash, reason:%s", tstrerror(code)); + } + iter = taosHashIterate(src->args, iter); + } + taosHashCleanup(src->args); +} +void* transCtxDumpVal(STransCtx* ctx, int32_t key) { + if (ctx->args == NULL) { + return NULL; + } + STransCtxVal* cVal = taosHashGet(ctx->args, (const void*)&key, sizeof(key)); + if (cVal == NULL) { + return NULL; + } + void* ret = NULL; + TAOS_UNUSED((*cVal->clone)(cVal->val, &ret)); + return ret; +} +void* transCtxDumpBrokenlinkVal(STransCtx* ctx, int32_t* msgType) { + void* ret = NULL; + if (ctx->brokenVal.clone == NULL) { + return ret; + } + TAOS_UNUSED((*ctx->brokenVal.clone)(ctx->brokenVal.val, &ret)); + + *msgType = ctx->brokenVal.msgType; + + return ret; +} + +// void transReqQueueInit(queue* q) { +// // init req queue +// QUEUE_INIT(q); +// } +// void* transReqQueuePush(queue* q) { +// STransReq* req = taosMemoryCalloc(1, sizeof(STransReq)); +// if (req == NULL) { +// return NULL; +// } +// req->wreq.data = req; +// QUEUE_PUSH(q, &req->q); +// return &req->wreq; +// } +// void* transReqQueueRemove(void* arg) { +// void* ret = NULL; +// uv_write_t* wreq = arg; + +// STransReq* req = wreq ? wreq->data : NULL; +// if (req == NULL) return NULL; +// QUEUE_REMOVE(&req->q); + +// ret = wreq && wreq->handle ? wreq->handle->data : NULL; +// taosMemoryFree(req); + +// return ret; +// } +// void transReqQueueClear(queue* q) { +// while (!QUEUE_IS_EMPTY(q)) { +// queue* h = QUEUE_HEAD(q); +// QUEUE_REMOVE(h); +// STransReq* req = QUEUE_DATA(h, STransReq, q); +// taosMemoryFree(req); +// } +// } + +// int32_t transQueueInit(STransQueue* queue, void (*freeFunc)(const void* arg)) { +// queue->q = taosArrayInit(2, sizeof(void*)); +// if (queue->q == NULL) { +// return terrno; +// } +// queue->freeFunc = (void (*)(const void*))freeFunc; + +// return 0; +// } +// bool transQueuePush(STransQueue* queue, void* arg) { +// if (queue->q == NULL) { +// return true; +// } +// if (taosArrayPush(queue->q, &arg) == NULL) { +// return false; +// } +// if (taosArrayGetSize(queue->q) > 1) { +// return false; +// } +// return true; +// } +// void* transQueuePop(STransQueue* queue) { +// if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { +// return NULL; +// } +// void* ptr = taosArrayGetP(queue->q, 0); +// taosArrayRemove(queue->q, 0); +// return ptr; +// } +// int32_t transQueueSize(STransQueue* queue) { +// if (queue->q == NULL) { +// return 0; +// } +// return taosArrayGetSize(queue->q); +// } +// void* transQueueGet(STransQueue* queue, int i) { +// if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { +// return NULL; +// } +// if (i >= taosArrayGetSize(queue->q)) { +// return NULL; +// } + +// void* ptr = taosArrayGetP(queue->q, i); +// return ptr; +// } + +// void* transQueueRm(STransQueue* queue, int i) { +// if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { +// return NULL; +// } +// if (i >= taosArrayGetSize(queue->q)) { +// return NULL; +// } +// void* ptr = taosArrayGetP(queue->q, i); +// taosArrayRemove(queue->q, i); +// return ptr; +// } + +// bool transQueueEmpty(STransQueue* queue) { +// if (queue->q == NULL) { +// return true; +// } +// return taosArrayGetSize(queue->q) == 0; +// } +// void transQueueClear(STransQueue* queue) { +// if (queue->freeFunc != NULL) { +// for (int i = 0; i < taosArrayGetSize(queue->q); i++) { +// void* p = taosArrayGetP(queue->q, i); +// queue->freeFunc(p); +// } +// } +// taosArrayClear(queue->q); +// } +// void transQueueDestroy(STransQueue* queue) { +// transQueueClear(queue); +// taosArrayDestroy(queue->q); +// } + +// static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) { +// SDelayTask* arg1 = container_of(a, SDelayTask, node); +// SDelayTask* arg2 = container_of(b, SDelayTask, node); +// if (arg1->execTime > arg2->execTime) { +// return 0; +// } else { +// return 1; +// } +// } + +// static void transDQTimeout(uv_timer_t* timer) { +// SDelayQueue* queue = timer->data; +// tTrace("timer %p timeout", timer); +// uint64_t timeout = 0; +// int64_t current = taosGetTimestampMs(); +// do { +// HeapNode* minNode = heapMin(queue->heap); +// if (minNode == NULL) break; +// SDelayTask* task = container_of(minNode, SDelayTask, node); +// if (task->execTime <= current) { +// heapRemove(queue->heap, minNode); +// task->func(task->arg); +// taosMemoryFree(task); +// timeout = 0; +// } else { +// timeout = task->execTime - current; +// break; +// } +// } while (1); +// if (timeout != 0) { +// TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeout, 0)); +// } +// } +// int32_t transDQCreate(uv_loop_t* loop, SDelayQueue** queue) { return 0; } + +// void transDQDestroy(SDelayQueue* queue, void (*freeFunc)(void* arg)) { return; } +// void transDQCancel(SDelayQueue* queue, SDelayTask* task) { return; } + +// SDelayTask* transDQSched(SDelayQueue* queue, void (*func)(void* arg), void* arg, uint64_t timeoutMs) { return NULL; } + +// void transPrintEpSet(SEpSet* pEpSet) { +// if (pEpSet == NULL) { +// tTrace("NULL epset"); +// return; +// } +// char buf[512] = {0}; +// int len = snprintf(buf, sizeof(buf), "epset:{"); +// for (int i = 0; i < pEpSet->numOfEps; i++) { +// if (i == pEpSet->numOfEps - 1) { +// len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); +// } else { +// len += snprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); +// } +// } +// len += snprintf(buf + len, sizeof(buf) - len, "}"); +// tTrace("%s, inUse:%d", buf, pEpSet->inUse); +// } +// bool transEpSetIsEqual(SEpSet* a, SEpSet* b) { +// if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { +// return false; +// } +// for (int i = 0; i < a->numOfEps; i++) { +// if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { +// return false; +// } +// } +// return true; +// } +// bool transEpSetIsEqual2(SEpSet* a, SEpSet* b) { +// if (a->numOfEps != b->numOfEps) { +// return false; +// } +// for (int i = 0; i < a->numOfEps; i++) { +// if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { +// return false; +// } +// } +// return true; +// } + +bool cliMayGetAhandle(STrans* pTrans, SRpcMsg* pMsg) { + int64_t seq = pMsg->info.seq; + int32_t* msgType = NULL; + + taosThreadMutexLock(&pTrans->seqMutex); + msgType = taosHashGet(pTrans->seqTable, &seq, sizeof(seq)); + taosThreadMutexUnlock(&pTrans->seqMutex); + if (msgType == NULL) { + STransCtx* ctx = taosHashGet(pTrans->sidTable, &pMsg->info.qId, sizeof(pMsg->info.qId)); + if (ctx == NULL) { + return false; + } + pMsg->info.ahandle = transCtxDumpVal(ctx, pMsg->msgType); + tError("failed to find msg type for seq:%" PRId64 ", gen ahandle for type %s" PRId64, seq, + TMSG_INFO(pMsg->msgType)); + } else { + taosThreadMutexLock(&pTrans->seqMutex); + taosHashRemove(pTrans->seqTable, &seq, sizeof(seq)); + msgType = taosHashGet(pTrans->seqTable, &seq, sizeof(seq)); + taosThreadMutexUnlock(&pTrans->seqMutex); + } + return true; +} + +void* processSvrMsg(void* arg) { + TThread* thread = (TThread*)arg; + + int32_t idx = thread->idx; + static int num = 0; + STaosQall* qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + SQueueInfo qinfo = {0}; + + taosAllocateQall(&qall); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(multiQ[1]->qset[idx], qall, &qinfo); + if (numOfMsgs <= 0) break; + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; i++) { + taosGetQitem(qall, (void**)&pRpcMsg); + taosThreadMutexLock(&mutex[1]); + RpcCfp fp = NULL; + void* parent = NULL; + STraceId* trace = &pRpcMsg->info.traceId; + tGDebug("taos %s received from taosd", TMSG_INFO(pRpcMsg->msgType)); + STrans* pTrans = NULL; + transGetCb(pRpcMsg->type, &pTrans); + + taosThreadMutexUnlock(&mutex[1]); + + if (pTrans != NULL) { + if (cliMayGetAhandle(pTrans, pRpcMsg)) { + if (pRpcMsg->info.reqWithSem == NULL) { + (pTrans->cfp)(pTrans->parent, pRpcMsg, NULL); + } else { + STransReqWithSem* reqWithSem = pRpcMsg->info.reqWithSem; + memcpy(&reqWithSem->pMsg, pRpcMsg, sizeof(SRpcMsg)); + tsem_post(reqWithSem->sem); + } + } else { + tDebug("taosd %s received from taosd, ignore", TMSG_INFO(pRpcMsg->msgType)); + } + } + } + taosUpdateItemSize(qinfo.queue, numOfMsgs); + } + + taosFreeQall(qall); + return NULL; +} +void* procClientMsg(void* arg) { + TThread* thread = (TThread*)arg; + + int32_t idx = thread->idx; + static int num = 0; + STaosQall* qall; + SRpcMsg * pRpcMsg, rpcMsg; + int type; + SQueueInfo qinfo = {0}; + + taosAllocateQall(&qall); + + while (1) { + int numOfMsgs = taosReadAllQitemsFromQset(multiQ[0]->qset[idx], qall, &qinfo); + tDebug("%d msgs are received", numOfMsgs); + if (numOfMsgs <= 0) break; + taosResetQitems(qall); + for (int i = 0; i < numOfMsgs; i++) { + taosGetQitem(qall, (void**)&pRpcMsg); + + STraceId* trace = &pRpcMsg->info.traceId; + tDebug("taosc %s received from taosc", TMSG_INFO(pRpcMsg->msgType)); + RpcCfp fp = NULL; + // void* parent; + STrans* pTrans = NULL; + taosThreadMutexLock(&mutex[1]); + if ((pRpcMsg->type & TD_ACORE_DSVR) != 0) { + transGetCb(TD_ACORE_DSVR, &pTrans); + } + taosThreadMutexUnlock(&mutex[1]); + if (pTrans->cfp != NULL) { + (pTrans->cfp)(pTrans->parent, pRpcMsg, NULL); + } else { + tError("taosc failed to find callback for msg type:%s", TMSG_INFO(pRpcMsg->msgType)); + } + } + taosUpdateItemSize(qinfo.queue, numOfMsgs); + } + + taosFreeQall(qall); + return NULL; +} +static void transInitEnv() { + refMgt = transOpenRefMgt(50000, transDestroyExHandle); + svrRefMgt = transOpenRefMgt(50000, transDestroyExHandle); + instMgt = taosOpenRef(50, rpcCloseImpl); + transSyncMsgMgt = taosOpenRef(50, transDestroySyncMsg); + + taosThreadMutexInit(&tableMutex, NULL); + hashTable = taosHashInit(2, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UINT), true, HASH_NO_LOCK); + + int32_t numOfAthread = 1; + + multiQ[0] = taosMemoryMalloc(sizeof(MultiThreadQhandle)); + multiQ[0]->numOfThread = numOfAthread; + multiQ[0]->qhandle = (STaosQueue**)taosMemoryMalloc(sizeof(STaosQueue*) * numOfAthread); + multiQ[0]->qset = (STaosQset**)taosMemoryMalloc(sizeof(STaosQset*) * numOfAthread); + + taosThreadMutexInit(&mutex[0], NULL); + + for (int i = 0; i < numOfAthread; i++) { + taosOpenQueue(&(multiQ[0]->qhandle[i])); + taosOpenQset(&multiQ[0]->qset[i]); + taosAddIntoQset(multiQ[0]->qset[i], multiQ[0]->qhandle[i], NULL); + } + { + TThread* threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); + for (int i = 0; i < numOfAthread; i++) { + threads[i].idx = i; + taosThreadCreate(&(threads[i].thread), NULL, procClientMsg, (void*)&threads[i]); + } + } + + multiQ[1] = taosMemoryMalloc(sizeof(MultiThreadQhandle)); + multiQ[1]->numOfThread = numOfAthread; + multiQ[1]->qhandle = (STaosQueue**)taosMemoryMalloc(sizeof(STaosQueue*) * numOfAthread); + multiQ[1]->qset = (STaosQset**)taosMemoryMalloc(sizeof(STaosQset*) * numOfAthread); + taosThreadMutexInit(&mutex[1], NULL); + + for (int i = 0; i < numOfAthread; i++) { + taosOpenQueue(&(multiQ[1]->qhandle[i])); + taosOpenQset(&multiQ[1]->qset[i]); + taosAddIntoQset(multiQ[1]->qset[i], multiQ[1]->qhandle[i], NULL); + } + { + TThread* threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); + for (int i = 0; i < numOfAthread; i++) { + threads[i].idx = i; + taosThreadCreate(&(threads[i].thread), NULL, processSvrMsg, (void*)&threads[i]); + } + } +} +static void transDestroyEnv() { + transCloseRefMgt(refMgt); + transCloseRefMgt(svrRefMgt); +} + +typedef struct { + void (*fp)(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); + RPC_TYPE type; + void* parant; + STrans* pTransport; +} FP_TYPE; +int32_t transUpdateCb(RPC_TYPE type, STrans* pTransport) { + taosThreadMutexLock(&tableMutex); + + FP_TYPE t = {.type = type, .pTransport = pTransport}; + int32_t code = taosHashPut(hashTable, &type, sizeof(type), &t, sizeof(t)); + taosThreadMutexUnlock(&tableMutex); + return 0; +} +int32_t transGetCb(RPC_TYPE type, STrans** ppTransport) { + taosThreadMutexLock(&tableMutex); + void* p = taosHashGet(hashTable, &type, sizeof(type)); + if (p == NULL) { + taosThreadMutexUnlock(&tableMutex); + return TSDB_CODE_INVALID_MSG; + } + FP_TYPE* t = p; + *ppTransport = t->pTransport; + // *fp = t->fp; + // *arg = t->parant; + taosThreadMutexUnlock(&tableMutex); + return 0; +} + +int32_t transSendReq(STrans* pTransport, SRpcMsg* pMsg, void* pEpSet) { + SRpcMsg* pTemp; + + taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void**)&pTemp); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + int64_t cidx = multiQ[0]->idx++; + int32_t idx = cidx % (multiQ[0]->numOfThread); + tDebug("taos request is sent , type:%s, contLen:%d, item:%p", TMSG_INFO(pMsg->msgType), pMsg->contLen, pTemp); + taosWriteQitem(multiQ[0]->qhandle[idx], pTemp); + return 0; +} +int32_t transSendResp(const SRpcMsg* pMsg) { + SRpcMsg* pTemp; + + taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void**)&pTemp); + memcpy(pTemp, pMsg, sizeof(SRpcMsg)); + + int64_t cidx = multiQ[1]->idx++; + int32_t idx = cidx % (multiQ[1]->numOfThread); + tDebug("taos resp is sent to, type:%s, contLen:%d, item:%p", TMSG_INFO(pMsg->msgType), pMsg->contLen, pTemp); + taosWriteQitem(multiQ[1]->qhandle[idx], pTemp); + return 0; +} + +int32_t transInit() { + // init env + int32_t code = taosThreadOnce(&transModuleInit, transInitEnv); + if (code != 0) { + code = TAOS_SYSTEM_ERROR(errno); + } + return code; +} + +int32_t transGetRefMgt() { return refMgt; } +int32_t transGetSvrRefMgt() { return svrRefMgt; } +int32_t transGetInstMgt() { return instMgt; } +int32_t transGetSyncMsgMgt() { return transSyncMsgMgt; } + +void transCleanup() { + // clean env + transDestroyEnv(); + return; +} +int32_t transOpenRefMgt(int size, void (*func)(void*)) { + /// add later + return taosOpenRef(size, func); +} +void transCloseRefMgt(int32_t mgt) { + // close ref + taosCloseRef(mgt); + return; +} +int64_t transAddExHandle(int32_t refMgt, void* p) { + return taosAddRef(refMgt, p); + // acquire extern handle +} +void transRemoveExHandle(int32_t refMgt, int64_t refId) { + // acquire extern handle + int32_t code = taosRemoveRef(refMgt, refId); + return; +} + +void* transAcquireExHandle(int32_t refMgt, int64_t refId) { + // acquire extern handle + return (void*)taosAcquireRef(refMgt, refId); +} + +void transReleaseExHandle(int32_t refMgt, int64_t refId) { + // release extern handle + int32_t code = taosReleaseRef(refMgt, refId); + return; +} +void transDestroyExHandle(void* handle) { + if (handle == NULL) { + return; + } + SExHandle* eh = handle; + if (!QUEUE_IS_EMPTY(&eh->q)) { + tDebug("handle %p mem leak", handle); + } + tDebug("free exhandle %p", handle); + taosMemoryFree(handle); + return; +} + +void transDestroySyncMsg(void* msg) { + if (msg == NULL) return; + + STransSyncMsg* pSyncMsg = msg; + TAOS_UNUSED(tsem2_destroy(pSyncMsg->pSem)); + taosMemoryFree(pSyncMsg->pSem); + transFreeMsg(pSyncMsg->pRsp->pCont); + taosMemoryFree(pSyncMsg->pRsp); + taosMemoryFree(pSyncMsg); + return; +} + +uint32_t subnetIpRang2Int(SIpV4Range* pRange) { return 0; } +int32_t subnetInit(SubnetUtils* pUtils, SIpV4Range* pRange) { return 0; } +int32_t subnetDebugInfoToBuf(SubnetUtils* pUtils, char* buf) { return 0; } +int32_t subnetCheckIp(SubnetUtils* pUtils, uint32_t ip) { return 0; } + +int32_t transUtilSIpRangeToStr(SIpV4Range* pRange, char* buf) { return 0; } + +int32_t transUtilSWhiteListToStr(SIpWhiteList* pList, char** ppBuf) { return 0; } + +#endif + +int32_t transInitBuffer(SConnBuffer* buf) { + buf->buf = taosMemoryCalloc(1, BUFFER_CAP); + if (buf->buf == NULL) { + return terrno; + } + + buf->cap = BUFFER_CAP; + buf->left = -1; + buf->len = 0; + buf->total = 0; + buf->invalid = 0; + return 0; +} +void transDestroyBuffer(SConnBuffer* p) { + taosMemoryFree(p->buf); + p->buf = NULL; +} + +int32_t transClearBuffer(SConnBuffer* buf) { + SConnBuffer* p = buf; + if (p->cap > BUFFER_CAP) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, BUFFER_CAP); + if (p->buf == NULL) { + return terrno; + } + } + p->left = -1; + p->len = 0; + p->total = 0; + p->invalid = 0; + return 0; +} + +int32_t transDumpFromBuffer(SConnBuffer* connBuf, char** buf, int8_t resetBuf) { + static const int HEADSIZE = sizeof(STransMsgHead); + int32_t code = 0; + SConnBuffer* p = connBuf; + if (p->left != 0 || p->total <= 0) { + return TSDB_CODE_INVALID_MSG; + } + int total = p->total; + if (total >= HEADSIZE && !p->invalid) { + *buf = taosMemoryCalloc(1, total); + if (*buf == NULL) { + return terrno; + } + memcpy(*buf, p->buf, total); + if ((code = transResetBuffer(connBuf, resetBuf)) < 0) { + return code; + } + } else { + total = -1; + return TSDB_CODE_INVALID_MSG; + } + return total; +} + +int32_t transResetBuffer(SConnBuffer* connBuf, int8_t resetBuf) { + SConnBuffer* p = connBuf; + if (p->total < p->len) { + int left = p->len - p->total; + memmove(p->buf, p->buf + p->total, left); + p->left = -1; + p->total = 0; + p->len = left; + } else if (p->total == p->len) { + p->left = -1; + p->total = 0; + p->len = 0; + if (p->cap > BUFFER_CAP) { + if (resetBuf) { + p->cap = BUFFER_CAP; + p->buf = taosMemoryRealloc(p->buf, p->cap); + if (p->buf == NULL) { + return terrno; + } + } + } + } else { + tError("failed to reset buffer, total:%d, len:%d since %s", p->total, p->len, tstrerror(TSDB_CODE_INVALID_MSG)); + return TSDB_CODE_INVALID_MSG; + } + return 0; +} + +int32_t transCreateReqEpsetFromUserEpset(const SEpSet* pEpset, SReqEpSet** pReqEpSet) { + if (pEpset == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + if (pReqEpSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + int32_t size = sizeof(SReqEpSet) + sizeof(SEp) * pEpset->numOfEps; + SReqEpSet* pReq = (SReqEpSet*)taosMemoryCalloc(1, size); + if (pReq == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy((char*)pReq, (char*)pEpset, size); + // clear previous + taosMemoryFree(*pReqEpSet); + + if (transValidReqEpset(pReq) != TSDB_CODE_SUCCESS) { + taosMemoryFree(pReq); + return TSDB_CODE_INVALID_PARA; + } + + *pReqEpSet = pReq; + return TSDB_CODE_SUCCESS; +} + +int32_t transCreateUserEpsetFromReqEpset(const SReqEpSet* pReqEpSet, SEpSet* pEpSet) { + if (pReqEpSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } + memcpy((char*)pEpSet, (char*)pReqEpSet, sizeof(SReqEpSet) + sizeof(SEp) * pReqEpSet->numOfEps); + return TSDB_CODE_SUCCESS; +} + +int32_t transValidReqEpset(SReqEpSet* pReqEpSet) { + if (pReqEpSet == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (pReqEpSet->numOfEps == 0 || pReqEpSet->numOfEps > TSDB_MAX_EP_NUM || pReqEpSet->inUse >= TSDB_MAX_EP_NUM) { + return TSDB_CODE_INVALID_PARA; + } + return TSDB_CODE_SUCCESS; +} + +int32_t transQueueInit(STransQueue* wq, void (*freeFunc)(void* arg)) { + QUEUE_INIT(&wq->node); + wq->freeFunc = (void (*)(void*))freeFunc; + wq->size = 0; + wq->inited = 1; + return 0; +} +void transQueuePush(STransQueue* q, void* arg) { + queue* node = arg; + QUEUE_PUSH(&q->node, node); + q->size++; +} +void* transQueuePop(STransQueue* q) { + if (q->size == 0) return NULL; + + queue* head = QUEUE_HEAD(&q->node); + QUEUE_REMOVE(head); + q->size--; + return head; +} +int32_t transQueueSize(STransQueue* q) { return q->size; } + +void* transQueueGet(STransQueue* q, int idx) { + if (q->size == 0) return NULL; + + while (idx-- > 0) { + queue* node = QUEUE_NEXT(&q->node); + if (node == &q->node) return NULL; + } + return NULL; +} + +void transQueueRemoveByFilter(STransQueue* q, bool (*filter)(void* e, void* arg), void* arg, void* dst, int32_t size) { + queue* d = dst; + queue* node = QUEUE_NEXT(&q->node); + while (node != &q->node) { + queue* next = QUEUE_NEXT(node); + if (filter && filter(node, arg)) { + QUEUE_REMOVE(node); + q->size--; + QUEUE_PUSH(d, node); + if (--size == 0) { + break; + } + } + node = next; + } +} + +void* tranQueueHead(STransQueue* q) { + if (q->size == 0) return NULL; + + queue* head = QUEUE_HEAD(&q->node); + return head; +} + +void* transQueueRm(STransQueue* q, int i) { + // if (queue->q == NULL || taosArrayGetSize(queue->q) == 0) { + // return NULL; + // } + // if (i >= taosArrayGetSize(queue->q)) { + // return NULL; + // } + // void* ptr = taosArrayGetP(queue->q, i); + // taosArrayRemove(queue->q, i); + // return ptr; + return NULL; +} + +void transQueueRemove(STransQueue* q, void* e) { + if (q->size == 0) return; + queue* node = e; + QUEUE_REMOVE(node); + q->size--; +} + +bool transQueueEmpty(STransQueue* q) { return q->size == 0 ? true : false; } + +void transQueueClear(STransQueue* q) { + if (q->inited == 0) return; + while (!QUEUE_IS_EMPTY(&q->node)) { + queue* h = QUEUE_HEAD(&q->node); + QUEUE_REMOVE(h); + if (q->freeFunc != NULL) (q->freeFunc)(h); + q->size--; + } +} +void transQueueDestroy(STransQueue* q) { transQueueClear(q); } + +void transPrintEpSet(SEpSet* pEpSet) { + if (pEpSet == NULL) { + tTrace("NULL epset"); + return; + } + char buf[512] = {0}; + int len = tsnprintf(buf, sizeof(buf), "epset:{"); + for (int i = 0; i < pEpSet->numOfEps; i++) { + if (i == pEpSet->numOfEps - 1) { + len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } else { + len += tsnprintf(buf + len, sizeof(buf) - len, "%d. %s:%d, ", i, pEpSet->eps[i].fqdn, pEpSet->eps[i].port); + } + } + len += tsnprintf(buf + len, sizeof(buf) - len, "}"); + tTrace("%s, inUse:%d", buf, pEpSet->inUse); +} +bool transReqEpsetIsEqual(SReqEpSet* a, SReqEpSet* b) { + if (a == NULL && b == NULL) { + return true; + } else if (a == NULL || b == NULL) { + return false; + } + + if (a->numOfEps != b->numOfEps || a->inUse != b->inUse) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; +} +bool transCompareReqAndUserEpset(SReqEpSet* a, SEpSet* b) { + if (a->numOfEps != b->numOfEps) { + return false; + } + for (int i = 0; i < a->numOfEps; i++) { + if (strncmp(a->eps[i].fqdn, b->eps[i].fqdn, TSDB_FQDN_LEN) != 0 || a->eps[i].port != b->eps[i].port) { + return false; + } + } + return true; } \ No newline at end of file diff --git a/source/libs/transport/src/transImpl2.c b/source/libs/transport/src/transImpl2.c new file mode 100644 index 000000000000..35c9e5f7d74c --- /dev/null +++ b/source/libs/transport/src/transImpl2.c @@ -0,0 +1,4667 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "theap.h" +#include "transComm.h" +#include "tversion.h" +#ifdef TD_ACORE +#include +#include +#include +#include +#include +#include +#include +#include +#include "tmisce.h" + +#endif + +#define DIV_ROUNDUP(x, y) (((x) + ((y)-1)) / (y)) + +#define EVT_TIMEOUT 0x01 +#define EVT_READ 0x02 +#define EVT_WRITE 0x04 +#define EVT_SIGNAL 0x08 +#define EVT_ASYNC 0x10 + +#define SELECT_ALLOC_SIZE(n) (DIV_ROUNDUP(n, NFDBITS) * sizeof(fd_mask)) + +typedef void (*AsyncCb)(void *async, int32_t status); + +typedef struct { + void *data; + int32_t sendFd; + AsyncCb cb; + queue q; + TdThreadMutex mutex; + void *hostThrd; + int8_t stop; +} SAsyncHandle; +typedef struct { + queue q; + int32_t acceptFd; +} SFdArg; +typedef struct { + queue q; +} SFdQueue; + +typedef struct STaskArg { + void *param1; + void *param2; +} STaskArg; +typedef struct SDelayTask { + void (*func)(void *arg); + void *arg; + uint64_t execTime; + HeapNode node; +} SDelayTask; + +typedef struct { + Heap *heap; + void *mgt; +} SDelayQueue; + +int32_t transDQCreate(void *loop, SDelayQueue **queue); +void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)); +SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs); +void transDQCancel(SDelayQueue *queue, SDelayTask *task); +int32_t transDQHandleTimeout(SDelayQueue *queue); +int32_t transDQGetNextTimeout(SDelayQueue *queue); +typedef struct { + int notifyCount; // + int init; // init or not + STransMsg msg; +} SSvrRegArg; +typedef struct SSvrConn { + int32_t ref; + // uv_tcp_t* pTcp; + // uv_timer_t pTimer; + + queue queue; + SConnBuffer readBuf; // read buf, + int inType; + void *pInst; // rpc init + void *ahandle; // + void *hostThrd; + STransQueue resps; + + // SSvrRegArg regArg; + bool broken; // conn broken; + + ConnStatus status; + + uint32_t serverIp; + uint32_t clientIp; + uint16_t port; + + char src[32]; + char dst[32]; + + int64_t refId; + int spi; + char info[64]; + char user[TSDB_UNI_LEN]; // user ID for the link + int8_t userInited; + char secret[TSDB_PASSWORD_LEN]; + char ckey[TSDB_PASSWORD_LEN]; // ciphering key + + int64_t whiteListVer; + + // state req dict + SHashObj *pQTable; + // uv_buf_t *buf; + int32_t bufSize; + queue wq; // uv_write_t queue + int32_t fd; + +} SSvrConn; + +typedef struct { + char *buf; + int32_t len; + int8_t inited; + int32_t offset; + void *data; + +} SEvtBuf; +typedef struct SSvrRespMsg { + SSvrConn *pConn; + STransMsg msg; + queue q; + STransMsgType type; + int64_t seqNum; + void *arg; + FilteFunc func; + int8_t sent; + + SEvtBuf buf; +} SSvrRespMsg; + +#define ASYNC_ERR_JRET(thrd) \ + do { \ + if (thrd->quit) { \ + tTrace("worker thread already quit, ignore msg"); \ + goto _return1; \ + } \ + } while (0) + +static FORCE_INLINE void destroySmsg(SSvrRespMsg *smsg) { + if (smsg == NULL) { + return; + } + transFreeMsg(smsg->msg.pCont); + taosMemoryFree(smsg); +} +typedef struct SWorkThrd { + TdThread thread; + // //uv_connect_t connect_req; + // uv_pipe_t* pipe; + // uv_os_fd_t fd; + // uv_loop_t* loop; + // SAsyncPool* asyncPool; + queue msg; + + queue conn; + void *pInst; + bool quit; + + // SIpWhiteListTab* pWhiteList; + int64_t whiteListVer; + int8_t enableIpWhiteList; + + int32_t connRefMgt; + + int32_t client_count; + int8_t inited; + TdThreadMutex mutex; + SFdQueue fdQueue; + + int32_t pipe_fd[2]; // + SAsyncHandle *notifyNewConnHandle; + + int32_t pipe_queue_fd[2]; + SAsyncHandle *asyncHandle; + + void *pEvtMgt; +} SWorkThrd2; +typedef struct SServerObj { + TdThread thread; + int workerIdx; + int numOfThreads; + int numOfWorkerReady; + SWorkThrd2 **pThreadObj; + + // uv_pipe_t pipeListen; + // uv_pipe_t** pipe; + uint32_t ip; + uint32_t port; + // uv_async_t* pAcceptAsync; // just to quit from from accept thread + int32_t serverFd; + bool inited; +} SServerObj2; + +typedef int32_t (*__sendCb)(void *arg, SEvtBuf *buf, int32_t status); +typedef int32_t (*__readCb)(void *arg, SEvtBuf *buf, int32_t status); +typedef int32_t (*__sendFinishCb)(void *arg, int32_t status); +typedef void (*__asyncCb)(void *arg, int32_t status); + +enum EVT_TYPE { EVT_ASYNC_T = 0x1, EVT_CONN_T = 0x2, EVT_SIGANL_T = 0x4, EVT_NEW_CONN_T = 0x8 }; + +typedef struct { + int32_t fd; + void *data; + int32_t event; + + __sendCb sendCb; + __sendFinishCb sendFinishCb; + __readCb readCb; + __asyncCb asyncCb; + + int8_t evtType; + void *arg; + SEvtBuf buf; + SEvtBuf sendBuf; + int8_t rwRef; +} SFdCbArg; + +typedef struct { + int32_t evtFds; // highest fd in the set + int32_t evtFdsSize; + int32_t resizeOutSets; + fd_set *evtReadSetIn; + fd_set *evtWriteSetIn; + fd_set *evtReadSetOut; + fd_set *evtWriteSetOut; + SHashObj *pFdTable; + void *hostThrd; + int32_t fd[2048]; + int32_t fdIdx; + + char label[128]; + void *arg; + int32_t (*timeoutFunc)(void *arg); + int32_t (*caclTimeout)(void *arg, int64_t *timeout); +} SEvtMgt; + +static int32_t evtMgtResize(SEvtMgt *pOpt, int32_t cap); +static int32_t evtMgtCreate(SEvtMgt **pOpt, char *label); +static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv); +static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg); +static int32_t evtMgtRemove(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg); +static void evtMgtDestroy(SEvtMgt *pOpt); + +int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd); + +static int32_t evtInitPipe(int32_t dst[2]) { + int32_t code = 0; + int fd[2] = {0}; + code = pipe(fd); + if (code < 0) { + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + dst[0] = fd[0]; + dst[1] = fd[1]; + return code; +} +static int32_t evtMgtCreate(SEvtMgt **pOpt, char *label) { + int32_t code = 0; + SEvtMgt *pRes = taosMemoryCalloc(1, sizeof(SEvtMgt)); + if (pRes == NULL) { + *pOpt = NULL; + return terrno; + } + pRes->evtFds = 0; + pRes->evtFdsSize = 0; + pRes->resizeOutSets = 0; + pRes->evtReadSetIn = NULL; + pRes->evtWriteSetIn = NULL; + pRes->evtReadSetOut = NULL; + pRes->evtWriteSetOut = NULL; + pRes->pFdTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_NO_LOCK); + pRes->fdIdx = 0; + memcpy(pRes->label, label, strlen(label)); + + code = evtMgtResize(pRes, 32); + if (code != 0) { + evtMgtDestroy(pRes); + } else { + *pOpt = pRes; + } + + return code; +} +static int32_t evtMgtAddTimoutFunc(SEvtMgt *pOpt, void *arg, int32_t (*timeoutFunc)(void *arg), + int32_t (*caclTimeout)(void *arg, int64_t *timeout)) { + int32_t code = 0; + + // pOpt->arg = arg; + pOpt->timeoutFunc = timeoutFunc; + pOpt->caclTimeout = caclTimeout; + return code; +} + +static int32_t evtBufInit(SEvtBuf *evtBuf) { + int32_t code = 0; + if (evtBuf->inited == 0) { + evtBuf->buf = taosMemoryCalloc(1, 4096); + if (evtBuf->buf == NULL) { + code = terrno; + } else { + evtBuf->len = 4096; + evtBuf->inited = 1; + evtBuf->offset = 0; + } + } + return code; +} + +static int32_t evtBufPush(SEvtBuf *evtBuf, char *buf, int32_t len) { + int32_t code = 0; + if (evtBuf->inited == 0) { + code = evtBufInit(evtBuf); + if (code != 0) { + return code; + } + } + int32_t need = evtBuf->offset + len; + if (need >= evtBuf->len) { + // TOOD opt need + char *tbuf = taosMemoryRealloc(evtBuf->buf, need); + if (tbuf == NULL) { + return terrno; + } + evtBuf->buf = tbuf; + evtBuf->len = need; + } + memcpy(evtBuf->buf + evtBuf->offset, buf, len); + evtBuf->offset += len; + return code; +} + +static int32_t evtBufClear(SEvtBuf *evtBuf) { + int32_t code = 0; + if (evtBuf->inited) { + evtBuf->offset = 0; + memset(evtBuf->buf, 0, evtBuf->len); + } + return code; +} + +static int32_t evtBufDestroy(SEvtBuf *evtBuf) { + if (evtBuf->inited) { + taosMemoryFree(evtBuf->buf); + } + return 0; +} +int32_t evtMgtHandleImpl(SEvtMgt *pOpt, SFdCbArg *pArg, int res) { + int32_t nBytes = 0; + char buf[512] = {0}; + int32_t code = 0; + if (pArg->evtType == EVT_NEW_CONN_T || pArg->evtType == EVT_ASYNC_T) { + // handle new coming conn; + if (res & EVT_READ) { + SAsyncHandle *handle = pArg->arg; + + nBytes = read(pArg->fd, buf, sizeof(buf)); + if (nBytes == 1 && buf[0] == '1') { + handle->cb(handle, 0); + } + tTrace("%s handle async read on fd:%d", pOpt->label, pArg->fd); + } + if (res & EVT_WRITE) { + tTrace("%s handle async write on fd:%d", pOpt->label, pArg->fd); + // handle err + } + } else if (pArg->evtType == EVT_CONN_T) { + if (res & EVT_READ) { + SEvtBuf *pBuf = &pArg->buf; + code = evtBufInit(pBuf); + if (code != 0) { + tError("%s failed to init buf since %s", pOpt->label, tstrerror(code)); + return code; + } + tTrace("%s handle read on fd:%d", pOpt->label, pArg->fd); + nBytes = recv(pArg->fd, pBuf->buf, pBuf->len, 0); + if (nBytes > 0) { + code = pArg->readCb(pArg, pBuf, nBytes); + } else { + code = pArg->readCb(pArg, pBuf, nBytes); + } + // handle read + } + + if (res & EVT_WRITE) { + SEvtBuf *pBuf = &pArg->sendBuf; + code = evtBufInit(pBuf); + if (code != 0) { + tError("%s failed to init wbuf since %s", pOpt->label, tstrerror(code)); + return code; + } + tTrace("%s handle write on fd:%d", pOpt->label, pArg->fd); + + code = pArg->sendCb(pArg, pBuf, 0); + if (code != 0) { + tError("%s failed to build send buf since %s", pOpt->label, tstrerror(code)); + } + int32_t total = pBuf->offset; + int32_t offset = 0; + do { + int32_t n = send(pArg->fd, pBuf->buf + offset, total, 0); + if (n < 0) { + if (errno == EAGAIN || errno == EWOULDBLOCK) { + code = TAOS_SYSTEM_ERROR(errno); + break; + } + code = TAOS_SYSTEM_ERROR(errno); + break; + } + offset += n; + total -= n; + } while (total > 0); + + evtBufClear(&pArg->sendBuf); + tTrace("%s handle write finish on fd:%d", pOpt->label, pArg->fd); + pArg->sendFinishCb(pArg, code); + if (code != 0) { + tError("%s failed to send buf since %s", pOpt->label, tstrerror(code)); + } else { + tTrace("%s succ to send buf", pOpt->label); + } + return code; + } + } + + return code; +} +int32_t evtMgtHandle(SEvtMgt *pOpt, int32_t res, int32_t fd) { + int32_t code = 0; + SFdCbArg *pArg = taosHashGet(pOpt->pFdTable, &fd, sizeof(fd)); + if (pArg == NULL) { + return TAOS_SYSTEM_ERROR(EBADF); + } + return evtMgtHandleImpl(pOpt, pArg, res); +} + +static int32_t evtCaclNextTimeout(SEvtMgt *pOpt, struct timeval *tv) { + int32_t code = 0; + int64_t timeout = 0; + if (pOpt->caclTimeout) { + code = pOpt->caclTimeout(pOpt->arg, &timeout); + if (timeout == 0) { + tv->tv_sec = 0; + tv->tv_usec = 10 * 1000; + } else if (timeout < 0) { + tv->tv_sec = 30; + tv->tv_usec = 0; + } else { + tv->tv_sec = timeout / 1000; + tv->tv_usec = (timeout % 1000) * 1000; + } + } else { + tv->tv_sec = 30; + tv->tv_usec = 0; + } + + if (tv->tv_sec <= 0) { + if (tv->tv_usec <= 50 * 1000) tv->tv_usec = 50 * 1000; + } + tTrace("%s next timeout %ld, sec:%d, usec:%d", pOpt->label, timeout, (int32_t)tv->tv_sec, (int32_t)tv->tv_usec); + return code; +} +static int32_t evtMgtDispath(SEvtMgt *pOpt, struct timeval *tv) { + int32_t code = 0, res = 0, j, nfds = 0, active_Fds = 0; + + if (pOpt->timeoutFunc != NULL) { + code = pOpt->timeoutFunc(pOpt->arg); + if (code != 0) { + tError("%s failed to handle timeout since %s", pOpt->label, tstrerror(code)); + } else { + tTrace("%s succ to handle timeout", pOpt->label); + } + } + + if (pOpt->resizeOutSets) { + fd_set *readSetOut = NULL, *writeSetOut = NULL; + int32_t sz = pOpt->evtFdsSize; + + readSetOut = taosMemoryRealloc(pOpt->evtReadSetOut, sz); + if (readSetOut == NULL) { + return terrno; + } + pOpt->evtReadSetOut = readSetOut; + + writeSetOut = taosMemoryRealloc(pOpt->evtWriteSetOut, sz); + if (writeSetOut == NULL) { + return terrno; + } + pOpt->evtWriteSetOut = writeSetOut; + + pOpt->resizeOutSets = 0; + } + + memcpy(pOpt->evtReadSetOut, pOpt->evtReadSetIn, pOpt->evtFdsSize); + memcpy(pOpt->evtWriteSetOut, pOpt->evtWriteSetIn, pOpt->evtFdsSize); + + nfds = pOpt->evtFds + 1; + // TODO lock or not + tTrace("%s start to select, timeout:%d s: %d ms", pOpt->label, (int32_t)tv->tv_sec, (int32_t)tv->tv_usec / 1000); + active_Fds = select(nfds, pOpt->evtReadSetOut, pOpt->evtWriteSetOut, NULL, tv); + if (active_Fds < 0) { + return TAOS_SYSTEM_ERROR(errno); + } else if (active_Fds == 0) { + tTrace("%s select timeout occurred", pOpt->label); + return code; + } else { + tTrace("%s select count %d", pOpt->label, active_Fds); + } + + for (j = 0; j < nfds && active_Fds > 0; j++) { + int32_t fd = pOpt->fd[j]; + res = 0; + if ((fd > 0) && FD_ISSET(fd, pOpt->evtReadSetOut)) { + res |= EVT_READ; + } + if ((fd > 0) && FD_ISSET(fd, pOpt->evtWriteSetOut)) { + res |= EVT_WRITE; + } + if (fd > 0) { + tTrace("%s start to handle fd %d", pOpt->label, fd); + } + + if (res == 0) { + continue; + } else { + active_Fds--; + code = evtMgtHandle(pOpt, res, fd); + if (code != 0) { + tError("%s failed to handle fd %d since %s", pOpt->label, fd, tstrerror(code)); + } else { + tTrace("%s succ to handle fd %d", pOpt->label, fd); + } + } + } + + return code; +} + +static int32_t evtMgtResize(SEvtMgt *pOpt, int32_t cap) { + int32_t code = 0; + + fd_set *readSetIn = NULL; + fd_set *writeSetIn = NULL; + + readSetIn = taosMemoryRealloc(pOpt->evtReadSetIn, cap); + if (readSetIn == NULL) { + return terrno; + } + pOpt->evtReadSetIn = readSetIn; + + writeSetIn = taosMemoryRealloc(pOpt->evtWriteSetIn, cap); + if (writeSetIn == NULL) { + return terrno; + } + + pOpt->evtWriteSetIn = writeSetIn; + pOpt->resizeOutSets = 1; + + memset((char *)pOpt->evtReadSetIn + pOpt->evtFdsSize, 0, cap - pOpt->evtFdsSize); + memset((char *)pOpt->evtWriteSetIn + pOpt->evtFdsSize, 0, cap - pOpt->evtFdsSize); + + pOpt->evtFdsSize = cap; + return code; +} + +static int32_t evtMgtAdd(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg) { + // add new fd to the set + int8_t rwRef = 0; + int32_t code = 0; + if (pOpt->evtFds < fd) { + int32_t fdSize = pOpt->evtFdsSize; + + if (fdSize < (int32_t)sizeof(fd_mask)) { + fdSize = (int32_t)sizeof(fd_mask); + } + while (fdSize < (int32_t)SELECT_ALLOC_SIZE(fd + 1)) { + fdSize *= 2; + } + if (fdSize != pOpt->evtFdsSize) { + if (evtMgtResize(pOpt, fdSize)) { + return -1; + } + } + pOpt->evtFds = fd; + } + + SFdCbArg *p = taosHashGet(pOpt->pFdTable, &fd, sizeof(fd)); + if (p != NULL) { + if (events & EVT_READ) { + FD_SET(fd, pOpt->evtReadSetIn); + rwRef++; + } + if (events & EVT_WRITE) { + FD_SET(fd, pOpt->evtWriteSetIn); + rwRef++; + } + p->rwRef += rwRef; + if (p->rwRef >= 2) { + p->rwRef = 2; + } + } else { + if (arg == NULL) { + return TSDB_CODE_INVALID_MSG; + } + if (pOpt->fdIdx >= 1024) { + return TSDB_CODE_INVALID_MSG; + // taosClose(fd); + } + if (fd != 0) { + if (events & EVT_READ) { + FD_SET(fd, pOpt->evtReadSetIn); + rwRef++; + } + + if (events & EVT_WRITE) { + FD_SET(fd, pOpt->evtWriteSetIn); + rwRef++; + } + arg->rwRef = rwRef; + if (arg->rwRef >= 2) { + arg->rwRef = 2; + } + pOpt->fd[pOpt->fdIdx++] = fd; + code = taosHashPut(pOpt->pFdTable, &fd, sizeof(fd), arg, sizeof(*arg)); + } + } + return 0; +} + +static int32_t evtMgtRemove(SEvtMgt *pOpt, int32_t fd, int32_t events, SFdCbArg *arg) { + int32_t line = 0; + int32_t code = 0; + ASSERT((events & EVT_SIGNAL) == 0); + int8_t rwRef = 0; + if (pOpt->evtFds < fd) { + return TAOS_SYSTEM_ERROR(EBADF); + } + + if (events & EVT_READ) { + FD_CLR(fd, pOpt->evtReadSetIn); + rwRef++; + } + + if (events & EVT_WRITE) { + FD_CLR(fd, pOpt->evtWriteSetIn); + rwRef++; + } + SFdCbArg *pArg = taosHashGet(pOpt->pFdTable, &fd, sizeof(fd)); + if (pArg == NULL) { + tError("%s %s failed to get fd %d since %s", pOpt->label, __func__, fd, tstrerror(TAOS_SYSTEM_ERROR(EBADF))); + return 0; + } + pArg->rwRef -= rwRef; + if (pArg->rwRef == 0) { + code = taosHashRemove(pOpt->pFdTable, &fd, sizeof(fd)); + for (int32_t i = 0; i < sizeof(pOpt->fd) / sizeof(pOpt->fd[0]); i++) { + if (pOpt->fd[i] == fd) { + pOpt->fd[i] = -1; + } + } + } + return code; +} +static void evtMgtDestroy(SEvtMgt *pOpt) { + if (pOpt == NULL) return; + + if (pOpt->evtReadSetIn) { + taosMemoryFree(pOpt->evtReadSetIn); + } + + if (pOpt->evtWriteSetIn) { + taosMemoryFree(pOpt->evtWriteSetIn); + } + + if (pOpt->evtReadSetOut) { + taosMemoryFree(pOpt->evtReadSetOut); + } + + if (pOpt->evtWriteSetOut) { + taosMemoryFree(pOpt->evtWriteSetOut); + } + tDebug("%s succ to destroy evt mgt", pOpt->label); + taosHashCleanup(pOpt->pFdTable); + taosMemoryFree(pOpt); +} + +static int32_t evtAsyncInit(SEvtMgt *pOpt, int32_t fd[2], SAsyncHandle **async, AsyncCb cb, int8_t evtType, + void *pThrd) { + int32_t code = 0; + SAsyncHandle *pAsync = taosMemoryCalloc(1, sizeof(SAsyncHandle)); + if (pAsync == NULL) { + return terrno; + } + pAsync->hostThrd = pThrd; + pAsync->data = pOpt; + + taosThreadMutexInit(&pAsync->mutex, NULL); + pAsync->sendFd = fd[1]; + pAsync->cb = cb; + + SFdCbArg arg = {.evtType = evtType, .arg = pAsync, .fd = fd[0]}; + arg.arg = pAsync; + code = evtMgtAdd(pOpt, fd[0], EVT_READ, &arg); + if (code != 0) { + taosMemoryFree(pAsync); + return code; + } + tTrace("%s succ to init async on fd:%d", pOpt->label, fd[0]); + QUEUE_INIT(&pAsync->q); + *async = pAsync; + return code; +} + +static int32_t evtAsyncSend(SAsyncHandle *async, queue *q) { + int32_t code = 0; + taosThreadMutexLock(&async->mutex); + QUEUE_PUSH(&async->q, q); + taosThreadMutexUnlock(&async->mutex); + int32_t nBytes = write(async->sendFd, "1", 1); + if (nBytes != 1) { + return TAOS_SYSTEM_ERROR(errno); + } + + return code; +} + +void *transAcceptThread(void *arg) { + int32_t code = 0; + + setThreadName("trans-accept-work"); + SServerObj2 *srv = arg; + struct sockaddr_in client_addr; + socklen_t client_addr_len = sizeof(client_addr); + int32_t workerIdx = 0; + + while (1) { + int32_t client_fd = accept(srv->serverFd, (struct sockaddr *)&client_addr, &client_addr_len); + if (client_fd < 0) { + tError("failed to accept since %s", tstrerror(TAOS_SYSTEM_ERROR(errno))); + continue; + } + workerIdx = (workerIdx + 1) % srv->numOfThreads; + SWorkThrd2 *pThrd = srv->pThreadObj[workerIdx]; + + SFdArg *arg = taosMemoryCalloc(1, sizeof(SFdArg)); + if (arg == NULL) { + tError("failed to create fd arg since %s", tstrerror(terrno)); + continue; + } + arg->acceptFd = client_fd; + QUEUE_INIT(&arg->q); + + code = evtAsyncSend(pThrd->notifyNewConnHandle, &arg->q); + if (code != 0) { + tError("failed to send async msg since %s", tstrerror(code)); + } + } + return NULL; +} +int32_t evtSvrGetConnRefOfThrd(SWorkThrd2 *thrd) { return thrd ? thrd->connRefMgt : -1; } + +void uvDestroyResp(void *e) { + SSvrRespMsg *pMsg = QUEUE_DATA(e, SSvrRespMsg, q); + destroySmsg(pMsg); +} +static int32_t connGetSockInfo(SSvrConn *pConn) { + int32_t code = 0; + + struct sockaddr_in addr; + socklen_t addr_len = sizeof(addr); + char ip_str[INET_ADDRSTRLEN]; + int port; + + if (getpeername(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + + inet_ntop(AF_INET, &addr.sin_addr, ip_str, sizeof(ip_str)); + port = ntohs(addr.sin_port); + snprintf(pConn->dst, sizeof(pConn->dst), "%s:%d", ip_str, port); + + if (getsockname(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + return code; + } + inet_ntop(AF_INET, &addr.sin_addr, ip_str, sizeof(ip_str)); + port = ntohs(addr.sin_port); + + snprintf(pConn->src, sizeof(pConn->src), "%s:%d", ip_str, port); + + return code; +} +static SSvrConn *createConn(void *tThrd, int32_t fd) { + int32_t code = 0; + int32_t lino = 0; + SWorkThrd2 *pThrd = tThrd; + SSvrConn *pConn = taosMemoryCalloc(1, sizeof(SSvrConn)); + if (pConn == NULL) { + return NULL; + } + pConn->fd = fd; + pConn->pInst = pThrd->pInst; + QUEUE_INIT(&pConn->queue); + + code = connGetSockInfo(pConn); + TAOS_CHECK_GOTO(code, &lino, _end); + + if ((code = transInitBuffer(&pConn->readBuf)) != 0) { + TAOS_CHECK_GOTO(code, &lino, _end); + } + + pConn->broken = false; + pConn->status = ConnNormal; + + SExHandle *exh = taosMemoryMalloc(sizeof(SExHandle)); + if (exh == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _end); + } + + exh->handle = pConn; + exh->pThrd = pThrd; + exh->refId = transAddExHandle(evtSvrGetConnRefOfThrd(pThrd), exh); + if (exh->refId < 0) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, &lino, _end); + } + + SExHandle *pSelf = transAcquireExHandle(evtSvrGetConnRefOfThrd(pThrd), exh->refId); + if (pSelf != exh) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _end); + } + + STrans *pInst = pThrd->pInst; + pConn->refId = exh->refId; + + QUEUE_INIT(&exh->q); + tTrace("%s handle %p, conn %p created, refId:%" PRId64, transLabel(pInst), exh, pConn, pConn->refId); + + pConn->pQTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pConn->pQTable == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &lino, _end); + } + QUEUE_PUSH(&pThrd->conn, &pConn->queue); + transQueueInit(&pConn->resps, NULL); + + pConn->hostThrd = pThrd; + return pConn; +_end: + if (code != 0) { + tError("failed to create conn since %s", tstrerror(code)); + // destroryConn(pConn); + } + return NULL; +} +static void destroryConn(SSvrConn *pConn) { + if (pConn == NULL) { + return; + } + // transDestroyBuffer(&pConn->readBuf); + // transQueueCleanup(&pConn->resps); + taosHashCleanup(pConn->pQTable); + QUEUE_REMOVE(&pConn->queue); + taosMemoryFree(pConn); +} +static int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp); + +bool connMayGetUserInfo(SSvrConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { + if (pConn->userInited) { + return false; + } + + STrans *pInst = pConn->pInst; + STransMsgHead *pHead = *ppHead; + int32_t len = *msgLen; + if (pHead->withUserInfo) { + STransMsgHead *tHead = taosMemoryCalloc(1, len - sizeof(pInst->user)); + if (tHead == NULL) { + tError("conn %p failed to get user info since %s", pConn, tstrerror(terrno)); + return false; + } + memcpy((char *)tHead, (char *)pHead, TRANS_MSG_OVERHEAD); + memcpy((char *)tHead + TRANS_MSG_OVERHEAD, (char *)pHead + TRANS_MSG_OVERHEAD + sizeof(pInst->user), + len - sizeof(STransMsgHead) - sizeof(pInst->user)); + tHead->msgLen = htonl(htonl(pHead->msgLen) - sizeof(pInst->user)); + + memcpy(pConn->user, (char *)pHead + TRANS_MSG_OVERHEAD, sizeof(pConn->user)); + pConn->userInited = 1; + + taosMemoryFree(pHead); + *ppHead = tHead; + *msgLen = len - sizeof(pInst->user); + return true; + } + return false; +} + +static int32_t evtSvrHandleReleaseReq(SSvrConn *pConn, STransMsgHead *pHead) { + int32_t code = 0; + STrans *pInst = pConn->pInst; + if (pHead->msgType == TDMT_SCH_TASK_RELEASE) { + int64_t qId = taosHton64(pHead->qid); + if (qId <= 0) { + tError("conn %p recv release, but invalid sid:%" PRId64 "", pConn, qId); + code = TSDB_CODE_RPC_NO_STATE; + } else { + void *p = taosHashGet(pConn->pQTable, &qId, sizeof(qId)); + if (p == NULL) { + code = TSDB_CODE_RPC_NO_STATE; + tTrace("conn %p recv release, and releady release by server sid:%" PRId64 "", pConn, qId); + } else { + SSvrRegArg *arg = p; + (pInst->cfp)(pInst->parent, &(arg->msg), NULL); + tTrace("conn %p recv release, notify server app, sid:%" PRId64 "", pConn, qId); + + code = taosHashRemove(pConn->pQTable, &qId, sizeof(qId)); + if (code != 0) { + tDebug("conn %p failed to remove sid:%" PRId64 "", pConn, qId); + } + tTrace("conn %p clear state,sid:%" PRId64 "", pConn, qId); + } + } + + STransMsg tmsg = {.code = code, + .msgType = pHead->msgType + 1, + .info.qId = qId, + .info.traceId = pHead->traceId, + .info.seq = taosHton64(pHead->seqNum)}; + + SSvrRespMsg *srvMsg = taosMemoryCalloc(1, sizeof(SSvrRespMsg)); + if (srvMsg == NULL) { + tError("conn %p recv release, failed to send release-resp since %s", pConn, tstrerror(terrno)); + taosMemoryFree(pHead); + return terrno; + } + srvMsg->msg = tmsg; + srvMsg->type = Normal; + srvMsg->pConn = pConn; + + evtSvrHandleSendResp(pConn->hostThrd, srvMsg); + taosMemoryFree(pHead); + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } + + return 0; +} + +static void evtSvrPerfLog_receive(SSvrConn *pConn, STransMsgHead *pHead, STransMsg *pTransMsg) { + if (!(rpcDebugFlag & DEBUG_DEBUG)) { + return; + } + + STrans *pInst = pConn->pInst; + STraceId *trace = &pHead->traceId; + + int64_t cost = taosGetTimestampUs() - taosNtoh64(pHead->timestamp); + static int64_t EXCEPTION_LIMIT_US = 100 * 1000; + + if (pConn->status == ConnNormal && pHead->noResp == 0) { + if (cost >= EXCEPTION_LIMIT_US) { + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, recv exception, seqNum:%" PRId64 + ", sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pTransMsg->msgType), pConn->dst, pConn->src, pTransMsg->contLen, + (int)cost, pTransMsg->info.seq, pTransMsg->info.qId); + } else { + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, cost:%dus, seqNum:%" PRId64 ", sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pTransMsg->msgType), pConn->dst, pConn->src, pTransMsg->contLen, + (int)cost, pTransMsg->info.seq, pTransMsg->info.qId); + } + } else { + if (cost >= EXCEPTION_LIMIT_US) { + tGDebug( + "%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, recv exception, " + "seqNum:%" PRId64 ", sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pTransMsg->msgType), pConn->dst, pConn->src, pTransMsg->contLen, + pHead->noResp, pTransMsg->code, (int)(cost), pTransMsg->info.seq, pTransMsg->info.qId); + } else { + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, noResp:%d, code:%d, cost:%dus, seqNum:%" PRId64 + ", " + "sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pTransMsg->msgType), pConn->dst, pConn->src, pTransMsg->contLen, + pHead->noResp, pTransMsg->code, (int)(cost), pTransMsg->info.seq, pTransMsg->info.qId); + } + } + tGTrace("%s handle %p conn:%p translated to app, refId:%" PRIu64, transLabel(pInst), pTransMsg->info.handle, pConn, + pConn->refId); +} +static int32_t evtSvrHandleReq(SSvrConn *pConn, char *req, int32_t len) { + SWorkThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + int32_t code = 0; + int32_t msgLen = len; + STransMsgHead *pHead = (STransMsgHead *)req; + if (connMayGetUserInfo(pConn, &pHead, &msgLen) == true) { + tDebug("%s conn %p get user info", transLabel(pInst), pConn); + } else { + if (pConn->userInited == 0) { + taosMemoryFree(pHead); + tDebug("%s conn %p failed get user info since %s", transLabel(pInst), pConn, tstrerror(terrno)); + return TSDB_CODE_INVALID_MSG; + } + tDebug("%s conn %p no need get user info", transLabel(pInst), pConn); + } + pHead->code = htonl(pHead->code); + pHead->msgLen = htonl(pHead->msgLen); + + pConn->inType = pHead->msgType; + + if (evtSvrHandleReleaseReq(pConn, pHead) != 0) { + return 0; + } + STransMsg transMsg = {0}; + transMsg.contLen = transContLenFromMsg(pHead->msgLen); + transMsg.pCont = pHead->content; + transMsg.msgType = pHead->msgType; + transMsg.code = pHead->code; + if (pHead->seqNum == 0) { + STraceId *trace = &pHead->traceId; + tGError("%s conn %p received invalid seqNum, msgType:%s", transLabel(pInst), pConn, TMSG_INFO(pHead->msgType)); + return TSDB_CODE_INVALID_MSG; + } + + transMsg.info.handle = (void *)transAcquireExHandle(evtSvrGetConnRefOfThrd(pThrd), pConn->refId); + transMsg.info.refIdMgt = pThrd->connRefMgt; + + transMsg.info.refId = pHead->noResp == 1 ? -1 : pConn->refId; + transMsg.info.traceId = pHead->traceId; + transMsg.info.cliVer = htonl(pHead->compatibilityVer); + transMsg.info.forbiddenIp = 0; + transMsg.info.noResp = pHead->noResp == 1 ? 1 : 0; + transMsg.info.seq = taosHton64(pHead->seqNum); + transMsg.info.qId = taosHton64(pHead->qid); + transMsg.info.msgType = pHead->msgType; + + evtSvrPerfLog_receive(pConn, pHead, &transMsg); + + SRpcConnInfo *pConnInfo = &(transMsg.info.conn); + pConnInfo->clientIp = pConn->clientIp; + pConnInfo->clientPort = pConn->port; + tstrncpy(pConnInfo->user, pConn->user, sizeof(pConnInfo->user)); + + transReleaseExHandle(evtSvrGetConnRefOfThrd(pThrd), pConn->refId); + + (*pInst->cfp)(pInst->parent, &transMsg, NULL); + + return code; +} +static int32_t evtSvrReadCb(void *arg, SEvtBuf *buf, int32_t bytes) { + int32_t code = 0; + int32_t lino = 0; + SFdCbArg *pArg = arg; + SSvrConn *pConn = pArg->data; + STrans *pInst = pConn->pInst; + + if (bytes <= 0) { + tDebug("%s conn %p closed", transLabel(pInst), pConn); + return TSDB_CODE_RPC_NETWORK_ERROR; + } + + SConnBuffer *p = &pConn->readBuf; + if (p->cap - p->len < bytes) { + int32_t newCap = p->cap + bytes; + char *newBuf = taosMemoryRealloc(p->buf, newCap); + if (newBuf == NULL) { + TAOS_CHECK_GOTO(terrno, &lino, _end); + } + p->buf = newBuf; + p->cap = newCap; + } + + memcpy(p->buf + p->len, buf->buf, bytes); + p->len += bytes; + + while (p->len >= sizeof(STransMsgHead)) { + STransMsgHead head; + memcpy(&head, p->buf, sizeof(head)); + int32_t msgLen = (int32_t)htonl(head.msgLen); + if (p->len >= msgLen) { + char *pMsg = taosMemoryCalloc(1, msgLen); + if (pMsg == NULL) { + TAOS_CHECK_GOTO(terrno, &lino, _end); + } + memcpy(pMsg, p->buf, msgLen); + memmove(p->buf, p->buf + msgLen, p->len - msgLen); + p->len -= msgLen; + + code = evtSvrHandleReq(pConn, pMsg, msgLen); + TAOS_CHECK_GOTO(terrno, &lino, _end); + } else { + break; + } + } + + return code; +_end: + if (code != 0) { + tError("%s failed to handle read at line %d since %s", __func__, lino, tstrerror(code)); + } + return code; +} +static int32_t evtSvrPreSendImpl(SSvrConn *pConn, SEvtBuf *pBuf) { + int32_t code = 0; + int32_t batchLimit = 32; + int32_t j = 0; + while (!transQueueEmpty(&pConn->resps)) { + queue *el = transQueuePop(&pConn->resps); + QUEUE_REMOVE(el); + + SSvrRespMsg *pResp = QUEUE_DATA(el, SSvrRespMsg, q); + STransMsg *pMsg = &pResp->msg; + if (pMsg->pCont == 0) { + pMsg->pCont = (void *)rpcMallocCont(0); + if (pMsg->pCont == NULL) { + return terrno; + } + pMsg->contLen = 0; + } + STransMsgHead *pHead = transHeadFromCont(pMsg->pCont); + pHead->traceId = pMsg->info.traceId; + pHead->hasEpSet = pMsg->info.hasEpSet; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); + pHead->compatibilityVer = htonl(((STrans *)pConn->pInst)->compatibilityVer); + pHead->version = TRANS_VER; + pHead->seqNum = taosHton64(pMsg->info.seq); + pHead->qid = taosHton64(pMsg->info.qId); + pHead->withUserInfo = pConn->userInited == 0 ? 1 : 0; + + pHead->msgType = (0 == pMsg->msgType ? pConn->inType + 1 : pMsg->msgType); + pHead->code = htonl(pMsg->code); + pHead->msgLen = htonl(pMsg->contLen + sizeof(STransMsgHead)); + + char *msg = (char *)pHead; + int32_t len = transMsgLenFromCont(pMsg->contLen); + + STrans *pInst = pConn->pInst; + STraceId *trace = &pMsg->info.traceId; + tGDebug("%s conn %p %s is sent to %s, local info:%s, len:%d, seqNum:%" PRId64 ", sid:%" PRId64 "", + transLabel(pInst), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, len, pMsg->info.seq, + pMsg->info.qId); + evtBufPush(pBuf, (char *)pHead, len); + pResp->sent = 1; + if (j++ > batchLimit) { + break; + } + destroySmsg(pResp); + } + return code; +} +static int32_t evtSvrPreSend(void *arg, SEvtBuf *buf, int32_t status) { + int32_t code = 0; + SFdCbArg *pArg = arg; + SSvrConn *pConn = pArg->data; + + code = evtSvrPreSendImpl(pConn, buf); + return code; +} + +static int32_t evtSvrSendFinishCb(void *arg, int32_t status) { + int32_t code = 0; + SFdCbArg *pArg = arg; + SSvrConn *pConn = pArg->data; + + SWorkThrd2 *pThrd = pConn->hostThrd; + + if (transQueueEmpty(&pConn->resps)) { + // stop write evt + tTrace("%s conn %p stop write evt on fd:%d", transLabel(pThrd->pInst), pConn, pConn->fd); + code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); + } + return code; +} +void evtNewConnNotifyCb(void *async, int32_t status) { + int32_t code = 0; + + SAsyncHandle *handle = async; + SEvtMgt *pEvtMgt = handle->data; + SWorkThrd2 *pThrd = handle->hostThrd; + STrans *pInst = pThrd->pInst; + + queue wq; + QUEUE_INIT(&wq); + + taosThreadMutexLock(&handle->mutex); + QUEUE_MOVE(&handle->q, &wq); + taosThreadMutexUnlock(&handle->mutex); + + if (QUEUE_IS_EMPTY(&wq)) { + return; + } + while (!QUEUE_IS_EMPTY(&wq)) { + queue *el = QUEUE_HEAD(&wq); + QUEUE_REMOVE(el); + + SFdArg *pArg = QUEUE_DATA(el, SFdArg, q); + + SSvrConn *pConn = createConn(pEvtMgt->hostThrd, pArg->acceptFd); + if (pConn == NULL) { + tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); + taosMemoryFree(pArg); + continue; + } else { + tDebug("%s succ to create svr conn %p, src:%s, dst:%s", pInst->label, pConn, pConn->dst, pConn->src); + } + + SFdCbArg arg = {.evtType = EVT_CONN_T, + .arg = pArg, + .fd = pArg->acceptFd, + .readCb = evtSvrReadCb, + .sendCb = evtSvrPreSend, + .sendFinishCb = evtSvrSendFinishCb, + .data = pConn}; + code = evtMgtAdd(pEvtMgt, pArg->acceptFd, EVT_READ, &arg); + + if (code != 0) { + tError("%s failed to add fd to evt since %s", pInst->label, tstrerror(code)); + } + taosMemoryFree(pArg); + } + + return; +} + +static int32_t evtSvrMayHandleReleaseResp(SSvrRespMsg *pMsg) { + int32_t code = 0; + SSvrConn *pConn = pMsg->pConn; + int64_t qid = pMsg->msg.info.qId; + if (pMsg->msg.msgType == TDMT_SCH_TASK_RELEASE && qid > 0) { + SSvrRegArg *p = taosHashGet(pConn->pQTable, &qid, sizeof(qid)); + if (p == NULL) { + tError("%s conn %p already release sid:%" PRId64 "", transLabel(pConn->pInst), pConn, qid); + return TSDB_CODE_RPC_NO_STATE; + } else { + transFreeMsg(p->msg.pCont); + code = taosHashRemove(pConn->pQTable, &qid, sizeof(qid)); + if (code != 0) { + tError("%s conn %p failed to release sid:%" PRId64 " since %s", transLabel(pConn->pInst), pConn, qid, + tstrerror(code)); + } + } + } + return 0; +} +static int32_t evtSvrHandleSendResp(SWorkThrd2 *pThrd, SSvrRespMsg *pResp) { + int32_t code = 0; + SSvrConn *pConn = pResp->pConn; + if (evtSvrMayHandleReleaseResp(pResp) == TSDB_CODE_RPC_NO_STATE) { + destroySmsg(pResp); + return 0; + } + transQueuePush(&pConn->resps, &pResp->q); + return evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); +} + +static void evtSvrHandleResp(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + int32_t code = evtSvrHandleSendResp(pThrd, pResp); + if (code != 0) { + tError("failed to send resp since %s", tstrerror(code)); + } +} +static void evtSvrHandleUpdate(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + tDebug("conn %p recv update, do nothing", pResp->pConn); + taosMemoryFree(pResp); +} + +static void evtSvrHandleRegiter(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + int32_t code = 0; + SSvrConn *pConn = pResp->pConn; + + int64_t qid = pResp->msg.info.qId; + SSvrRegArg *p = taosHashGet(pConn->pQTable, &qid, sizeof(qid)); + if (p != NULL) { + transFreeMsg(p->msg.pCont); + } + + SSvrRegArg arg = {.notifyCount = 0, .msg = pResp->msg}; + code = taosHashPut(pConn->pQTable, &arg.msg.info.qId, sizeof(arg.msg.info.qId), &arg, sizeof(arg)); + if (code != 0) { + tError("failed to put qid to hash since %s", tstrerror(code)); + return; + } + tDebug("conn %p succ to register sid:%" PRId64 "", pConn, arg.msg.info.qId); + taosMemoryFree(pResp); +} +static void evtSvrHandleQuit(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + pThrd->quit = 1; + taosMemoryFree(pResp); +} +static void evtSvrHandleRelease(SSvrRespMsg *pResp, SWorkThrd2 *pThrd) { + int32_t code = 0; + taosMemoryFree(pResp); + return; +} + +static void (*transAsyncFuncs[])(SSvrRespMsg *, SWorkThrd2 *) = { + evtSvrHandleResp, evtSvrHandleQuit, evtSvrHandleRelease, evtSvrHandleRegiter, evtSvrHandleUpdate}; + +void evtSvrHandleAyncCb(void *async, int32_t status) { + int32_t code = 0; + + SAsyncHandle *handle = async; + SWorkThrd2 *pThrd = handle->hostThrd; + // SEvtMgt *pEvtMgt = handle->data; + + queue wq; + QUEUE_INIT(&wq); + taosThreadMutexLock(&handle->mutex); + QUEUE_MOVE(&handle->q, &wq); + taosThreadMutexUnlock(&handle->mutex); + + if (QUEUE_IS_EMPTY(&wq)) { + return; + } + while (!QUEUE_IS_EMPTY(&wq)) { + queue *el = QUEUE_HEAD(&wq); + QUEUE_REMOVE(el); + + SSvrRespMsg *pResp = QUEUE_DATA(el, SSvrRespMsg, q); + if (pResp == NULL) { + continue; + } + + if (pResp->type == Quit || pResp->type == Update) { + transAsyncFuncs[pResp->type](pResp, pThrd); + } else { + STransMsg transMsg = pResp->msg; + SExHandle *exh1 = transMsg.info.handle; + int64_t refId = transMsg.info.refId; + SExHandle *exh2 = transAcquireExHandle(evtSvrGetConnRefOfThrd(pThrd), refId); + if (exh1 != exh2) { + tError("failed to acquire handle since %s", tstrerror(TSDB_CODE_REF_INVALID_ID)); + continue; + } + + pResp->seqNum = transMsg.info.seq; + pResp->pConn = exh2->handle; + + transReleaseExHandle(evtSvrGetConnRefOfThrd(pThrd), refId); + transAsyncFuncs[pResp->type](pResp, pThrd); + } + } + + return; +} + +void *transWorkerThread(void *arg) { + int32_t code = 0; + int32_t line = 0; + + setThreadName("trans-svr-work"); + SWorkThrd2 *pThrd = (SWorkThrd2 *)arg; + STrans *pInst = pThrd->pInst; + + SEvtMgt *pOpt = NULL; + + code = evtMgtCreate(&pOpt, pInst->label); + if (code != 0) { + tError("%s failed to create select op since %s", pInst->label, tstrerror(code)); + TAOS_CHECK_GOTO(code, &line, _end); + } + + pThrd->pEvtMgt = pOpt; + pOpt->hostThrd = pThrd; + + code = evtAsyncInit(pOpt, pThrd->pipe_fd, &pThrd->notifyNewConnHandle, evtNewConnNotifyCb, EVT_NEW_CONN_T, + (void *)pThrd); + if (code != 0) { + tError("%s failed to create evt since %s", pInst->label, tstrerror(code)); + TAOS_CHECK_GOTO(code, &line, _end); + } + + code = evtAsyncInit(pOpt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtSvrHandleAyncCb, EVT_ASYNC_T, (void *)pThrd); + if (code != 0) { + tError("%s failed to create select op since %s", pInst->label, tstrerror(code)); + TAOS_CHECK_GOTO(code, &line, _end); + } + int32_t count = 0; + while (!pThrd->quit) { + struct timeval tv = {30, 0}; + tTrace("%s-------------------- dispatch count:%d -----------------------------", pInst->label, count++); + code = evtMgtDispath(pOpt, &tv); + if (code != 0) { + tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); + continue; + } + } +_end: + if (code != 0) { + tError("%s failed to do work %s", pInst->label, tstrerror(code)); + } + evtMgtDestroy(pOpt); + return NULL; +} +static int32_t addHandleToAcceptloop(void *arg) { + // impl later + int32_t code = 0; + SServerObj2 *srv = arg; + int32_t server_fd = socket(AF_INET, SOCK_STREAM, 0); + if (server_fd < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + srv->serverFd = server_fd; + + int32_t opt = 1; + setsockopt(server_fd, SOL_SOCKET, SO_REUSEADDR, &opt, sizeof(opt)); + + struct sockaddr_in server_addr; + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_addr.s_addr = inet_addr("0.0.0.0"); + server_addr.sin_port = htons(srv->port); + + if (bind(server_fd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + if (listen(server_fd, 128) < 0) { + return TAOS_SYSTEM_ERROR(errno); + } + code = taosThreadCreate(&srv->thread, NULL, transAcceptThread, srv); + return code; +} + +void *transInitServer2(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *arg) { + int32_t code = 0; + int32_t lino = 0; + STrans *pInst = arg; + + SServerObj2 *srv = taosMemoryCalloc(1, sizeof(SServerObj2)); + if (srv == NULL) { + code = terrno; + TAOS_CHECK_EXIT(code); + } + + srv->ip = ip; + srv->port = port; + srv->numOfThreads = numOfThreads; + srv->workerIdx = 0; + srv->numOfWorkerReady = 0; + srv->pThreadObj = (SWorkThrd2 **)taosMemoryCalloc(srv->numOfThreads, sizeof(SWorkThrd2 *)); + if (srv->pThreadObj == NULL) { + code = terrno; + TAOS_CHECK_EXIT(code); + } + for (int i = 0; i < srv->numOfThreads; i++) { + SWorkThrd2 *thrd = (SWorkThrd2 *)taosMemoryCalloc(1, sizeof(SWorkThrd2)); + thrd->pInst = arg; + thrd->connRefMgt = transOpenRefMgt(50000, transDestroyExHandle); + if (thrd->connRefMgt < 0) { + code = thrd->connRefMgt; + TAOS_CHECK_EXIT(code); + } + + TAOS_CHECK_EXIT(evtInitPipe(thrd->pipe_fd)); + TAOS_CHECK_EXIT(evtInitPipe(thrd->pipe_queue_fd)); + QUEUE_INIT(&thrd->conn); + + code = taosThreadCreate(&(thrd->thread), NULL, transWorkerThread, (void *)(thrd)); + TAOS_CHECK_EXIT(code); + thrd->inited = 1; + thrd->quit = false; + srv->pThreadObj[i] = thrd; + } + code = addHandleToAcceptloop(srv); + TAOS_CHECK_EXIT(code); + return NULL; +_exit: + if (code != 0) { + tError("%s failed to init server at line %d since %s", pInst->label, lino, tstrerror(code)); + } + return NULL; +} + +int32_t transReleaseSrvHandle2(void *handle, int32_t status) { + int32_t code = 0; + SRpcHandleInfo *info = handle; + SExHandle *exh = info->handle; + int64_t qId = info->qId; + int64_t refId = info->refId; + + ASYNC_CHECK_HANDLE(info->refIdMgt, refId, exh); + + SWorkThrd2 *pThrd = exh->pThrd; + ASYNC_ERR_JRET(pThrd); + + STransMsg tmsg = {.msgType = TDMT_SCH_TASK_RELEASE, + .code = status, + .info.handle = exh, + .info.ahandle = NULL, + .info.refId = refId, + .info.qId = qId, + .info.traceId = info->traceId}; + + SSvrRespMsg *m = taosMemoryCalloc(1, sizeof(SSvrRespMsg)); + if (m == NULL) { + code = terrno; + goto _return1; + } + + m->msg = tmsg; + m->type = Normal; + + tDebug("%s conn %p start to send %s, sid:%" PRId64 "", transLabel(pThrd->pInst), exh->handle, TMSG_INFO(tmsg.msgType), + qId); + if ((code = evtAsyncSend(pThrd->asyncHandle, &m->q)) != 0) { + destroySmsg(m); + transReleaseExHandle(info->refIdMgt, refId); + return code; + } + + transReleaseExHandle(info->refIdMgt, refId); + return 0; +_return1: + tDebug("handle %p failed to send to release handle", exh); + transReleaseExHandle(info->refIdMgt, refId); + return code; +_return2: + tDebug("handle %p failed to send to release handle", exh); + return code; +} +int32_t transRegisterMsg2(const STransMsg *msg) { + int32_t code = 0; + + SExHandle *exh = msg->info.handle; + int64_t refId = msg->info.refId; + ASYNC_CHECK_HANDLE(msg->info.refIdMgt, refId, exh); + + STransMsg tmsg = *msg; + tmsg.info.noResp = 1; + + tmsg.info.qId = msg->info.qId; + tmsg.info.seq = msg->info.seq; + tmsg.info.refId = refId; + tmsg.info.refIdMgt = msg->info.refIdMgt; + + SWorkThrd2 *pThrd = exh->pThrd; + ASYNC_ERR_JRET(pThrd); + + SSvrRespMsg *m = taosMemoryCalloc(1, sizeof(SSvrRespMsg)); + if (m == NULL) { + code = terrno; + goto _return1; + } + + m->msg = tmsg; + m->type = Register; + + STrans *pInst = pThrd->pInst; + tDebug("%s conn %p start to register brokenlink callback", transLabel(pInst), exh->handle); + if ((code = evtAsyncSend(pThrd->asyncHandle, &m->q)) != 0) { + destroySmsg(m); + transReleaseExHandle(msg->info.refIdMgt, refId); + return code; + } + + transReleaseExHandle(msg->info.refIdMgt, refId); + return 0; + +_return1: + tDebug("handle %p failed to register brokenlink", exh); + rpcFreeCont(msg->pCont); + transReleaseExHandle(msg->info.refIdMgt, refId); + return code; +_return2: + tDebug("handle %p failed to register brokenlink", exh); + rpcFreeCont(msg->pCont); + return code; +} +int32_t transSetIpWhiteList2(void *thandle, void *arg, FilteFunc *func) { return 0; } +int32_t transSendResponse2(STransMsg *msg) { + int32_t code = 0; + + if (msg->info.noResp) { + rpcFreeCont(msg->pCont); + tTrace("no need send resp"); + return 0; + } + + STraceId *trace = &msg->info.traceId; + tGDebug("start to send resp %p", msg); + + SExHandle *exh = msg->info.handle; + + if (exh == NULL) { + rpcFreeCont(msg->pCont); + return 0; + } + int64_t refId = msg->info.refId; + ASYNC_CHECK_HANDLE(msg->info.refIdMgt, refId, exh); + + STransMsg tmsg = *msg; + tmsg.info.refId = refId; + if (tmsg.info.qId == 0) { + tmsg.msgType = msg->info.msgType + 1; + } + + SWorkThrd2 *pThrd = exh->pThrd; + ASYNC_ERR_JRET(pThrd); + + SSvrRespMsg *m = taosMemoryCalloc(1, sizeof(SSvrRespMsg)); + if (m == NULL) { + code = terrno; + goto _return1; + } + m->msg = tmsg; + + m->type = Normal; + + // STraceId *trace = (STraceId *)&msg->info.traceId; + tGDebug("conn %p start to send resp (1/2)", exh->handle); + if ((code = evtAsyncSend(pThrd->asyncHandle, &m->q)) != 0) { + destroySmsg(m); + transReleaseExHandle(msg->info.refIdMgt, refId); + return code; + } + + transReleaseExHandle(msg->info.refIdMgt, refId); + return 0; + +_return1: + tGDebug("handle %p failed to send resp", exh); + rpcFreeCont(msg->pCont); + transReleaseExHandle(msg->info.refIdMgt, refId); + return code; +_return2: + tGDebug("handle %p failed to send resp", exh); + rpcFreeCont(msg->pCont); + return code; +} +// int32_t transRegisterMsg(const STransMsg *msg) { return 0; } +// int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } + +void transCloseServer2(void *arg) { + // impl later + return; +} +// impl client with poll + +typedef struct SConnList { + queue conns; + int32_t size; + int32_t totalSize; +} SConnList; + +typedef struct SCliConn { + int32_t ref; + // uv_connect_t connReq; + // uv_stream_t *stream; + + // uv_timer_t *timer; // read timer, forbidden + + void *hostThrd; + + SConnBuffer readBuf; + STransQueue reqsToSend; + STransQueue reqsSentOut; + + queue q; + SConnList *list; + + STransCtx ctx; + bool broken; // link broken or not + ConnStatus status; // + + // SCliBatch *pBatch; + + SDelayTask *task; + + HeapNode node; // for heap + int8_t inHeap; + int32_t reqRefCnt; + uint32_t clientIp; + uint32_t serverIp; + + char *dstAddr; + char src[32]; + char dst[32]; + + char *ipStr; + int32_t port; + + int64_t seq; + + int8_t registered; + int8_t connnected; + SHashObj *pQTable; + int8_t userInited; + void *pInitUserReq; + + void *heap; // point to req conn heap + int32_t heapMissHit; + int64_t lastAddHeapTime; + int8_t forceDelFromHeap; + + // uv_buf_t *buf; + int32_t bufSize; + int32_t readerStart; + + queue wq; // uv_write_t queue + + queue batchSendq; + int8_t inThreadSendq; + int32_t fd; +} SCliConn; + +typedef struct { + SCliConn *conn; + void *arg; +} SReqState; + +typedef struct { + int64_t seq; + int32_t msgType; +} SFiterArg; +typedef struct SCliReq { + SReqCtx *ctx; + queue q; + queue sendQ; + STransMsgType type; + uint64_t st; + int64_t seq; + int32_t sent; //(0: no send, 1: alread sent) + int8_t inSendQ; + STransMsg msg; + int8_t inRetry; + +} SCliReq; + +#define EPSET_IS_VALID(epSet) ((epSet) != NULL && (epSet)->numOfEps >= 0 && (epSet)->inUse >= 0) +#define EPSET_GET_SIZE(epSet) (epSet)->numOfEps +#define EPSET_GET_INUSE_IP(epSet) ((epSet)->eps[(epSet)->inUse].fqdn) +#define EPSET_GET_INUSE_PORT(epSet) ((epSet)->eps[(epSet)->inUse].port) +#define EPSET_FORWARD_INUSE(epSet) \ + do { \ + if ((epSet)->numOfEps != 0) { \ + ++((epSet)->inUse); \ + (epSet)->inUse = ((epSet)->inUse) % ((epSet)->numOfEps); \ + } \ + } while (0) +typedef struct SCliThrd2 { + TdThread thread; // tid + int64_t pid; // pid + + // uv_loop_t *loop; + // SAsyncPool *asyncPool; + void *pool; // conn pool + // timer handles + SArray *timerList; + // msg queue + queue msg; + TdThreadMutex msgMtx; + // SDelayQueue *delayQueue; + // SDelayQueue *timeoutQueue; + // SDelayQueue *waitConnQueue; + uint64_t nextTimeout; // next timeout + STrans *pInst; // + + void (*destroyAhandleFp)(void *ahandle); + SHashObj *fqdn2ipCache; + SCvtAddr *pCvtAddr; + + SHashObj *failFastCache; + SHashObj *batchCache; + SHashObj *connHeapCache; + + SCliReq *stopMsg; + bool quit; + + int32_t (*initCb)(void *arg, SCliReq *pReq, STransMsg *pResp); + int32_t (*notifyCb)(void *arg, SCliReq *pReq, STransMsg *pResp); + int32_t (*notifyExceptCb)(void *arg, SCliReq *pReq, STransMsg *pResp); + + SHashObj *pIdConnTable; // + + SArray *pQIdBuf; // tmp buf to avoid alloc buf; + queue batchSendSet; + int8_t thrdInited; + int32_t pipe_queue_fd[2]; + SEvtMgt *pEvtMgt; + SAsyncHandle *asyncHandle; + int32_t shareConnLimit; +} SCliThrd2; + +typedef struct { + SDelayQueue *queue; + void *pEvtMgt; +} STimeoutArg; + +int32_t createTimeoutArg(void *pEvtMgt, STimeoutArg **pArg) { + int32_t code = 0; + int32_t line = 0; + + STimeoutArg *arg = taosMemoryCalloc(1, sizeof(STimeoutArg)); + if (arg == NULL) { + code = terrno; + TAOS_CHECK_GOTO(code, &line, _end); + } + SDelayQueue *queue = NULL; + transDQCreate(arg, &queue); + + arg->queue = queue; + arg->pEvtMgt = pEvtMgt; + *pArg = arg; + return code; +_end: + if (code != 0) { + tError("%s failed to create timeout arg at line %d since %s", __func__, line, tstrerror(code)); + } + return code; +} + +void destroyTimeoutArg(STimeoutArg *arg) { + if (arg == NULL) { + return; + } + + SDelayQueue *queue = arg->queue; + transDQDestroy(queue, NULL); + taosMemoryFree(arg); + return; +} + +typedef struct SCliObj2 { + char label[TSDB_LABEL_LEN]; + int32_t index; + int numOfThreads; + SCliThrd2 **pThreadObj; +} SCliObj2; +#define REQS_ON_CONN(conn) (conn ? (transQueueSize(&conn->reqsToSend) + transQueueSize(&conn->reqsSentOut)) : 0) +typedef struct { + void *p; + HeapNode node; +} SHeapNode; +typedef struct { + // void* p; + Heap *heap; + int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b); + int64_t lastUpdateTs; + int64_t lastConnFailTs; +} SHeap; + +static FORCE_INLINE void destroyReq(void *arg); +static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req); + +static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b); +static int32_t transHeapInit(SHeap *heap, int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b)); +static void transHeapDestroy(SHeap *heap); + +static int32_t transHeapGet(SHeap *heap, SCliConn **p); +static int32_t transHeapInsert(SHeap *heap, SCliConn *p); +static int32_t transHeapDelete(SHeap *heap, SCliConn *p); +static int32_t transHeapBalance(SHeap *heap, SCliConn *p); +static int32_t transHeapUpdateFailTs(SHeap *heap, SCliConn *p); +static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p); + +static int32_t getOrCreateHeapCache(SHashObj *pConnHeapCache, char *key, SHeap **pHeap); +static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SCliConn **pNewConn); +static int32_t delConnFromHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn); + +static SCliConn *getConnFromHeapCache(SHashObj *pConnHeap, char *key); +static int32_t addConnToHeapCache(SHashObj *pConnHeap, SCliConn *pConn); + +static void evtCliHandlReq(SCliReq *pReq, SCliThrd2 *pThrd); +static void evtCliHandleQuit(SCliReq *pReq, SCliThrd2 *pThrd); +static void evtCliHandleRelease(SCliReq *pReq, SCliThrd2 *pThrd); +static void evtCliHandleRegiter(SCliReq *pReq, SCliThrd2 *pThrd); +static void evtCliHandleUpdate(SCliReq *pReq, SCliThrd2 *pThrd); +static void (*transCliAsyncFuncs[])(SCliReq *, SCliThrd2 *) = {evtCliHandlReq, evtCliHandleQuit, evtCliHandleRelease, + evtCliHandleRelease, evtCliHandleUpdate}; + +static FORCE_INLINE void removeReqFromSendQ(SCliReq *pReq); + +static void transRefCliHandle(void *handle); +static int32_t transUnrefCliHandle(void *handle); +// static int32_t transGetRefCount(void *handle); + +static void evtCliHandlReq(SCliReq *pReq, SCliThrd2 *pThrd) { + int32_t code = evtHandleCliReq(pThrd, pReq); + if (code != 0) { + tDebug("failed to handle req"); + } +} +static void evtCliHandleQuit(SCliReq *pReq, SCliThrd2 *pThrd) { + tDebug("recv quit, set quit flag"); + pThrd->quit = 1; + destroyReq(pReq); + return; +} +static void evtCliHandleRelease(SCliReq *pReq, SCliThrd2 *pThrd) { + tDebug("recv release, do nothing"); + destroyReq(pReq); + return; +} +static void evtCliHandleRegiter(SCliReq *pReq, SCliThrd2 *pThrd) { + tDebug("recv register, do nothing"); + destroyReq(pReq); + return; +} +static void evtCliHandleUpdate(SCliReq *pReq, SCliThrd2 *pThrd) { + tDebug("recv update, do nothing"); + destroyReq(pReq); + return; +} +static int32_t createSocket(uint32_t ip, int32_t port, int32_t *fd) { + int32_t code = 0; + int32_t line = 0; + int32_t sockfd = socket(AF_INET, SOCK_STREAM, 0); + if (sockfd < 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + } + struct sockaddr_in server_addr; + memset(&server_addr, 0, sizeof(server_addr)); + server_addr.sin_family = AF_INET; + server_addr.sin_port = htons(port); + server_addr.sin_addr.s_addr = ip; + + // if (inet_pton(AF_INET, ip, &server_addr.sin_addr) <= 0) { + // TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + // } + + if (connect(sockfd, (struct sockaddr *)&server_addr, sizeof(server_addr)) < 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + } + *fd = sockfd; + return code; +_end: + if (code != 0) { + tError("%s failed to connect to %d:%d at line %d since %s", __func__, ip, port, line, tstrerror(code)); + } + return code; +} +static int32_t cliConnGetSockInfo(SCliConn *pConn) { + int32_t code = 0; + int32_t line = 0; + struct sockaddr_in addr; + + socklen_t addr_len = sizeof(addr); + char ip_str[INET_ADDRSTRLEN]; + int port; + + // get peer address + if (getpeername(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + } + inet_ntop(AF_INET, &addr.sin_addr, ip_str, sizeof(ip_str)); + port = ntohs(addr.sin_port); + snprintf(pConn->dst, sizeof(pConn->dst), "%s:%d", ip_str, port); + + // get local address + if (getsockname(pConn->fd, (struct sockaddr *)&addr, &addr_len) < 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), &line, _end); + } + inet_ntop(AF_INET, &addr.sin_addr, ip_str, sizeof(ip_str)); + port = ntohs(addr.sin_port); + snprintf(pConn->src, sizeof(pConn->src), "%s:%d", ip_str, port); + return code; +_end: + if (code != 0) { + tError("%s failed to get sock info at line %d since %s", __func__, line, tstrerror(code)); + } + return code; +} + +static int32_t getOrCreateConnList(SCliThrd2 *pThrd, const char *key, SConnList **ppList) { + int32_t code = 0; + void *pool = pThrd->pool; + size_t klen = strlen(key); + SConnList *plist = taosHashGet((SHashObj *)pool, key, klen); + if (plist == NULL) { + SConnList list = {0}; + QUEUE_INIT(&list.conns); + code = taosHashPut((SHashObj *)pool, key, klen, (void *)&list, sizeof(list)); + if (code != 0) { + return code; + } + + plist = taosHashGet(pool, key, klen); + if (plist == NULL) { + return TSDB_CODE_INVALID_PTR; + } + QUEUE_INIT(&plist->conns); + *ppList = plist; + tDebug("create conn list %p for key %s", plist, key); + } else { + *ppList = plist; + } + return 0; +} +static void evtCliCloseIdleConn(void *param) { + STaskArg *arg = param; + SCliConn *conn = arg->param1; + SCliThrd2 *thrd = arg->param2; + tDebug("%s conn %p idle, close it", thrd->pInst->label, conn); + conn->task = NULL; + taosMemoryFree(arg); + + // taosCloseSocketNoCheck1(conn->fd); + int32_t ref = transUnrefCliHandle(conn); + if (ref <= 0) { + return; + } +} +static void addConnToPool(void *pool, SCliConn *conn) { + if (conn->status == ConnInPool) { + return; + } + + SCliThrd2 *thrd = conn->hostThrd; + if (thrd->quit == true) { + return; + } + + // cliResetConnTimer(conn); + if (conn->list == NULL && conn->dstAddr != NULL) { + conn->list = taosHashGet((SHashObj *)pool, conn->dstAddr, strlen(conn->dstAddr)); + } + + conn->status = ConnInPool; + QUEUE_INIT(&conn->q); + QUEUE_PUSH(&conn->list->conns, &conn->q); + conn->list->size += 1; + tDebug("conn %p added to pool, pool size: %d, dst: %s", conn, conn->list->size, conn->dstAddr); + + conn->heapMissHit = 0; + + if (conn->list->size >= 5) { + STaskArg *arg = taosMemoryCalloc(1, sizeof(STaskArg)); + if (arg == NULL) return; + arg->param1 = conn; + arg->param2 = thrd; + + STrans *pInst = thrd->pInst; + conn->task = transDQSched(thrd->pEvtMgt->arg, evtCliCloseIdleConn, arg, (1000 * (pInst->idleTime))); + } +} + +static int32_t getConnFromPool(SCliThrd2 *pThrd, const char *key, SCliConn **ppConn) { + int32_t code = 0; + void *pool = pThrd->pool; + STrans *pInst = pThrd->pInst; + + SConnList *plist = NULL; + code = getOrCreateConnList(pThrd, key, &plist); + if (code != 0) { + return code; + } + + if (QUEUE_IS_EMPTY(&plist->conns)) { + if (plist->totalSize >= pInst->connLimitNum) { + return TSDB_CODE_RPC_MAX_SESSIONS; + } + return TSDB_CODE_RPC_NETWORK_BUSY; + } + + queue *h = QUEUE_HEAD(&plist->conns); + plist->size -= 1; + QUEUE_REMOVE(h); + + SCliConn *conn = QUEUE_DATA(h, SCliConn, q); + conn->status = ConnNormal; + QUEUE_INIT(&conn->q); + conn->list = plist; + + if (conn->task != NULL) { + SDelayTask *task = conn->task; + conn->task = NULL; + transDQCancel(((SCliThrd2 *)conn->hostThrd)->pEvtMgt->arg, task); + } + + tDebug("conn %p get from pool, pool size:%d, dst:%s", conn, conn->list->size, conn->dstAddr); + + *ppConn = conn; + return 0; +} +static int32_t evtCliGetIpFromFqdn(SHashObj *pTable, char *fqdn, uint32_t *ip) { + int32_t code = 0; + uint32_t addr = 0; + size_t len = strlen(fqdn); + uint32_t *v = taosHashGet(pTable, fqdn, len); + if (v == NULL) { + code = taosGetIpv4FromFqdn(fqdn, &addr); + if (code != 0) { + code = TSDB_CODE_RPC_FQDN_ERROR; + tError("failed to get ip from fqdn:%s since %s", fqdn, tstrerror(code)); + return code; + } + + if ((code = taosHashPut(pTable, fqdn, len, &addr, sizeof(addr)) != 0)) { + return code; + } + *ip = addr; + } else { + *ip = *v; + } + return 0; +} +static int32_t getOrCreateConn(SCliThrd2 *pThrd, char *ip, int32_t port, SCliConn **ppConn) { + int32_t code = 0; + int32_t line = 0; + STrans *pInst = pThrd->pInst; + SCliConn *pConn = NULL; + uint32_t ipAddr; + + char addr[TSDB_FQDN_LEN + 64] = {0}; + snprintf(addr, sizeof(addr), "%s:%d", ip, port); + pConn = getConnFromHeapCache(pThrd->connHeapCache, addr); + if (pConn != NULL) { + *ppConn = pConn; + return code; + } + + code = getConnFromPool(pThrd, addr, &pConn); + if (pConn != NULL) { + *ppConn = pConn; + addConnToHeapCache(pThrd->connHeapCache, pConn); + return 0; + } + if (code == TSDB_CODE_RPC_MAX_SESSIONS) { + return code; + } + + pConn = taosMemoryCalloc(1, sizeof(SCliConn)); + if (pConn == NULL) { + TAOS_CHECK_GOTO(terrno, &line, _end); + } + pConn->hostThrd = pThrd; + pConn->dstAddr = taosStrdup(addr); + pConn->ipStr = taosStrdup(ip); + pConn->port = port; + if (pConn->dstAddr == NULL || pConn->ipStr == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, &line, _end); + } + pConn->status = ConnNormal; + pConn->broken = false; + QUEUE_INIT(&pConn->q); + + TAOS_CHECK_GOTO(transInitBuffer(&pConn->readBuf), NULL, _end); + pConn->seq = 0; + + pConn->pQTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); + if (pConn->pQTable == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + QUEUE_INIT(&pConn->batchSendq); + pConn->bufSize = pInst->shareConnLimit; + + transQueueInit(&pConn->reqsToSend, NULL); + transQueueInit(&pConn->reqsSentOut, NULL); + + TAOS_CHECK_GOTO(evtCliGetIpFromFqdn(pThrd->fqdn2ipCache, ip, &ipAddr), &line, _end); + + TAOS_CHECK_GOTO(createSocket(ipAddr, port, &pConn->fd), &line, _end); + TAOS_CHECK_GOTO(cliConnGetSockInfo(pConn), &line, _end); + + pConn->connnected = 1; + addConnToHeapCache(pThrd->connHeapCache, pConn); + + pConn->list = taosHashGet(pThrd->pool, addr, strlen(addr)); + pConn->list->totalSize += 1; + + tDebug("%s succ to create cli conn %p src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + + *ppConn = pConn; + return code; + +_end: + if (code != 0) { + // TODO, delete conn mem + tError("%s failed to create conn at line %d since %s", __func__, line, tstrerror(code)); + taosMemoryFree(pConn); + } + return code; +} + +static FORCE_INLINE void destroyReqCtx(SReqCtx *ctx) { + if (ctx) { + taosMemoryFree(ctx->epSet); + taosMemoryFree(ctx->origEpSet); + taosMemoryFree(ctx); + } +} + +static FORCE_INLINE bool filterAllReq(void *e, void *arg) { return 1; } + +static int32_t cliNotifyCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp); +static void notifyAndDestroyReq(SCliConn *pConn, SCliReq *pReq, int32_t code) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + SReqCtx *pCtx = pReq ? pReq->ctx : NULL; + STransMsg resp = {0}; + resp.code = (pConn->connnected ? TSDB_CODE_RPC_BROKEN_LINK : TSDB_CODE_RPC_NETWORK_UNAVAIL); + if (code != 0) { + resp.code = code; + } + + resp.msgType = pReq ? pReq->msg.msgType + 1 : 0; + resp.info.cliVer = pInst->compatibilityVer; + resp.info.ahandle = pCtx ? pCtx->ahandle : 0; + resp.info.handle = pReq->msg.info.handle; + if (pReq) { + resp.info.traceId = pReq->msg.info.traceId; + } + + STraceId *trace = &resp.info.traceId; + tDebug("%s conn %p notify user and destroy msg %s since %s", pInst->label, pConn, TMSG_INFO(pReq->msg.msgType), + tstrerror(resp.code)); + + // handle noresp and inter manage msg + if (pCtx == NULL || pReq->msg.info.noResp) { + tDebug("%s conn %p destroy %s msg directly since %s", pInst->label, pConn, TMSG_INFO(pReq->msg.msgType), + tstrerror(resp.code)); + destroyReq(pReq); + return; + } + + pReq->seq = 0; + code = cliNotifyCb(pConn, pReq, &resp); + if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { + return; + } else { + // already notify user + destroyReq(pReq); + } +} + +static FORCE_INLINE void destroyReqInQueue(SCliConn *conn, queue *set, int32_t code) { + while (!QUEUE_IS_EMPTY(set)) { + queue *el = QUEUE_HEAD(set); + QUEUE_REMOVE(el); + + SCliReq *pReq = QUEUE_DATA(el, SCliReq, q); + removeReqFromSendQ(pReq); + notifyAndDestroyReq(conn, pReq, code); + } +} +static FORCE_INLINE int32_t destroyAllReqs(SCliConn *conn) { + int32_t code = 0; + SCliThrd2 *pThrd = conn->hostThrd; + STrans *pInst = pThrd->pInst; + queue set; + QUEUE_INIT(&set); + // TODO + // 1. from qId from thread table + // 2. not itera to all reqs + transQueueRemoveByFilter(&conn->reqsSentOut, filterAllReq, NULL, &set, -1); + transQueueRemoveByFilter(&conn->reqsToSend, filterAllReq, NULL, &set, -1); + + destroyReqInQueue(conn, &set, 0); + return 0; +} +static void evtCliDestroyAllQidFromThrd(SCliConn *conn) { + int32_t code = 0; + SCliThrd2 *pThrd = conn->hostThrd; + STrans *pInst = pThrd->pInst; + + void *pIter = taosHashIterate(conn->pQTable, NULL); + while (pIter != NULL) { + int64_t *qid = taosHashGetKey(pIter, NULL); + + code = taosHashRemove(pThrd->pIdConnTable, qid, sizeof(*qid)); + if (code != 0) { + tDebug("%s conn %p failed to remove state %" PRId64 " since %s", pInst->label, conn, *qid, tstrerror(code)); + } else { + tDebug("%s conn %p destroy sid::%" PRId64 "", pInst->label, conn, *qid); + } + + STransCtx *ctx = pIter; + transCtxCleanup(ctx); + + transReleaseExHandle(transGetRefMgt(), *qid); + transRemoveExHandle(transGetRefMgt(), *qid); + + pIter = taosHashIterate(conn->pQTable, pIter); + } + taosHashCleanup(conn->pQTable); + conn->pQTable = NULL; +} +static void evtCliDestroy(SCliConn *pConn) { + int32_t code = 0; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + // cliResetConnTimer(conn); + + tDebug("%s conn %p try to destroy", pInst->label, pConn); + + code = destroyAllReqs(pConn); + if (code != 0) { + tDebug("%s conn %p failed to all reqs since %s", pInst->label, pConn, tstrerror(code)); + } + + pConn->forceDelFromHeap = 1; + code = delConnFromHeapCache(pThrd->connHeapCache, pConn); + if (code != 0) { + tDebug("%s conn %p failed to del conn from heapcach since %s", pInst->label, pConn, tstrerror(code)); + } + + taosMemoryFree(pConn->dstAddr); + // taosMemoryFree(conn->stream); + taosMemoryFree(pConn->ipStr); + // cliDestroyAllQidFromThrd(conn); + + if (pConn->pInitUserReq) { + taosMemoryFree(pConn->pInitUserReq); + pConn->pInitUserReq = NULL; + } + + // taosMemoryFree(conn->buf); + // destroyWQ(&conn->wq); + + taosCloseSocketNoCheck1(pConn->fd); + transDestroyBuffer(&pConn->readBuf); + + tTrace("%s conn %p destroy successfully", pInst->label, pConn); + + taosMemoryFree(pConn); +} +static void evtCliDestroyConn(SCliConn *pConn, bool force) { + int32_t code = 0; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + // cliResetConnTimer(conn); + code = destroyAllReqs(pConn); + if (code != 0) { + tError("%s conn %p failed to destroy all reqs on conn since %s", pInst->label, pConn, tstrerror(code)); + } + + evtCliDestroyAllQidFromThrd(pConn); + if (pThrd->quit == false && pConn->list) { + QUEUE_REMOVE(&pConn->q); + pConn->list->totalSize -= 1; + pConn->list = NULL; + } + + if (pConn->task != NULL) { + transDQCancel(((SCliThrd2 *)pConn->hostThrd)->pEvtMgt->arg, pConn->task); + pConn->task = NULL; + } + pConn->forceDelFromHeap = 1; + code = delConnFromHeapCache(pThrd->connHeapCache, pConn); + if (code != 0) { + tError("%s conn %p failed to del conn from heapcach since %s", pInst->label, pConn, tstrerror(code)); + } + + if (pConn->registered) { + int8_t ref = pConn->ref; + if (ref == 0) { + // uv_close((uv_handle_t *)conn->stream, cliDestroy); + } + } + return; +} +static void transRefCliHandle(void *handle) { + int32_t ref = 0; + if (handle == NULL) { + return; + } + SCliConn *conn = (SCliConn *)handle; + SCliThrd2 *thrd = conn->hostThrd; + conn->ref++; + + tTrace("%s conn %p ref %d", thrd->pInst->label, conn, conn->ref); +} +static int32_t transUnrefCliHandle(void *handle) { + if (handle == NULL) { + return 0; + } + int32_t ref = 0; + SCliConn *conn = (SCliConn *)handle; + SCliThrd2 *thrd = conn->hostThrd; + conn->ref--; + ref = conn->ref; + + tTrace("%s conn %p ref:%d", thrd->pInst->label, conn, conn->ref); + if (conn->ref == 0) { + evtCliDestroyConn(conn, false); + } + return ref; +} +// static int32_t transGet + +static FORCE_INLINE void removeReqFromSendQ(SCliReq *pReq) { + if (pReq == NULL || pReq->inSendQ == 0) { + return; + } + QUEUE_REMOVE(&pReq->sendQ); + pReq->inSendQ = 0; +} +static void destroyThrdObj(SCliThrd2 *pThrd) { + if (pThrd == NULL) { + return; + } + if (pThrd->pEvtMgt) { + evtMgtDestroy(pThrd->pEvtMgt); + } + taosMemoryFree(pThrd); +} +static int32_t createThrdObj(void *trans, SCliThrd2 **ppThrd) { + int32_t line = 0; + int32_t code = 0; + + STrans *pInst = trans; + + SCliThrd2 *pThrd = (SCliThrd2 *)taosMemoryCalloc(1, sizeof(SCliThrd2)); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(terrno, &line, _end); + } + pThrd->pool = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + + QUEUE_INIT(&pThrd->msg); + taosThreadMutexInit(&pThrd->msgMtx, NULL); + + pThrd->destroyAhandleFp = pInst->destroyFp; + + pThrd->fqdn2ipCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pThrd->fqdn2ipCache == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + + pThrd->batchCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pThrd->batchCache == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _end); + } + + pThrd->connHeapCache = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); + if (pThrd->connHeapCache == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + + pThrd->pIdConnTable = taosHashInit(1024, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); + if (pThrd->connHeapCache == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + + pThrd->pQIdBuf = taosArrayInit(8, sizeof(int64_t)); + if (pThrd->pQIdBuf == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_OUT_OF_MEMORY, NULL, _end); + } + + *ppThrd = pThrd; + return code; +_end: + if (code != 0) { + destroyThrdObj(pThrd); + tError("%s failed to init rpc client at line %d since %s,", __func__, line, tstrerror(code)); + } + return code; +} + +FORCE_INLINE int cliRBChoseIdx(STrans *pInst) { + int32_t index = pInst->index; + if (pInst->numOfThreads == 0) { + return -1; + } + /* + * no lock, and to avoid CPU load imbalance, set limit pInst->numOfThreads * 2000; + */ + if (pInst->index++ >= pInst->numOfThreads * 2000) { + pInst->index = 0; + } + return index % pInst->numOfThreads; +} +static FORCE_INLINE SCliThrd2 *transGetWorkThrdFromHandle(STrans *trans, int64_t handle) { + SCliThrd2 *pThrd = NULL; + SExHandle *exh = transAcquireExHandle(transGetRefMgt(), handle); + if (exh == NULL) { + return NULL; + } + taosWLockLatch(&exh->latch); + if (exh->pThrd == NULL && trans != NULL) { + int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; + exh->pThrd = ((SCliObj2 *)trans->tcphandle)->pThreadObj[idx]; + } + + pThrd = exh->pThrd; + taosWUnLockLatch(&exh->latch); + transReleaseExHandle(transGetRefMgt(), handle); + + return pThrd; +} +static SCliThrd2 *transGetWorkThrd(STrans *trans, int64_t handle) { + if (handle == 0) { + int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; + return ((SCliObj2 *)trans->tcphandle)->pThreadObj[idx]; + } + SCliThrd2 *pThrd = transGetWorkThrdFromHandle(trans, handle); + return pThrd; +} +static int32_t transInitMsg(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransCtx *ctx, SCliReq **pCliMsg) { + int32_t code = 0; + if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + SCliReq *pCliReq = NULL; + SReqCtx *pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->epSet); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _exception); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->origEpSet); + if (code != 0) { + TAOS_CHECK_GOTO(code, NULL, _exception); + } + + pCtx->ahandle = pReq->info.ahandle; + pCtx->msgType = pReq->msgType; + + if (ctx != NULL) pCtx->userCtx = *ctx; + + pCliReq = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pCliReq == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + + pCliReq->ctx = pCtx; + pCliReq->msg = *pReq; + pCliReq->st = taosGetTimestampUs(); + pCliReq->type = Normal; + + *pCliMsg = pCliReq; + return code; +_exception: + if (pCtx != NULL) { + taosMemoryFree(pCtx->epSet); + taosMemoryFree(pCtx->origEpSet); + taosMemoryFree(pCtx); + } + taosMemoryFree(pCliReq); + return code; +} + +static FORCE_INLINE void destroyReq(void *arg) { + SCliReq *pReq = arg; + if (pReq == NULL) { + return; + } + + removeReqFromSendQ(pReq); + STraceId *trace = &pReq->msg.info.traceId; + tGDebug("free memory:%p, free ctx: %p", pReq, pReq->ctx); + + if (pReq->ctx) { + destroyReqCtx(pReq->ctx); + } + transFreeMsg(pReq->msg.pCont); + taosMemoryFree(pReq); +} + +int32_t transReleaseCliHandle2(void *handle, int32_t status) { + int32_t code = 0; + SCliThrd2 *pThrd = transGetWorkThrdFromHandle(NULL, (int64_t)handle); + if (pThrd == NULL) { + return TSDB_CODE_RPC_BROKEN_LINK; + } + + STransMsg tmsg = {.msgType = TDMT_SCH_TASK_RELEASE, + .info.handle = handle, + .info.ahandle = (void *)0, + .info.qId = (int64_t)handle, + code = status}; + + TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64()); + + SReqCtx *pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + return terrno; + } + pCtx->ahandle = tmsg.info.ahandle; + SCliReq *cmsg = taosMemoryCalloc(1, sizeof(SCliReq)); + + if (cmsg == NULL) { + taosMemoryFree(pCtx); + return terrno; + } + cmsg->msg = tmsg; + cmsg->st = taosGetTimestampUs(); + cmsg->type = Normal; + cmsg->ctx = pCtx; + + STraceId *trace = &tmsg.info.traceId; + tGDebug("send release request at thread:%08" PRId64 ", malloc memory:%p", pThrd->pid, cmsg); + + if ((code = evtAsyncSend(pThrd->asyncHandle, &cmsg->q)) != 0) { + destroyReq(cmsg); + return code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code; + } + return code; +} + +int32_t transSendRequest2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransCtx *ctx) { + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + return TSDB_CODE_RPC_MODULE_QUIT; + } + int32_t code = 0; + int64_t handle = (int64_t)pReq->info.handle; + SCliThrd2 *pThrd = transGetWorkThrd(pInst, handle); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _exception); + } + + pReq->info.qId = handle; + + SCliReq *pCliMsg = NULL; + TAOS_CHECK_GOTO(transInitMsg(pInstRef, pEpSet, pReq, ctx, &pCliMsg), NULL, _exception); + + STraceId *trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, + EPSET_GET_INUSE_IP(pEpSet), EPSET_GET_INUSE_PORT(pEpSet), pReq->info.ahandle); + + code = evtAsyncSend(pThrd->asyncHandle, &pCliMsg->q); + if (code != 0) { + destroyReq(pCliMsg); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); + } + + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return 0; + +_exception: + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (code != 0) { + tError("failed to send request since %s", tstrerror(code)); + } + return code; +} +static bool isReqExceedLimit(STransMsg *pReq) { + if (pReq != NULL && pReq->contLen >= TRANS_MSG_LIMIT) { + return true; + } + return false; +} +int32_t transSendRequestWithId2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, int64_t *transpointId) { + if (transpointId == NULL) { + return TSDB_CODE_INVALID_PARA; + } + if (isReqExceedLimit(pReq)) { + return TSDB_CODE_RPC_MSG_EXCCED_LIMIT; + } + int32_t code = 0; + int8_t transIdInited = 0; + + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_MODULE_QUIT, NULL, _exception); + } + + TAOS_CHECK_GOTO(transAllocHandle2(transpointId), NULL, _exception); + + SCliThrd2 *pThrd = transGetWorkThrd(pInst, *transpointId); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _exception); + } + + SExHandle *exh = transAcquireExHandle(transGetRefMgt(), *transpointId); + if (exh == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_MODULE_QUIT, NULL, _exception); + } + transIdInited = 1; + + pReq->info.handle = (void *)(*transpointId); + pReq->info.qId = *transpointId; + + SCliReq *pCliMsg = NULL; + TAOS_CHECK_GOTO(transInitMsg(pInstRef, pEpSet, pReq, NULL, &pCliMsg), NULL, _exception); + + STraceId *trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, + EPSET_GET_INUSE_IP(pEpSet), EPSET_GET_INUSE_PORT(pEpSet), pReq->info.ahandle); + if ((code = evtAsyncSend(pThrd->asyncHandle, &(pCliMsg->q))) != 0) { + destroyReq(pCliMsg); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); + } + + transReleaseExHandle(transGetRefMgt(), *transpointId); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return 0; + +_exception: + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + if (transIdInited) transReleaseExHandle(transGetRefMgt(), *transpointId); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + + tError("failed to send request since %s", tstrerror(code)); + return code; +} +int32_t transSendRecv2(void *pInstRef, const SEpSet *pEpSet, STransMsg *pReq, STransMsg *pRsp) { + if (isReqExceedLimit(pReq)) { + return TSDB_CODE_RPC_MSG_EXCCED_LIMIT; + } + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + return TSDB_CODE_RPC_MODULE_QUIT; + } + int32_t code = 0; + SCliReq *pCliReq = NULL; + SReqCtx *pCtx = NULL; + + STransMsg *pTransRsp = taosMemoryCalloc(1, sizeof(STransMsg)); + if (pTransRsp == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + SCliThrd2 *pThrd = transGetWorkThrd(pInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _RETURN1); + } + + tsem_t *sem = taosMemoryCalloc(1, sizeof(tsem_t)); + if (sem == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + code = tsem_init(sem, 0, 0); + if (code != 0) { + taosMemoryFree(sem); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + TAOS_UNUSED(tsem_destroy(sem)); + taosMemoryFree(sem); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->epSet); + if (code != 0) { + (TAOS_UNUSED(tsem_destroy(sem))); + taosMemoryFree(sem); + TAOS_CHECK_GOTO(code, NULL, _RETURN1); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->origEpSet); + if (code != 0) { + (TAOS_UNUSED(tsem_destroy(sem))); + taosMemoryFree(sem); + TAOS_CHECK_GOTO(code, NULL, _RETURN1); + } + + pCtx->ahandle = pReq->info.ahandle; + pCtx->msgType = pReq->msgType; + pCtx->pSem = sem; + pCtx->pRsp = pTransRsp; + + pCliReq = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pCliReq == NULL) { + (TAOS_UNUSED(tsem_destroy(sem))); + taosMemoryFree(sem); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN1); + } + + pCliReq->ctx = pCtx; + pCliReq->msg = *pReq; + pCliReq->st = taosGetTimestampUs(); + pCliReq->type = Normal; + + STraceId *trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, + EPSET_GET_INUSE_IP(pCtx->epSet), EPSET_GET_INUSE_PORT(pCtx->epSet), pReq->info.ahandle); + + code = evtAsyncSend(pThrd->asyncHandle, &pCliReq->q); + if (code != 0) { + destroyReq(pReq); + TAOS_CHECK_GOTO((code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code), NULL, _RETURN); + } + TAOS_UNUSED(tsem_wait(sem)); + + memcpy(pRsp, pTransRsp, sizeof(STransMsg)); + +_RETURN: + tsem_destroy(sem); + taosMemoryFree(sem); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + taosMemoryFree(pTransRsp); + return code; +_RETURN1: + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + taosMemoryFree(pTransRsp); + taosMemoryFree(pReq->pCont); + pReq->pCont = NULL; + if (pCtx != NULL) { + taosMemoryFree(pCtx->epSet); + taosMemoryFree(pCtx->origEpSet); + taosMemoryFree(pCtx); + } + return code; +} +static int32_t transCreateSyncMsg(STransMsg *pTransMsg, int64_t *refId) { + int32_t code = 0; + tsem2_t *sem = taosMemoryCalloc(1, sizeof(tsem2_t)); + if (sem == NULL) { + return terrno; + } + + if (tsem2_init(sem, 0, 0) != 0) { + TAOS_CHECK_GOTO(TAOS_SYSTEM_ERROR(errno), NULL, _EXIT); + } + + STransSyncMsg *pSyncMsg = taosMemoryCalloc(1, sizeof(STransSyncMsg)); + if (pSyncMsg == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _EXIT); + } + + taosInitRWLatch(&pSyncMsg->latch); + pSyncMsg->inited = 0; + pSyncMsg->pRsp = pTransMsg; + pSyncMsg->pSem = sem; + pSyncMsg->hasEpSet = 0; + + int64_t id = taosAddRef(transGetSyncMsgMgt(), pSyncMsg); + if (id < 0) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _EXIT); + } else { + *refId = id; + } + return 0; + +_EXIT: + TAOS_UNUSED(tsem2_destroy(sem)); + taosMemoryFree(sem); + taosMemoryFree(pSyncMsg); + return code; +} + +int32_t transSendRecvWithTimeout2(void *pInstRef, SEpSet *pEpSet, STransMsg *pReq, STransMsg *pRsp, int8_t *epUpdated, + int32_t timeoutMs) { + int32_t code = 0; + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + transFreeMsg(pReq->pCont); + pReq->pCont = NULL; + return TSDB_CODE_RPC_MODULE_QUIT; + } + + STransMsg *pTransMsg = taosMemoryCalloc(1, sizeof(STransMsg)); + if (pTransMsg == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); + } + + SCliThrd2 *pThrd = transGetWorkThrd(pInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_RPC_BROKEN_LINK, NULL, _RETURN2); + } + + if (pReq->info.traceId.msgId == 0) TRACE_SET_MSGID(&pReq->info.traceId, tGenIdPI64()); + + SReqCtx *pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); + } + + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->epSet); + if (code != 0) { + taosMemoryFreeClear(pCtx->epSet); + TAOS_CHECK_GOTO(code, NULL, _RETURN2); + } + code = transCreateReqEpsetFromUserEpset(pEpSet, &pCtx->origEpSet); + if (code != 0) { + taosMemoryFreeClear(pCtx->epSet); + TAOS_CHECK_GOTO(code, NULL, _RETURN2); + } + pCtx->ahandle = pReq->info.ahandle; + pCtx->msgType = pReq->msgType; + + if ((code = transCreateSyncMsg(pTransMsg, &pCtx->syncMsgRef)) != 0) { + taosMemoryFree(pCtx); + TAOS_CHECK_GOTO(code, NULL, _RETURN2); + } + + int64_t ref = pCtx->syncMsgRef; + STransSyncMsg *pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), ref); + if (pSyncMsg == NULL) { + taosMemoryFree(pCtx); + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _RETURN2); + } + + SCliReq *pCliReq = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pReq == NULL) { + taosMemoryFree(pCtx); + TAOS_CHECK_GOTO(terrno, NULL, _RETURN2); + } + + pCliReq->ctx = pCtx; + pCliReq->msg = *pReq; + pCliReq->st = taosGetTimestampUs(); + pCliReq->type = Normal; + + STraceId *trace = &pReq->info.traceId; + tGDebug("%s send request at thread:%08" PRId64 ", dst:%s:%d, app:%p", transLabel(pInst), pThrd->pid, + EPSET_GET_INUSE_IP(pCtx->epSet), EPSET_GET_INUSE_PORT(pCtx->epSet), pReq->info.ahandle); + + code = evtAsyncSend(pThrd->asyncHandle, &pCliReq->q); + if (code != 0) { + destroyReq(pReq); + TAOS_CHECK_GOTO(code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code, NULL, _RETURN); + goto _RETURN; + } + + code = tsem2_timewait(pSyncMsg->pSem, timeoutMs); + if (code != 0) { + pRsp->code = code; + } else { + memcpy(pRsp, pSyncMsg->pRsp, sizeof(STransMsg)); + pSyncMsg->pRsp->pCont = NULL; + if (pSyncMsg->hasEpSet == 1) { + epsetAssign(pEpSet, &pSyncMsg->epSet); + *epUpdated = 1; + } + } +_RETURN: + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + TAOS_UNUSED(taosReleaseRef(transGetSyncMsgMgt(), ref)); + TAOS_UNUSED(taosRemoveRef(transGetSyncMsgMgt(), ref)); + return code; +_RETURN2: + transFreeMsg(pReq->pCont); + + if (pCtx != NULL) { + taosMemoryFree(pCtx->epSet); + taosMemoryFree(pCtx->origEpSet); + taosMemoryFree(pCtx); + } + pReq->pCont = NULL; + taosMemoryFree(pTransMsg); + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return code; +} + +int32_t transSetDefaultAddr2(void *pInstRef, const char *ip, const char *fqdn) { + if (ip == NULL || fqdn == NULL) return TSDB_CODE_INVALID_PARA; + + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + return TSDB_CODE_RPC_MODULE_QUIT; + } + + SCvtAddr cvtAddr = {0}; + tstrncpy(cvtAddr.ip, ip, sizeof(cvtAddr.ip)); + tstrncpy(cvtAddr.fqdn, fqdn, sizeof(cvtAddr.fqdn)); + cvtAddr.cvt = true; + + int32_t code = 0; + for (int8_t i = 0; i < pInst->numOfThreads; i++) { + SReqCtx *pCtx = taosMemoryCalloc(1, sizeof(SReqCtx)); + if (pCtx == NULL) { + code = terrno; + break; + } + + pCtx->pCvtAddr = (SCvtAddr *)taosMemoryCalloc(1, sizeof(SCvtAddr)); + if (pCtx->pCvtAddr == NULL) { + taosMemoryFree(pCtx); + code = terrno; + break; + } + + memcpy(pCtx->pCvtAddr, &cvtAddr, sizeof(SCvtAddr)); + + SCliReq *pReq = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pReq == NULL) { + taosMemoryFree(pCtx->pCvtAddr); + taosMemoryFree(pCtx); + code = terrno; + break; + } + + pReq->ctx = pCtx; + pReq->type = Update; + + SCliThrd2 *thrd = ((SCliObj2 *)pInst->tcphandle)->pThreadObj[i]; + tDebug("%s update epset at thread:%08" PRId64, pInst->label, thrd->pid); + + if ((code = evtAsyncSend(thrd->asyncHandle, &(pReq->q))) != 0) { + taosMemoryFree(pCtx->pCvtAddr); + destroyReq(pReq); + if (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT) { + code = TSDB_CODE_RPC_MODULE_QUIT; + } + break; + } + } + + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return code; +} +int32_t transAllocHandle2(int64_t *refId) { + SExHandle *exh = taosMemoryCalloc(1, sizeof(SExHandle)); + if (exh == NULL) { + return terrno; + } + + exh->refId = transAddExHandle(transGetRefMgt(), exh); + if (exh->refId < 0) { + taosMemoryFree(exh); + return TSDB_CODE_REF_INVALID_ID; + } + + SExHandle *self = transAcquireExHandle(transGetRefMgt(), exh->refId); + if (exh != self) { + taosMemoryFree(exh); + return TSDB_CODE_REF_INVALID_ID; + } + + QUEUE_INIT(&exh->q); + taosInitRWLatch(&exh->latch); + tDebug("trans alloc sid:%" PRId64 ", malloc:%p", exh->refId, exh); + *refId = exh->refId; + return 0; +} + +int32_t transFreeConnById2(void *pInstRef, int64_t transpointId) { + int32_t code = 0; + STrans *pInst = (STrans *)transAcquireExHandle(transGetInstMgt(), (int64_t)pInstRef); + if (pInst == NULL) { + return TSDB_CODE_RPC_MODULE_QUIT; + } + if (transpointId == 0) { + tDebug("not free by refId:%" PRId64 "", transpointId); + TAOS_CHECK_GOTO(0, NULL, _exception); + } + + SCliThrd2 *pThrd = transGetWorkThrdFromHandle(pInst, transpointId); + if (pThrd == NULL) { + TAOS_CHECK_GOTO(TSDB_CODE_REF_INVALID_ID, NULL, _exception); + } + + SCliReq *pCli = taosMemoryCalloc(1, sizeof(SCliReq)); + if (pCli == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _exception); + } + pCli->type = Normal; + + STransMsg msg = {.msgType = TDMT_SCH_TASK_RELEASE, .info.handle = (void *)transpointId}; + TRACE_SET_MSGID(&msg.info.traceId, tGenIdPI64()); + msg.info.qId = transpointId; + pCli->msg = msg; + + STraceId *trace = &pCli->msg.info.traceId; + tGDebug("%s start to free conn sid:%" PRId64 "", pInst->label, transpointId); + + code = evtAsyncSend(pThrd->asyncHandle, &pCli->q); + if (code != 0) { + taosMemoryFreeClear(pCli); + TAOS_CHECK_GOTO(code, NULL, _exception); + } + +_exception: + transReleaseExHandle(transGetInstMgt(), (int64_t)pInstRef); + return code; +} + +static int32_t evtCliRecycleConn(SCliConn *pConn) { + int32_t code = 0; + + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + tTrace("%s conn %p in-process req summary:reqsToSend:%d, reqsSentOut:%d, statusTableSize:%d", pInst->label, pConn, + transQueueSize(&pConn->reqsToSend), transQueueSize(&pConn->reqsSentOut), taosHashGetSize(pConn->pQTable)); + + if (transQueueSize(&pConn->reqsToSend) == 0 && transQueueSize(&pConn->reqsSentOut) == 0 && + taosHashGetSize(pConn->pQTable) == 0) { + // cliResetConnTimer(conn); + pConn->forceDelFromHeap = 1; + code = delConnFromHeapCache(pThrd->connHeapCache, pConn); + if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { + tDebug("%s conn %p failed to remove conn from heap cache since %s", pInst->label, pConn, tstrerror(code)); + + TAOS_UNUSED(transHeapMayBalance(pConn->heap, pConn)); + return 1; + } else { + if (code != 0) { + tDebug("%s conn %p failed to remove conn from heap cache since %s", pInst->label, pConn, tstrerror(code)); + return 0; + } + } + addConnToPool(pThrd->pool, pConn); + return 1; + } else if ((transQueueSize(&pConn->reqsToSend) == 0) && (transQueueSize(&pConn->reqsSentOut) == 0) && + (taosHashGetSize(pConn->pQTable) != 0)) { + tDebug("%s conn %p do balance directly", pInst->label, pConn); + TAOS_UNUSED(transHeapMayBalance(pConn->heap, pConn)); + } else { + // tTrace("%s conn %p may do balance", CONN_GET_INST_LABEL(conn), conn); + TAOS_UNUSED(transHeapMayBalance(pConn->heap, pConn)); + } + + return code; +} +static bool filteBySeq(void *key, void *arg) { + SFiterArg *targ = arg; + SCliReq *pReq = QUEUE_DATA(key, SCliReq, q); + if (pReq->seq == targ->seq && pReq->msg.msgType + 1 == targ->msgType) { + removeReqFromSendQ(pReq); + return true; + } else { + return false; + } +} +static int32_t cliGetReqBySeq(SCliConn *pConn, int64_t seq, int32_t msgType, SCliReq **ppReq) { + int32_t code = 0; + queue set; + QUEUE_INIT(&set); + + SFiterArg arg = {.seq = seq, .msgType = msgType}; + transQueueRemoveByFilter(&pConn->reqsSentOut, filteBySeq, &arg, &set, 1); + while (QUEUE_IS_EMPTY(&set)) { + return TSDB_CODE_OUT_OF_RANGE; + } + + queue *e = QUEUE_HEAD(&set); + SCliReq *pReq = QUEUE_DATA(e, SCliReq, q); + + *ppReq = pReq; + return 0; +} + +#define CONN_GET_INST_LABEL(conn) (((STrans *)(((SCliThrd2 *)(conn)->hostThrd)->pInst))->label) +static bool cliIsEpsetUpdated(int32_t code, SReqCtx *pCtx) { + if (code != 0) return false; + return transReqEpsetIsEqual(pCtx->epSet, pCtx->origEpSet) ? false : true; +} +static int32_t evtCliNotifyImplCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { + int32_t code = 0; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + SReqCtx *pCtx = pReq ? pReq->ctx : NULL; + STraceId *trace = &pResp->info.traceId; + + if (pCtx == NULL) { + pInst->cfp(pInst->parent, pResp, NULL); + return 0; + } + if (pCtx->pSem || pCtx->syncMsgRef != 0) { + tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); + if (pCtx->pSem) { + if (pCtx->pRsp == NULL) { + tGTrace("%s conn %p(sync) failed to resp, ignore", CONN_GET_INST_LABEL(pConn), pConn); + } else { + memcpy((char *)pCtx->pRsp, (char *)pResp, sizeof(*pResp)); + } + TAOS_UNUSED(tsem_post(pCtx->pSem)); + pCtx->pRsp = NULL; + } else { + STransSyncMsg *pSyncMsg = taosAcquireRef(transGetSyncMsgMgt(), pCtx->syncMsgRef); + if (pSyncMsg != NULL) { + memcpy(pSyncMsg->pRsp, (char *)pResp, sizeof(*pResp)); + if (cliIsEpsetUpdated(pResp->code, pCtx)) { + pSyncMsg->hasEpSet = 1; + + SEpSet epset = {0}; + if (transCreateUserEpsetFromReqEpset(pCtx->epSet, &epset) != 0) { + tError("failed to create user epset from req epset"); + } + epsetAssign(&pSyncMsg->epSet, &epset); + } + TAOS_UNUSED(tsem2_post(pSyncMsg->pSem)); + TAOS_UNUSED(taosReleaseRef(transGetSyncMsgMgt(), pCtx->syncMsgRef)); + } else { + rpcFreeCont(pResp->pCont); + } + } + } else { + tGTrace("%s conn %p handle resp", CONN_GET_INST_LABEL(pConn), pConn); + if (pResp->info.hasEpSet == 1) { + SEpSet epset = {0}; + if (transCreateUserEpsetFromReqEpset(pCtx->epSet, &epset) != 0) { + tError("failed to create user epset from req epset"); + } + pInst->cfp(pInst->parent, pResp, &epset); + } else { + if (!cliIsEpsetUpdated(pResp->code, pCtx)) { + pInst->cfp(pInst->parent, pResp, NULL); + } else { + SEpSet epset = {0}; + if (transCreateUserEpsetFromReqEpset(pCtx->epSet, &epset) != 0) { + tError("failed to create user epset from req epset"); + } + pInst->cfp(pInst->parent, pResp, &epset); + } + } + } + + return code; +} +FORCE_INLINE bool evtCliTryUpdateEpset(SCliReq *pReq, STransMsg *pResp) { + int32_t code = 0; + SReqCtx *ctx = pReq->ctx; + + if ((pResp == NULL || pResp->info.hasEpSet == 0)) { + return false; + } + // rebuild resp msg + SEpSet epset; + if ((code = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epset)) < 0) { + tError("failed to deserialize epset, code:%d", code); + return false; + } + SEpSet tepset; + int32_t tlen = tSerializeSEpSet(NULL, 0, &tepset); + + char *buf = NULL; + int32_t len = pResp->contLen - tlen; + if (len != 0) { + buf = rpcMallocCont(len); + if (buf == NULL) { + pResp->code = TSDB_CODE_OUT_OF_MEMORY; + return false; + } + // TODO: check buf + memcpy(buf, (char *)pResp->pCont + tlen, len); + } + rpcFreeCont(pResp->pCont); + + pResp->pCont = buf; + pResp->contLen = len; + + pResp->info.hasEpSet = 1; + + if (transCreateReqEpsetFromUserEpset(&epset, &ctx->epSet) != 0) { + return false; + } + return true; +} +static void evtCliMayResetRespCode(SCliReq *pReq, STransMsg *pResp) { + SReqCtx *pCtx = pReq->ctx; + if (pCtx->retryCode != TSDB_CODE_SUCCESS) { + int32_t code = pResp->code; + // return internal code app + if (code == TSDB_CODE_RPC_NETWORK_UNAVAIL || code == TSDB_CODE_RPC_BROKEN_LINK || + code == TSDB_CODE_RPC_SOMENODE_NOT_CONNECTED) { + pResp->code = pCtx->retryCode; + } + } + + // check whole vnodes is offline on this vgroup + if (((pCtx->epSet != NULL) && pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) || pCtx->retryStep > 0) { + if (pResp->code == TSDB_CODE_RPC_BROKEN_LINK) { + pResp->code = TSDB_CODE_RPC_NETWORK_UNAVAIL; // TSDB_CODE_RPC_SOMENODE_BROKEN_LINK; + } + } +} +void cliRetryMayInitCtx(STrans *pInst, SCliReq *pReq) { + SReqCtx *pCtx = pReq->ctx; + if (!pCtx->retryInit) { + pCtx->retryMinInterval = pInst->retryMinInterval; + pCtx->retryMaxInterval = pInst->retryMaxInterval; + pCtx->retryStepFactor = pInst->retryStepFactor; + pCtx->retryMaxTimeout = pInst->retryMaxTimeout; + pCtx->retryInitTimestamp = taosGetTimestampMs(); + pCtx->retryNextInterval = pCtx->retryMinInterval; + pCtx->retryStep = 0; + pCtx->retryInit = 1; + pCtx->retryCode = TSDB_CODE_SUCCESS; + pReq->msg.info.handle = 0; + } +} +int32_t cliRetryIsTimeout(STrans *pInst, SCliReq *pReq) { + SReqCtx *pCtx = pReq->ctx; + if (pCtx->retryMaxTimeout != -1 && taosGetTimestampMs() - pCtx->retryInitTimestamp >= pCtx->retryMaxTimeout) { + return 1; + } + return 0; +} + +int8_t cliRetryShouldRetry(STrans *pInst, STransMsg *pResp) { + bool retry = pInst->retry != NULL ? pInst->retry(pResp->code, pResp->msgType - 1) : false; + return retry == false ? 0 : 1; +} + +void cliRetryUpdateRule(SReqCtx *pCtx, int8_t noDelay) { + if (noDelay == false) { + pCtx->epsetRetryCnt = 1; + pCtx->retryStep++; + + int64_t factor = pow(pCtx->retryStepFactor, pCtx->retryStep - 1); + pCtx->retryNextInterval = factor * pCtx->retryMinInterval; + if (pCtx->retryNextInterval >= pCtx->retryMaxInterval) { + pCtx->retryNextInterval = pCtx->retryMaxInterval; + } + } else { + pCtx->retryNextInterval = 0; + pCtx->epsetRetryCnt++; + } +} +bool cliResetEpset(SReqCtx *pCtx, STransMsg *pResp, bool hasEpSet) { + bool noDelay = true; + if (hasEpSet == false) { + if (pResp->contLen == 0) { + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } else if (pResp->contLen != 0) { + SEpSet epSet; + int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + if (valid < 0) { + tDebug("get invalid epset, epset equal, continue"); + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } else { + if (!transCompareReqAndUserEpset(pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset1"); + transPrintEpSet((SEpSet *)pCtx->epSet); + transPrintEpSet(&epSet); + + if (transCreateReqEpsetFromUserEpset(&epSet, &pCtx->epSet) != 0) { + tDebug("failed to create req epset from user epset"); + } + noDelay = false; + } else { + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } + } + } + } else { + SEpSet epSet; + int32_t valid = tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + if (valid < 0) { + tDebug("get invalid epset, epset equal, continue"); + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } else { + if (!transCompareReqAndUserEpset(pCtx->epSet, &epSet)) { + tDebug("epset not equal, retry new epset2"); + transPrintEpSet((SEpSet *)pCtx->epSet); + transPrintEpSet(&epSet); + if (transCreateReqEpsetFromUserEpset(&epSet, &pCtx->epSet) != 0) { + tError("failed to create req epset from user epset"); + } + noDelay = false; + } else { + if (pCtx->epsetRetryCnt >= pCtx->epSet->numOfEps) { + noDelay = false; + } else { + tDebug("epset equal, continue"); + EPSET_FORWARD_INUSE(pCtx->epSet); + } + } + } + } + return noDelay; +} + +static void doDelayTask(void *param) { + STaskArg *arg = param; + if (arg && arg->param1) { + SCliReq *pReq = arg->param1; + pReq->inRetry = 1; + } + evtCliHandlReq((SCliReq *)arg->param1, (SCliThrd2 *)arg->param2); + taosMemoryFree(arg); +} +static int32_t evtCliDoSched(SCliReq *pReq, SCliThrd2 *pThrd) { + int32_t code = 0; + STrans *pInst = pThrd->pInst; + SReqCtx *pCtx = pReq->ctx; + + STaskArg *arg = taosMemoryMalloc(sizeof(STaskArg)); + if (arg == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + arg->param1 = pReq; + arg->param2 = pThrd; + + SDelayTask *pTask = transDQSched(pThrd->pEvtMgt->arg, doDelayTask, arg, pCtx->retryNextInterval); + if (pTask == NULL) { + taosMemoryFree(arg); + return TSDB_CODE_OUT_OF_MEMORY; + } + return code; +} +static bool evtCliMayRetry(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + SReqCtx *pCtx = pReq->ctx; + int32_t code = pResp->code; + + if (pReq && pReq->msg.info.qId != 0) { + return false; + } + cliRetryMayInitCtx(pInst, pReq); + + if (!cliRetryShouldRetry(pInst, pResp)) { + return false; + } + + if (cliRetryIsTimeout(pInst, pReq)) { + return false; + } + + bool noDelay = false; + if (code == TSDB_CODE_RPC_BROKEN_LINK || code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { + tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, false); + transFreeMsg(pResp->pCont); + } else if (code == TSDB_CODE_SYN_NOT_LEADER || code == TSDB_CODE_SYN_INTERNAL_ERROR || + code == TSDB_CODE_SYN_PROPOSE_NOT_READY || code == TSDB_CODE_VND_STOPPED || + code == TSDB_CODE_MNODE_NOT_FOUND || code == TSDB_CODE_APP_IS_STARTING || + code == TSDB_CODE_APP_IS_STOPPING || code == TSDB_CODE_VND_STOPPED) { + tTrace("code str %s, contlen:%d 1", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, true); + transFreeMsg(pResp->pCont); + } else if (code == TSDB_CODE_SYN_RESTORING) { + tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, true); + transFreeMsg(pResp->pCont); + } else { + tTrace("code str %s, contlen:%d 0", tstrerror(code), pResp->contLen); + noDelay = cliResetEpset(pCtx, pResp, false); + transFreeMsg(pResp->pCont); + } + pResp->pCont = NULL; + pResp->info.hasEpSet = 0; + if (code != TSDB_CODE_RPC_BROKEN_LINK && code != TSDB_CODE_RPC_NETWORK_UNAVAIL && code != TSDB_CODE_SUCCESS) { + // save one internal code + pCtx->retryCode = code; + } + + cliRetryUpdateRule(pCtx, noDelay); + + pReq->sent = 0; + pReq->seq = 0; + + code = evtCliDoSched(pReq, pThrd); + if (code != 0) { + pResp->code = code; + tError("failed to sched msg to next node since %s", tstrerror(code)); + return false; + } + return true; +} +static int32_t cliNotifyCb(SCliConn *pConn, SCliReq *pReq, STransMsg *pResp) { + int32_t code = 0; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + if (pReq != NULL) { + removeReqFromSendQ(pReq); + if (pResp->code != TSDB_CODE_SUCCESS) { + if (evtCliMayRetry(pConn, pReq, pResp)) { + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } + evtCliMayResetRespCode(pReq, pResp); + } + if (evtCliTryUpdateEpset(pReq, pResp)) { + // cliPerfLog_epset(pConn, pReq); + } + } + return evtCliNotifyImplCb(pConn, pReq, pResp); +} +static int32_t evtCliBuildRespFromCont(SCliReq *pReq, STransMsg *pResp, STransMsgHead *pHead) { + pResp->contLen = transContLenFromMsg(pHead->msgLen); + pResp->pCont = transContFromHead((char *)pHead); + pResp->code = pHead->code; + pResp->msgType = pHead->msgType; + if (pResp->info.ahandle == 0) { + pResp->info.ahandle = (pReq && pReq->ctx) ? pReq->ctx->ahandle : NULL; + } + pResp->info.traceId = pHead->traceId; + pResp->info.hasEpSet = pHead->hasEpSet; + pResp->info.cliVer = htonl(pHead->compatibilityVer); + pResp->info.seq = taosHton64(pHead->seqNum); + + int64_t qid = taosHton64(pHead->qid); + pResp->info.handle = (void *)qid; + return 0; +} + +static bool filterByQid(void *key, void *arg) { + int64_t *qid = arg; + SCliReq *pReq = QUEUE_DATA(key, SCliReq, q); + + if (pReq->msg.info.qId == *qid) { + removeReqFromSendQ(pReq); + return true; + } else { + return false; + } +} + +static FORCE_INLINE void destroyReqWrapper(void *arg, void *param) { + if (arg == NULL) return; + + SCliReq *pReq = arg; + SCliThrd2 *pThrd = param; + + if (pReq->ctx != NULL && pReq->ctx->ahandle != NULL) { + if (pReq->msg.info.notFreeAhandle == 0 && pThrd != NULL && pThrd->destroyAhandleFp != NULL) { + (*pThrd->destroyAhandleFp)(pReq->ctx->ahandle); + } + } + destroyReq(pReq); +} +static FORCE_INLINE void destroyReqAndAhanlde(void *param) { + if (param == NULL) return; + + STaskArg *arg = param; + SCliReq *pReq = arg->param1; + SCliThrd2 *pThrd = arg->param2; + destroyReqWrapper(pReq, pThrd); +} + +int32_t cliHandleStateMayHandleReleaseResp(SCliConn *conn, STransMsgHead *pHead) { + int32_t code = 0; + SCliThrd2 *pThrd = conn->hostThrd; + if (pHead->msgType == TDMT_SCH_TASK_RELEASE || pHead->msgType == TDMT_SCH_TASK_RELEASE + 1) { + int64_t qId = taosHton64(pHead->qid); + STraceId *trace = &pHead->traceId; + int64_t seqNum = taosHton64(pHead->seqNum); + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, seqNum:%" PRId64 ", sid:%" PRId64 "", + CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(pHead->msgType), conn->dst, conn->src, pHead->msgLen, seqNum, + qId); + + STransCtx *p = taosHashGet(conn->pQTable, &qId, sizeof(qId)); + transCtxCleanup(p); + + code = taosHashRemove(conn->pQTable, &qId, sizeof(qId)); + if (code != 0) { + tDebug("%s conn %p failed to release req:%" PRId64 " from conn", CONN_GET_INST_LABEL(conn), conn, qId); + } + + code = taosHashRemove(pThrd->pIdConnTable, &qId, sizeof(qId)); + if (code != 0) { + tDebug("%s conn %p failed to release req:%" PRId64 " from thrd ", CONN_GET_INST_LABEL(conn), conn, qId); + } + + tDebug("%s %p reqToSend:%d, sentOut:%d", CONN_GET_INST_LABEL(conn), conn, transQueueSize(&conn->reqsToSend), + transQueueSize(&conn->reqsSentOut)); + + queue set; + QUEUE_INIT(&set); + transQueueRemoveByFilter(&conn->reqsSentOut, filterByQid, &qId, &set, -1); + transQueueRemoveByFilter(&conn->reqsToSend, filterByQid, &qId, &set, -1); + + transReleaseExHandle(transGetRefMgt(), qId); + transRemoveExHandle(transGetRefMgt(), qId); + + while (!QUEUE_IS_EMPTY(&set)) { + queue *el = QUEUE_HEAD(&set); + QUEUE_REMOVE(el); + SCliReq *pReq = QUEUE_DATA(el, SCliReq, q); + removeReqFromSendQ(pReq); + STraceId *trace = &pReq->msg.info.traceId; + tGDebug("start to free msg %p", pReq); + destroyReqWrapper(pReq, pThrd); + } + taosMemoryFree(pHead); + return 1; + } + return 0; +} +int32_t cliHandleStateMayCreateAhandle(SCliConn *conn, STransMsgHead *pHead, STransMsg *pResp) { + int32_t code = 0; + int64_t qId = taosHton64(pHead->qid); + if (qId == 0) { + return TSDB_CODE_RPC_NO_STATE; + } + + STransCtx *pCtx = taosHashGet(conn->pQTable, &qId, sizeof(qId)); + if (pCtx == 0) { + return TSDB_CODE_RPC_NO_STATE; + } + pCtx->st = taosGetTimestampUs(); + STraceId *trace = &pHead->traceId; + pResp->info.ahandle = transCtxDumpVal(pCtx, pHead->msgType); + tGDebug("%s conn %p %s received from %s, local info:%s, sid:%" PRId64 ", create ahandle %p by %s", + CONN_GET_INST_LABEL(conn), conn, TMSG_INFO(pHead->msgType), conn->dst, conn->src, qId, pResp->info.ahandle, + TMSG_INFO(pHead->msgType)); + return 0; +} + +static FORCE_INLINE void cliConnClearInitUserMsg(SCliConn *conn) { + if (conn->pInitUserReq) { + taosMemoryFree(conn->pInitUserReq); + conn->pInitUserReq = NULL; + } +} +int32_t cliHandleStateMayUpdateStateTime(SCliConn *pConn, SCliReq *pReq) { + int64_t qid = pReq->msg.info.qId; + if (qid > 0) { + STransCtx *pUserCtx = taosHashGet(pConn->pQTable, &qid, sizeof(qid)); + if (pUserCtx != NULL) { + pUserCtx->st = taosGetTimestampUs(); + } + } + return 0; +} + +int32_t cliHandleStateMayUpdateStateCtx(SCliConn *pConn, SCliReq *pReq) { + int32_t code = 0; + int64_t qid = pReq->msg.info.qId; + SReqCtx *pCtx = pReq->ctx; + SCliThrd2 *pThrd = pConn->hostThrd; + if (pCtx == NULL) { + tDebug("%s conn %p not need to update statue ctx, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + return 0; + } + + STransCtx *pUserCtx = taosHashGet(pConn->pQTable, &qid, sizeof(qid)); + if (pUserCtx == NULL) { + pCtx->userCtx.st = taosGetTimestampUs(); + code = taosHashPut(pConn->pQTable, &qid, sizeof(qid), &pCtx->userCtx, sizeof(pCtx->userCtx)); + tDebug("%s conn %p succ to add statue ctx, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + } else { + transCtxMerge(pUserCtx, &pCtx->userCtx); + pUserCtx->st = taosGetTimestampUs(); + tDebug("%s conn %p succ to update statue ctx, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + } + return 0; +} +int32_t cliHandleStateMayUpdateState(SCliConn *pConn, SCliReq *pReq) { + SCliThrd2 *pThrd = pConn->hostThrd; + int32_t code = 0; + int64_t qid = pReq->msg.info.qId; + if (qid == 0) { + return TSDB_CODE_RPC_NO_STATE; + } + + SReqState state = {.conn = pConn, .arg = NULL}; + code = taosHashPut(pThrd->pIdConnTable, &qid, sizeof(qid), &state, sizeof(state)); + if (code != 0) { + tDebug("%s conn %p failed to statue, sid:%" PRId64 " since %s", transLabel(pThrd->pInst), pConn, qid, + tstrerror(code)); + } else { + tDebug("%s conn %p succ to add statue, sid:%" PRId64 " (1)", transLabel(pThrd->pInst), pConn, qid); + } + + TAOS_UNUSED(cliHandleStateMayUpdateStateCtx(pConn, pReq)); + return code; +} + +static int32_t evtCliHandleResp(SCliConn *pConn, char *msg, int32_t msgLen) { + int32_t code = 0; + + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + SCliReq *pReq = NULL; + STransMsgHead *pHead = (STransMsgHead *)msg; + + int64_t qId = taosHton64(pHead->qid); + pHead->code = htonl(pHead->code); + pHead->msgLen = htonl(pHead->msgLen); + int64_t seq = taosHton64(pHead->seqNum); + + STransMsg resp = {0}; + + cliConnClearInitUserMsg(pConn); + + if (cliHandleStateMayHandleReleaseResp(pConn, pHead)) { + if (evtCliRecycleConn(pConn)) { + return code; + } + return 0; + } + + code = cliGetReqBySeq(pConn, seq, pHead->msgType, &pReq); + if (code == TSDB_CODE_OUT_OF_RANGE) { + code = cliHandleStateMayCreateAhandle(pConn, pHead, &resp); + if (code == 0) { + code = evtCliBuildRespFromCont(NULL, &resp, pHead); + code = cliNotifyCb(pConn, NULL, &resp); + return 0; + } + + if (code != 0) { + tWarn("%s conn %p recv unexpected packet, msgType:%s, seqNum:%" PRId64 ", sid:%" PRId64 + ", the sever may sends repeated response since %s", + CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), seq, qId, tstrerror(code)); + // TODO: notify cb + taosMemoryFree(pHead); + if (evtCliRecycleConn(pConn)) { + return 0; + } + return 0; + } + } else { + code = cliHandleStateMayUpdateStateTime(pConn, pReq); + if (code != 0) { + tDebug("%s conn %p failed to update state time sid:%" PRId64 " since %s", CONN_GET_INST_LABEL(pConn), pConn, qId, + tstrerror(code)); + } + } + removeReqFromSendQ(pReq); + + code = evtCliBuildRespFromCont(pReq, &resp, pHead); + STraceId *trace = &resp.info.traceId; + tGDebug("%s conn %p %s received from %s, local info:%s, len:%d, seq:%" PRId64 ", sid:%" PRId64 ", code:%s", + CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(resp.msgType), pConn->dst, pConn->src, pHead->msgLen, seq, qId, + tstrerror(pHead->code)); + code = cliNotifyCb(pConn, pReq, &resp); + if (code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { + tGWarn("%s msg need retry", CONN_GET_INST_LABEL(pConn)); + } else { + destroyReq(pReq); + } + + if (evtCliRecycleConn(pConn)) { + return 0; + } + return code; +} + +bool connMayAddUserInfo(SCliConn *pConn, STransMsgHead **ppHead, int32_t *msgLen) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + if (pConn->userInited == 1) { + return false; + } + STransMsgHead *pHead = *ppHead; + STransMsgHead *tHead = taosMemoryCalloc(1, *msgLen + sizeof(pInst->user)); + if (tHead == NULL) { + return false; + } + memcpy((char *)tHead, (char *)pHead, TRANS_MSG_OVERHEAD); + memcpy((char *)tHead + TRANS_MSG_OVERHEAD, pInst->user, sizeof(pInst->user)); + + memcpy((char *)tHead + TRANS_MSG_OVERHEAD + sizeof(pInst->user), (char *)pHead + TRANS_MSG_OVERHEAD, + *msgLen - TRANS_MSG_OVERHEAD); + + tHead->withUserInfo = 1; + *ppHead = tHead; + *msgLen += sizeof(pInst->user); + + pConn->pInitUserReq = tHead; + pConn->userInited = 1; + return true; +} +static int32_t evtCliPreSendReq(void *arg, SEvtBuf *buf, int32_t status) { + int32_t code = 0, line = 0; + code = evtBufInit(buf); + TAOS_CHECK_GOTO(code, &line, _end); + + SFdCbArg *pArg = arg; + SCliConn *pConn = pArg->data; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + int32_t j = 0; + int32_t batchLimit = 16; + queue reqToSend; + QUEUE_INIT(&reqToSend); + while (!transQueueEmpty(&pConn->reqsToSend)) { + queue *el = transQueuePop(&pConn->reqsToSend); + QUEUE_REMOVE(el); + + SCliReq *pCliMsg = QUEUE_DATA(el, SCliReq, q); + SReqCtx *pCtx = pCliMsg->ctx; + pConn->seq++; + + STransMsg *pReq = (STransMsg *)(&pCliMsg->msg); + if (pReq->pCont == 0) { + pReq->pCont = (void *)rpcMallocCont(0); + if (pReq->pCont == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pReq->contLen = 0; + } + int32_t msgLen = transMsgLenFromCont(pReq->contLen); + STransMsgHead *pHead = transHeadFromCont(pReq->pCont); + + char *content = pReq->pCont; + int32_t contLen = pReq->contLen; + if (connMayAddUserInfo(pConn, &pHead, &msgLen)) { + content = transContFromHead(pHead); + contLen = transContLenFromMsg(msgLen); + // pReq->pCont = (void *)pHead; + } else { + if (pConn->userInited == 0) { + return terrno; + } + } + if (pHead->comp == 0) { + pHead->noResp = (pReq->info.noResp ? 1 : 0); + pHead->msgType = pReq->msgType; + pHead->msgLen = (int32_t)htonl((uint32_t)msgLen); + pHead->traceId = pReq->info.traceId; + pHead->magicNum = htonl(TRANS_MAGIC_NUM); + pHead->version = TRANS_VER; + pHead->compatibilityVer = htonl(pInst->compatibilityVer); + } + pHead->timestamp = taosHton64(pCliMsg->st); + pHead->seqNum = taosHton64(pConn->seq); + pHead->qid = taosHton64(pReq->info.qId); + + pCliMsg->seq = pConn->seq; + pCliMsg->sent = 1; + + transQueuePush(&pConn->reqsSentOut, &pCliMsg->q); + + QUEUE_INIT(&pCliMsg->sendQ); + QUEUE_PUSH(&reqToSend, &pCliMsg->sendQ); + + code = evtBufPush(buf, (char *)pHead, msgLen); + TAOS_CHECK_GOTO(code, &line, _end); + j++; + + if (j >= batchLimit) { + break; + } + STraceId *trace = &pReq->info.traceId; + tGDebug("%s conn %p %s is sent to %s, local info:%s, len:%d, seqNum:%" PRId64 ", sid:%" PRId64 "", + CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pHead->msgType), pConn->dst, pConn->src, contLen, pConn->seq, + pReq->info.qId); + } + return code; +_end: + if (code != 0) { + tError("%s failed to send request at line %d since %s", __func__, line, tstrerror(code)); + } + return code; +} + +static int32_t evtCliSendCb(void *arg, int32_t status) { + int32_t code = status; + SFdCbArg *pArg = arg; + SCliConn *pConn = pArg->data; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + if (code != 0) { + tError("failed to send request since %s", tstrerror(code)); + return code; + } else { + tDebug("%s succ to send out request", pInst->label); + } + if (transQueueEmpty(&pConn->reqsToSend)) { + tDebug("%s conn %p stop write evt on fd:%d", transLabel(pThrd->pInst), pConn, pConn->fd); + code = evtMgtRemove(pThrd->pEvtMgt, pConn->fd, EVT_WRITE, NULL); + } + + return code; +} + +static int32_t evtCliReadResp(void *arg, SEvtBuf *buf, int32_t bytes) { + int32_t code; + int32_t line = 0; + SFdCbArg *pArg = arg; + SCliConn *pConn = pArg->data; + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + if (bytes == 0) { + tDebug("%s client %p closed", pInst->label, pConn); + return TSDB_CODE_RPC_NETWORK_ERROR; + } + SConnBuffer *p = &pConn->readBuf; + if (p->cap - p->len < bytes) { + int32_t newCap = p->cap + bytes; + char *newBuf = taosMemoryRealloc(p->buf, newCap); + if (newBuf == NULL) { + TAOS_CHECK_GOTO(terrno, &line, _end); + } + p->buf = newBuf; + p->cap = newCap; + } + + memcpy(p->buf + p->len, buf->buf, bytes); + p->len += bytes; + + while (p->len >= sizeof(STransMsgHead)) { + STransMsgHead head; + memcpy(&head, p->buf, sizeof(head)); + int32_t msgLen = (int32_t)htonl(head.msgLen); + if (p->len >= msgLen) { + char *pMsg = taosMemoryCalloc(1, msgLen); + if (pMsg == NULL) { + TAOS_CHECK_GOTO(terrno, &line, _end); + } + memcpy(pMsg, p->buf, msgLen); + memmove(p->buf, p->buf + msgLen, p->len - msgLen); + p->len -= msgLen; + + code = evtCliHandleResp(pConn, pMsg, msgLen); + TAOS_CHECK_GOTO(code, &line, _end); + } else { + break; + } + } + tDebug("%s conn %p succ to read resp", pInst->label, pConn); + return code; +_end: + if (code != 0) { + tError("%s %s conn %p failed to handle resp at line %d since %s", pInst->label, __func__, pConn, line, + tstrerror(code)); + } + return code; +} + +static FORCE_INLINE void evtLogConnMissHit(SCliConn *pConn) { + // queue set; + // QUEUE_INIT(&set); + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + pConn->heapMissHit++; + tDebug("conn %p has %d reqs, %d sentout and %d status in process, total limit:%d, switch to other conn", pConn, + transQueueSize(&pConn->reqsToSend), transQueueSize(&pConn->reqsSentOut), taosHashGetSize(pConn->pQTable), + pThrd->shareConnLimit); +} +static int32_t getOrCreateHeapCache(SHashObj *pConnHeapCache, char *key, SHeap **pHeap) { + int32_t code = 0; + size_t klen = strlen(key); + + SHeap *p = taosHashGet(pConnHeapCache, key, klen); + if (p == NULL) { + SHeap heap = {0}; + code = transHeapInit(&heap, compareHeapNode); + if (code != 0) { + tError("failed to init heap cache for key:%s since %s", key, tstrerror(code)); + return code; + } + + code = taosHashPut(pConnHeapCache, key, klen, &heap, sizeof(heap)); + if (code != 0) { + transHeapDestroy(&heap); + tError("failed to put heap to cache for key:%s since %s", key, tstrerror(code)); + return code; + } + p = taosHashGet(pConnHeapCache, key, klen); + if (p == NULL) { + code = TSDB_CODE_INVALID_PARA; + } + } + *pHeap = p; + return code; +} +static FORCE_INLINE int8_t shouldSWitchToOtherConn(SCliConn *pConn, char *key) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + + int32_t reqsNum = transQueueSize(&pConn->reqsToSend); + int32_t reqsSentOut = transQueueSize(&pConn->reqsSentOut); + int32_t stateNum = taosHashGetSize(pConn->pQTable); + int32_t totalReqs = reqsNum + reqsSentOut; + + tDebug("%s conn %p get from heap cache for key:%s, status:%d, refCnt:%d, totalReqs:%d", pInst->label, pConn, key, + pConn->inHeap, pConn->reqRefCnt, totalReqs); + + if (totalReqs >= pThrd->shareConnLimit) { + evtLogConnMissHit(pConn); + + if (pConn->list == NULL && pConn->dstAddr != NULL) { + pConn->list = taosHashGet((SHashObj *)pThrd->pool, pConn->dstAddr, strlen(pConn->dstAddr)); + if (pConn->list != NULL) { + tTrace("conn %p get list %p from pool for key:%s", pConn, pConn->list, key); + } + } + if (pConn->list && pConn->list->totalSize >= pInst->connLimitNum / 4) { + tWarn("%s conn %p try to remove timeout msg since too many conn created", transLabel(pInst), pConn); + pThrd->shareConnLimit = pThrd->shareConnLimit * 2; + // TODO + // if (cliConnRemoveTimeoutMsg(pConn)) { + // tWarn("%s conn %p succ to remove timeout msg", transLabel(pInst), pConn); + // } + return 1; + } + // check req timeout or not + return 1; + } + + return 0; +} +static SCliConn *getConnFromHeapCache(SHashObj *pConnHeap, char *key) { + int32_t code = 0; + SCliConn *pConn = NULL; + SHeap *pHeap = NULL; + + code = getOrCreateHeapCache(pConnHeap, key, &pHeap); + if (code != 0) { + tTrace("failed to get conn heap from cache for key:%s", key); + return NULL; + } + + code = transHeapGet(pHeap, &pConn); + if (code != 0) { + tTrace("failed to get conn from heap cache for key:%s", key); + return NULL; + } else { + tTrace("conn %p get conn from heap cache for key:%s", pConn, key); + if (shouldSWitchToOtherConn(pConn, key)) { + SCliConn *pNewConn = NULL; + code = balanceConnHeapCache(pConnHeap, pConn, &pNewConn); + if (code == 1) { + tTrace("conn %p start to handle reqs", pNewConn); + return pNewConn; + } + return NULL; + } + } + + return pConn; +} +static int32_t addConnToHeapCache(SHashObj *pConnHeapCacahe, SCliConn *pConn) { + SHeap *p = NULL; + int32_t code = 0; + + if (pConn->heap != NULL) { + p = pConn->heap; + tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d, add direct", pConn, pConn->dstAddr, + pConn->inHeap, pConn->reqRefCnt); + } else { + code = getOrCreateHeapCache(pConnHeapCacahe, pConn->dstAddr, &p); + if (code != 0) { + return code; + } + if (pConn->connnected == 0) { + int64_t now = taosGetTimestampMs(); + if (now - p->lastConnFailTs < 3000) { + return TSDB_CODE_RPC_NETWORK_UNAVAIL; + } + } + } + + code = transHeapInsert(p, pConn); + tTrace("conn %p add to heap cache for key:%s,status:%d, refCnt:%d", pConn, pConn->dstAddr, pConn->inHeap, + pConn->reqRefCnt); + return code; +} + +static int32_t delConnFromHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn) { + if (pConn->heap != NULL) { + tTrace("conn %p try to delete from heap cache direct", pConn); + return transHeapDelete(pConn->heap, pConn); + } + + SHeap *p = taosHashGet(pConnHeapCache, pConn->dstAddr, strlen(pConn->dstAddr)); + if (p == NULL) { + tTrace("failed to get heap cache for key:%s, no need to del", pConn->dstAddr); + return 0; + } + int32_t code = transHeapDelete(p, pConn); + if (code != 0) { + tTrace("conn %p failed delete from heap cache since %s", pConn, tstrerror(code)); + } + return code; +} + +static int8_t balanceConnHeapCache(SHashObj *pConnHeapCache, SCliConn *pConn, SCliConn **pNewConn) { + SCliThrd2 *pThrd = pConn->hostThrd; + STrans *pInst = pThrd->pInst; + SCliConn *pTopConn = NULL; + if (pConn->heap != NULL && pConn->inHeap != 0) { + TAOS_UNUSED(transHeapBalance(pConn->heap, pConn)); + if (transHeapGet(pConn->heap, &pTopConn) == 0 && pConn != pTopConn) { + int32_t curReqs = REQS_ON_CONN(pConn); + int32_t topReqs = REQS_ON_CONN(pTopConn); + if (curReqs > topReqs && topReqs < pThrd->shareConnLimit) { + *pNewConn = pTopConn; + return 1; + } + } + } + return 0; +} +FORCE_INLINE int32_t evtCliBuildExceptResp(SCliThrd2 *pThrd, SCliReq *pReq, STransMsg *pResp) { + if (pReq == NULL) return -1; + + STrans *pInst = pThrd->pInst; + + SReqCtx *pCtx = pReq ? pReq->ctx : NULL; + pResp->msgType = pReq ? pReq->msg.msgType + 1 : 0; + pResp->info.cliVer = pInst->compatibilityVer; + pResp->info.ahandle = pCtx ? pCtx->ahandle : 0; + if (pReq) { + pResp->info.traceId = pReq->msg.info.traceId; + } + + // handle noresp and inter manage msg + if (pCtx == NULL || pReq->msg.info.noResp == 1) { + return TSDB_CODE_RPC_NO_STATE; + } + if (pResp->code == 0) { + pResp->code = TSDB_CODE_RPC_BROKEN_LINK; + } + + return 0; +} +static int32_t evtCliHandleExcept(void *thrd, SCliReq *pReq, STransMsg *pResp) { + SCliThrd2 *pThrd = thrd; + STrans *pInst = pThrd->pInst; + int32_t code = evtCliBuildExceptResp(pThrd, pReq, pResp); + if (code != 0) { + destroyReq(pReq); + return code; + } + pInst->cfp(pInst->parent, pResp, NULL); + destroyReq(pReq); + return code; +} + +static int32_t evtCliMayGetStateByQid(SCliThrd2 *pThrd, SCliReq *pReq, SCliConn **pConn) { + int32_t code = 0; + int64_t qid = pReq->msg.info.qId; + if (qid == 0) { + return TSDB_CODE_RPC_NO_STATE; + } else { + SExHandle *exh = transAcquireExHandle(transGetRefMgt(), qid); + if (exh == NULL) { + return TSDB_CODE_RPC_STATE_DROPED; + } + + SReqState *pState = taosHashGet(pThrd->pIdConnTable, &qid, sizeof(qid)); + + if (pState == NULL) { + if (pReq->ctx == NULL) { + transReleaseExHandle(transGetRefMgt(), qid); + return TSDB_CODE_RPC_STATE_DROPED; + } + tDebug("%s conn %p failed to get statue, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + transReleaseExHandle(transGetRefMgt(), qid); + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } else { + *pConn = pState->conn; + tDebug("%s conn %p succ to get conn of statue, sid:%" PRId64 "", transLabel(pThrd->pInst), pConn, qid); + } + transReleaseExHandle(transGetRefMgt(), qid); + return 0; + } +} +static int32_t evtHandleCliReq(SCliThrd2 *pThrd, SCliReq *req) { + int32_t lino = 0; + int32_t code = 0; + STransMsg resp = {0}; + STrans *pInst = pThrd->pInst; + + STraceId *trace = &req->msg.info.traceId; + SCliConn *pConn = NULL; + + code = evtCliMayGetStateByQid(pThrd, req, &pConn); + if (code == 0) { + cliHandleStateMayUpdateStateCtx(pConn, req); + } else if (code == TSDB_CODE_RPC_STATE_DROPED) { + TAOS_CHECK_GOTO(code, &lino, _end); + } else if (code == TSDB_CODE_RPC_NO_STATE || code == TSDB_CODE_RPC_ASYNC_IN_PROCESS) { + char *fqdn = EPSET_GET_INUSE_IP(req->ctx->epSet); + int32_t port = EPSET_GET_INUSE_PORT(req->ctx->epSet); + char addr[TSDB_FQDN_LEN + 64] = {0}; + snprintf(addr, sizeof(addr), "%s:%d", fqdn, port); + + code = getOrCreateConn(pThrd, fqdn, port, &pConn); + if (code != 0) { + tError("%s failed to create conn since %s", pInst->label, tstrerror(code)); + TAOS_CHECK_GOTO(code, &lino, _end); + } else { + } + cliHandleStateMayUpdateState(pConn, req); + } + + transQueuePush(&pConn->reqsToSend, &req->q); + tGDebug("%s conn %p get from pool, src:%s, dst:%s", pInst->label, pConn, pConn->src, pConn->dst); + + SFdCbArg arg = {.evtType = EVT_CONN_T, + .arg = pConn, + .fd = pConn->fd, + .readCb = evtCliReadResp, + .sendCb = evtCliPreSendReq, + .sendFinishCb = evtCliSendCb, + .data = pConn}; + + code = evtMgtAdd(pThrd->pEvtMgt, pConn->fd, EVT_READ | EVT_WRITE, &arg); + + return code; +_end: + resp.code = code; + if (code != 0) { + tGError("%s failed to handle req since %s", pInst->label, tstrerror(code)); + } + evtCliHandleExcept(pThrd, req, &resp); + return code; +} +static void evtCliHandleAsyncCb(void *arg, int32_t status) { + int32_t code = 0; + SAsyncHandle *handle = arg; + SEvtMgt *pEvtMgt = handle->data; + + SCliThrd2 *pThrd = pEvtMgt->hostThrd; + STrans *pInst = pThrd->pInst; + + queue wq; + QUEUE_INIT(&wq); + + taosThreadMutexLock(&handle->mutex); + QUEUE_MOVE(&handle->q, &wq); + taosThreadMutexUnlock(&handle->mutex); + + if (QUEUE_IS_EMPTY(&wq)) { + return; + } + while (!QUEUE_IS_EMPTY(&wq)) { + queue *el = QUEUE_HEAD(&wq); + QUEUE_REMOVE(el); + + SCliReq *pReq = QUEUE_DATA(el, SCliReq, q); + if (pReq == NULL) { + continue; + } + if (pReq->type == Quit) { + transCliAsyncFuncs[pReq->type](pReq, pThrd); + break; + } + transCliAsyncFuncs[pReq->type](pReq, pThrd); + } +} + +static int32_t evtCliTimeoutFunc(void *arg) { + if (arg == 0) return 0; + + int32_t code = 0; + STimeoutArg *pArg = arg; + transDQHandleTimeout(pArg->queue); + return code; +} +static int32_t evtCliCalcTimeout(void *arg, int64_t *timeout) { + if (arg == 0) return 0; + STimeoutArg *pArg = arg; + int32_t code = 0; + + *timeout = transDQGetNextTimeout(pArg->queue); + return code; +} +static void *cliWorkThread2(void *arg) { + int32_t line = 0; + int32_t code = 0; + char threadName[TSDB_LABEL_LEN] = {0}; + SCliThrd2 *pThrd = (SCliThrd2 *)arg; + STrans *pInst = pThrd->pInst; + + pThrd->pid = taosGetSelfPthreadId(); + + tsEnableRandErr = false; + TAOS_UNUSED(strtolower(threadName, pThrd->pInst->label)); + setThreadName(threadName); + + code = evtMgtCreate(&pThrd->pEvtMgt, pInst->label); + TAOS_CHECK_GOTO(code, &line, _end); + + pThrd->pEvtMgt->hostThrd = pThrd; + + code = evtAsyncInit(pThrd->pEvtMgt, pThrd->pipe_queue_fd, &pThrd->asyncHandle, evtCliHandleAsyncCb, EVT_ASYNC_T, + (void *)pThrd); + TAOS_CHECK_GOTO(code, &line, _end); + + createTimeoutArg(pThrd->pEvtMgt, (STimeoutArg **)&pThrd->pEvtMgt->arg); + TAOS_CHECK_GOTO(code, &line, _end); + + code = evtMgtAddTimoutFunc(pThrd->pEvtMgt, NULL, evtCliTimeoutFunc, evtCliCalcTimeout); + + while (!pThrd->quit) { + struct timeval tv = {30, 0}; + code = evtCaclNextTimeout(pThrd->pEvtMgt, &tv); + if (code != 0) { + tTrace("%s failed to calc next timeout", pInst->label); + } else { + tTrace("%s succ to calc next timeout", pInst->label); + } + code = evtMgtDispath(pThrd->pEvtMgt, &tv); + if (code != 0) { + tError("%s failed to dispatch since %s", pInst->label, tstrerror(code)); + continue; + } else { + tTrace("%s succe to dispatch", pInst->label); + } + } + + tDebug("%s thread quit-thread:%08" PRId64 "", pInst->label, pThrd->pid); + return NULL; +_end: + if (code != 0) { + tError("%s failed to do work %s", pInst->label, tstrerror(code)); + } + return NULL; +} +void *transInitClient2(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *arg) { + int32_t code = 0; + int32_t lino = 0; + SCliObj2 *cli = taosMemoryCalloc(1, sizeof(SCliObj2)); + if (cli == NULL) { + TAOS_CHECK_EXIT(terrno); + } + + STrans *pInst = arg; + if (pInst->connLimitNum >= 512) { + pInst->connLimitNum = 512; + } + if (pInst->shareConnLimit <= 0) { + pInst->shareConnLimit = 64; + } + + memcpy(cli->label, label, TSDB_LABEL_LEN); + cli->numOfThreads = numOfThreads; + cli->pThreadObj = (SCliThrd2 **)taosMemoryCalloc(cli->numOfThreads, sizeof(SCliThrd2 *)); + if (cli->pThreadObj == NULL) { + TAOS_CHECK_EXIT(terrno); + } + + for (int i = 0; i < cli->numOfThreads; i++) { + SCliThrd2 *pThrd = NULL; + code = createThrdObj(arg, &pThrd); + TAOS_CHECK_EXIT(code); + pThrd->pInst = pInst; + pThrd->shareConnLimit = pInst->shareConnLimit; + code = evtInitPipe(pThrd->pipe_queue_fd); + TAOS_CHECK_EXIT(code); + + code = taosThreadCreate(&pThrd->thread, NULL, cliWorkThread2, (void *)(pThrd)); + TAOS_CHECK_EXIT(code); + pThrd->thrdInited = 1; + cli->pThreadObj[i] = pThrd; + } + + return cli; + +_exit: + if (code != 0) { + tError("%s failed to init client since %s", pInst->label, tstrerror(code)); + } + if (cli) { + for (int i = 0; i < cli->numOfThreads; i++) { + // send quit msg + destroyThrdObj(cli->pThreadObj[i]); + } + taosMemoryFree(cli->pThreadObj); + taosMemoryFree(cli); + } + terrno = code; + return NULL; +} +static int32_t transSendQuit(SCliThrd2 *pThrd) { + if (pThrd->thrdInited == 0) { + return 0; + } + int32_t code = 0; + SCliReq *msg = taosMemoryCalloc(1, sizeof(SCliReq)); + if (msg == NULL) { + return terrno; + } + + msg->type = Quit; + if ((code = evtAsyncSend(pThrd->asyncHandle, &msg->q)) != 0) { + code = (code == TSDB_CODE_RPC_ASYNC_MODULE_QUIT ? TSDB_CODE_RPC_MODULE_QUIT : code); + taosMemoryFree(msg); + return code; + } + + atomic_store_8(&pThrd->asyncHandle->stop, 1); + return 0; +} +void transCloseClient2(void *arg) { + int32_t code = 0; + SCliObj2 *cli = arg; + for (int i = 0; i < cli->numOfThreads; i++) { + code = transSendQuit(cli->pThreadObj[i]); + if (code != 0) { + tError("failed to send quit to thread:%d since %s", i, tstrerror(code)); + } + + destroyThrdObj(cli->pThreadObj[i]); + } + taosMemoryFree(cli->pThreadObj); + taosMemoryFree(cli); + return; +} + +static int32_t compareHeapNode(const HeapNode *a, const HeapNode *b) { + SCliConn *args1 = container_of(a, SCliConn, node); + SCliConn *args2 = container_of(b, SCliConn, node); + + int32_t totalReq1 = REQS_ON_CONN(args1); + int32_t totalReq2 = REQS_ON_CONN(args2); + if (totalReq1 > totalReq2) { + return 0; + } + return 1; +} +static int32_t transHeapInit(SHeap *heap, int32_t (*cmpFunc)(const HeapNode *a, const HeapNode *b)) { + heap->heap = heapCreate(cmpFunc); + if (heap->heap == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + heap->cmpFunc = cmpFunc; + return 0; +} +static void transHeapDestroy(SHeap *heap) { + if (heap != NULL) { + heapDestroy(heap->heap); + } +} + +static int32_t transHeapGet(SHeap *heap, SCliConn **p) { + if (heapSize(heap->heap) == 0) { + *p = NULL; + return -1; + } + HeapNode *minNode = heapMin(heap->heap); + if (minNode == NULL) { + *p = NULL; + return -1; + } + *p = container_of(minNode, SCliConn, node); + return 0; +} +static int32_t transHeapInsert(SHeap *heap, SCliConn *p) { + int32_t code = 0; + p->reqRefCnt++; + if (p->inHeap == 1) { + tTrace("failed to insert conn %p since already in heap", p); + return TSDB_CODE_DUP_KEY; + } + + heapInsert(heap->heap, &p->node); + p->inHeap = 1; + p->lastAddHeapTime = taosGetTimestampMs(); + p->heap = heap; + return 0; +} +static int32_t transHeapDelete(SHeap *heap, SCliConn *p) { + if (p->connnected == 0) { + TAOS_UNUSED(transHeapUpdateFailTs(heap, p)); + } + + if (p->inHeap == 0) { + tTrace("failed to del conn %p since not in heap", p); + return 0; + } else { + int64_t now = taosGetTimestampMs(); + if (p->forceDelFromHeap == 0 && now - p->lastAddHeapTime < 10000) { + tTrace("conn %p not added/delete to heap frequently", p); + return TSDB_CODE_RPC_ASYNC_IN_PROCESS; + } + } + + p->inHeap = 0; + p->reqRefCnt--; + if (p->reqRefCnt == 0) { + heapRemove(heap->heap, &p->node); + tTrace("conn %p delete from heap", p); + } else if (p->reqRefCnt < 0) { + tTrace("conn %p has %d reqs, not delete from heap,assert", p, p->reqRefCnt); + } else { + tTrace("conn %p has %d reqs, not delete from heap", p, p->reqRefCnt); + } + return 0; +} +static int32_t transHeapBalance(SHeap *heap, SCliConn *p) { + if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + return 0; + } + heapRemove(heap->heap, &p->node); + heapInsert(heap->heap, &p->node); + return 0; +} +static int32_t transHeapUpdateFailTs(SHeap *heap, SCliConn *p) { + heap->lastConnFailTs = taosGetTimestampMs(); + return 0; +} +static int32_t transHeapMayBalance(SHeap *heap, SCliConn *p) { + if (p->inHeap == 0 || heap == NULL || heap->heap == NULL) { + return 0; + } + SCliThrd2 *pThrd = p->hostThrd; + STrans *pInst = pThrd->pInst; + int32_t balanceLimit = pThrd->shareConnLimit >= 4 ? pThrd->shareConnLimit / 2 : 2; + + SCliConn *topConn = NULL; + int32_t code = transHeapGet(heap, &topConn); + if (code != 0) { + return code; + } + + if (topConn == p) return code; + + int32_t reqsOnTop = REQS_ON_CONN(topConn); + int32_t reqsOnCur = REQS_ON_CONN(p); + + if (reqsOnTop >= balanceLimit && reqsOnCur < balanceLimit) { + TAOS_UNUSED(transHeapBalance(heap, p)); + } + return code; +} +static FORCE_INLINE int32_t timeCompare(const HeapNode *a, const HeapNode *b) { + SDelayTask *arg1 = container_of(a, SDelayTask, node); + SDelayTask *arg2 = container_of(b, SDelayTask, node); + if (arg1->execTime > arg2->execTime) { + return 0; + } else { + return 1; + } +} +int32_t transDQCreate(void *loop, SDelayQueue **queue) { + int32_t code = 0; + Heap *heap = NULL; + // uv_timer_t* timer = NULL; + SDelayQueue *q = NULL; + + // timer = taosMemoryCalloc(1, sizeof(uv_timer_t)); + // if (timer == NULL) { + // return terrno; + // } + + heap = heapCreate(timeCompare); + if (heap == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _return1); + } + + q = taosMemoryCalloc(1, sizeof(SDelayQueue)); + if (q == NULL) { + TAOS_CHECK_GOTO(terrno, NULL, _return1); + } + q->heap = heap; + q->mgt = loop; + + *queue = q; + return 0; + +_return1: + heapDestroy(heap); + taosMemoryFree(q); + return TSDB_CODE_OUT_OF_MEMORY; +} +void transDQDestroy(SDelayQueue *queue, void (*freeFunc)(void *arg)) { + int32_t code = 0; + while (heapSize(queue->heap) > 0) { + HeapNode *minNode = heapMin(queue->heap); + if (minNode == NULL) { + return; + } + heapRemove(queue->heap, minNode); + + SDelayTask *task = container_of(minNode, SDelayTask, node); + + STaskArg *arg = task->arg; + if (freeFunc) freeFunc(arg); + taosMemoryFree(arg); + + taosMemoryFree(task); + } + heapDestroy(queue->heap); + taosMemoryFree(queue); + return; +} +SDelayTask *transDQSched(SDelayQueue *queue, void (*func)(void *arg), void *arg, uint64_t timeoutMs) { + uint64_t now = taosGetTimestampMs(); + SDelayTask *task = taosMemoryCalloc(1, sizeof(SDelayTask)); + if (task == NULL) { + return NULL; + } + + task->func = func; + task->arg = arg; + task->execTime = now + timeoutMs; + // HeapNode *minNode = heapMin(queue->heap); + // if (minNode) { + // SDelayTask *minTask = container_of(minNode, SDelayTask, node); + // if (minTask->execTime < task->execTime) { + // timeoutMs = minTask->execTime <= now ? 0 : minTask->execTime - now; + // } + // } + + // tTrace("timer %p put task into delay queue, timeoutMs:%" PRIu64, queue->mgt, timeoutMs); + heapInsert(queue->heap, &task->node); + // TAOS_UNUSED(uv_timer_start(queue->timer, transDQTimeout, timeoutMs, 0)); + return task; +} +void transDQCancel(SDelayQueue *queue, SDelayTask *task) { + // TAOS_UNUSED(uv_timer_stop(queue->timer)); + + if (heapSize(queue->heap) <= 0) { + taosMemoryFree(task->arg); + taosMemoryFree(task); + return; + } + heapRemove(queue->heap, &task->node); + + taosMemoryFree(task->arg); + taosMemoryFree(task); + + // if (heapSize(queue->heap) != 0) { + // HeapNode *minNode = heapMin(queue->heap); + // if (minNode == NULL) return; + + // uint64_t now = taosGetTimestampMs(); + // SDelayTask *task = container_of(minNode, SDelayTask, node); + // uint64_t timeout = now > task->execTime ? now - task->execTime : 0; + + // } +} + +int32_t transDQHandleTimeout(SDelayQueue *queue) { + int32_t code = 0; + uint64_t now = taosGetTimestampMs(); + while (heapSize(queue->heap) > 0) { + HeapNode *minNode = heapMin(queue->heap); + if (minNode == NULL) { + return 0; + } + SDelayTask *task = container_of(minNode, SDelayTask, node); + if (task->execTime > now) { + break; + } + heapRemove(queue->heap, minNode); + task->func(task->arg); + taosMemoryFree(task); + } + return code; +} +int32_t transDQGetNextTimeout(SDelayQueue *queue) { + if (heapSize(queue->heap) <= 0) { + return -1; + } + HeapNode *minNode = heapMin(queue->heap); + if (minNode == NULL) { + return -1; + } + SDelayTask *task = container_of(minNode, SDelayTask, node); + uint64_t now = taosGetTimestampMs(); + return task->execTime > now ? task->execTime - now : 0; +} diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index ecbdd0db8426..456b9a61d3a0 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -13,9 +13,11 @@ */ #include "transComm.h" +#include "tversion.h" static TdThreadOnce transModuleInit = PTHREAD_ONCE_INIT; +#ifndef TD_ACORE static char* notify = "a"; typedef struct { @@ -1489,6 +1491,13 @@ static void uvPipeListenCb(uv_stream_t* handle, int status) { srv->numOfWorkerReady++; } +#ifdef TD_ACORE +void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* shandle) { + int32_t code = 0; + return NULL; +} +#else + void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, void* fp, void* pInit) { int32_t code = 0; @@ -1692,6 +1701,7 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, terrno = code; return NULL; } +#endif void uvHandleQuit(SSvrRespMsg* msg, SWorkThrd* thrd) { thrd->quit = true; @@ -1808,39 +1818,6 @@ void sendQuitToWorkThrd(SWorkThrd* pThrd) { TAOS_UNUSED(transAsyncSend(pThrd->asyncPool, &msg->q)); } -void transCloseServer(void* arg) { - // impl later - SServerObj* srv = arg; - - if (srv->inited) { - tDebug("send quit msg to accept thread"); - TAOS_UNUSED(uv_async_send(srv->pAcceptAsync)); - if (taosThreadJoin(srv->thread, NULL) != 0) { - tError("failed to join accept-thread"); - } - - SRV_RELEASE_UV(srv->loop); - for (int i = 0; i < srv->numOfThreads; i++) { - destroyWorkThrd(srv->pThreadObj[i]); - } - } else { - SRV_RELEASE_UV(srv->loop); - } - - taosMemoryFree(srv->pThreadObj); - taosMemoryFree(srv->pAcceptAsync); - taosMemoryFree(srv->loop); - - for (int i = 0; i < srv->numOfThreads; i++) { - if (srv->pipe[i] != NULL) { - taosMemoryFree(srv->pipe[i]); - } - } - taosMemoryFree(srv->pipe); - - taosMemoryFree(srv); -} - void transRefSrvHandle(void* handle) { if (handle == NULL) { return; @@ -2059,3 +2036,69 @@ int32_t transSetIpWhiteList(void* thandle, void* arg, FilteFunc* func) { } return code; } + +int32_t transGetConnInfo(void* thandle, STransHandleInfo* pConnInfo) { return -1; } + +void transCloseServer(void* arg) { + // impl later + SServerObj* srv = arg; + + if (srv->inited) { + tDebug("send quit msg to accept thread"); + TAOS_UNUSED(uv_async_send(srv->pAcceptAsync)); + if (taosThreadJoin(srv->thread, NULL) != 0) { + tError("failed to join accept-thread"); + } + + SRV_RELEASE_UV(srv->loop); + for (int i = 0; i < srv->numOfThreads; i++) { + destroyWorkThrd(srv->pThreadObj[i]); + } + } else { + SRV_RELEASE_UV(srv->loop); + } + + taosMemoryFree(srv->pThreadObj); + taosMemoryFree(srv->pAcceptAsync); + taosMemoryFree(srv->loop); + + for (int i = 0; i < srv->numOfThreads; i++) { + if (srv->pipe[i] != NULL) { + taosMemoryFree(srv->pipe[i]); + } + } + taosMemoryFree(srv->pipe); + + taosMemoryFree(srv); +} +#else + +int32_t transReleaseSrvHandle(void *handle) { return 0; } +void transRefSrvHandle(void *handle) { return; } + +void transUnrefSrvHandle(void *handle) { return; } +int32_t transSendResponse(STransMsg *msg) { + int32_t code = 0; + if (rpcIsReq(msg->info.msgType) && msg->info.msgType != 0) { + msg->msgType = msg->info.msgType + 1; + } + if (msg->info.noResp) { + rpcFreeCont(msg->pCont); + return 0; + } + int32_t svrVer = 0; + code = taosVersionStrToInt(td_version, &svrVer); + msg->info.cliVer = svrVer; + msg->type = msg->info.connType; + return transSendResp(msg); +} +int32_t transRegisterMsg(const STransMsg *msg) { return 0; } +int32_t transSetIpWhiteList(void *thandle, void *arg, FilteFunc *func) { return 0; } + +void *transInitServer(uint32_t ip, uint32_t port, char *label, int numOfThreads, void *fp, void *pInit) { return NULL; } +void transCloseServer(void *arg) { + // impl later + return; +} + +#endif diff --git a/source/libs/transport/test/CMakeLists.txt b/source/libs/transport/test/CMakeLists.txt index f30dfc56a2de..34c906295e44 100644 --- a/source/libs/transport/test/CMakeLists.txt +++ b/source/libs/transport/test/CMakeLists.txt @@ -4,6 +4,7 @@ add_executable(transUT2 "") add_executable(svrBench "") add_executable(cliBench "") add_executable(httpBench "") +add_executable(transImpl2 "") target_sources(transUT PRIVATE @@ -32,6 +33,13 @@ target_sources(httpBench "http_test.c" ) +target_sources(transImpl2 + PRIVATE + "transImpl2_test.cpp" +) + + + target_include_directories(transportTest PUBLIC "${TD_SOURCE_DIR}/include/libs/transport" @@ -100,6 +108,20 @@ target_include_directories(httpBench "${TD_SOURCE_DIR}/include/libs/transport" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(transImpl2 + PUBLIC + "${TD_SOURCE_DIR}/include/libs/transport" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) + +target_link_libraries(transImpl2 + os + util + common + gtest_main + transport + function +) target_link_libraries(cliBench os diff --git a/source/libs/transport/test/svrBench.c b/source/libs/transport/test/svrBench.c index 44299d86a3af..760bd8b33ef5 100644 --- a/source/libs/transport/test/svrBench.c +++ b/source/libs/transport/test/svrBench.c @@ -41,7 +41,7 @@ typedef struct TThread { int idx; } TThread; -MultiThreadQhandle *multiQ = NULL; +MultiThreadQhandle *tMultiQ = NULL; void initLogEnv() { const char *logDir = "/tmp/trans_svr"; @@ -70,7 +70,7 @@ void *processShellMsg(void *arg) { taosAllocateQall(&qall); while (1) { - int numOfMsgs = taosReadAllQitemsFromQset(multiQ->qset[idx], qall, &qinfo); + int numOfMsgs = taosReadAllQitemsFromQset(tMultiQ->qset[idx], qall, &qinfo); tDebug("%d shell msgs are received", numOfMsgs); if (numOfMsgs <= 0) break; @@ -106,11 +106,11 @@ void processRequestMsg(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet) { taosAllocateQitem(sizeof(SRpcMsg), DEF_QITEM, 0, (void **)&pTemp); memcpy(pTemp, pMsg, sizeof(SRpcMsg)); - int32_t idx = balance % multiQ->numOfThread; + int32_t idx = balance % tMultiQ->numOfThread; tDebug("request is received, type:%d, contLen:%d, item:%p", pMsg->msgType, pMsg->contLen, pTemp); - taosWriteQitem(multiQ->qhandle[idx], pTemp); + taosWriteQitem(tMultiQ->qhandle[idx], pTemp); balance++; - if (balance >= multiQ->numOfThread) balance = 0; + if (balance >= tMultiQ->numOfThread) balance = 0; } int main(int argc, char *argv[]) { @@ -181,15 +181,15 @@ int main(int argc, char *argv[]) { } int32_t numOfAthread = 1; - multiQ = taosMemoryMalloc(sizeof(MultiThreadQhandle)); - multiQ->numOfThread = numOfAthread; - multiQ->qhandle = (STaosQueue **)taosMemoryMalloc(sizeof(STaosQueue *) * numOfAthread); - multiQ->qset = (STaosQset **)taosMemoryMalloc(sizeof(STaosQset *) * numOfAthread); + tMultiQ = taosMemoryMalloc(sizeof(MultiThreadQhandle)); + tMultiQ->numOfThread = numOfAthread; + tMultiQ->qhandle = (STaosQueue **)taosMemoryMalloc(sizeof(STaosQueue *) * numOfAthread); + tMultiQ->qset = (STaosQset **)taosMemoryMalloc(sizeof(STaosQset *) * numOfAthread); for (int i = 0; i < numOfAthread; i++) { - taosOpenQueue(&multiQ->qhandle[i]); - taosOpenQset(&multiQ->qset[i]); - taosAddIntoQset(multiQ->qset[i], multiQ->qhandle[i], NULL); + taosOpenQueue(&tMultiQ->qhandle[i]); + taosOpenQset(&tMultiQ->qset[i]); + taosAddIntoQset(tMultiQ->qset[i], tMultiQ->qhandle[i], NULL); } TThread *threads = taosMemoryMalloc(sizeof(TThread) * numOfAthread); for (int i = 0; i < numOfAthread; i++) { diff --git a/source/libs/transport/test/transImpl2_test.cpp b/source/libs/transport/test/transImpl2_test.cpp new file mode 100644 index 000000000000..bb48360beb25 --- /dev/null +++ b/source/libs/transport/test/transImpl2_test.cpp @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include +#include +#include +#include "tdatablock.h" +#include "tglobal.h" +#include "tlog.h" +#include "tmisce.h" +#include "transLog.h" +#include "trpc.h" +#include "tversion.h" +using namespace std; + +const char *label = "APP"; +const char *svrLabel = "SERVER"; +const char *cliLabel = "CLIENT"; +const char *secret = "secret"; +const char *user = "user"; +const char *ckey = "ckey"; + +class Server; +int port = 7000; +// server process +// server except + +typedef void (*CB)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); + +static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +// client process; +static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet); +class Client { + public: + void Init(int nThread) { + memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH)); + memset(&rpcInit_, 0, sizeof(rpcInit_)); + rpcInit_.localPort = 0; + rpcInit_.label = (char *)cliLabel; + rpcInit_.numOfThreads = 1; + rpcInit_.cfp = processResp; + rpcInit_.user = (char *)user; + rpcInit_.parent = this; + rpcInit_.connType = TAOS_CONN_CLIENT; + rpcInit_.shareConnLimit = 200; + + taosVersionStrToInt(td_version, &(rpcInit_.compatibilityVer)); + this->transCli = rpcOpen(&rpcInit_); + tsem_init(&this->sem, 0, 0); + } + void SetResp(SRpcMsg *pMsg) { + // set up resp; + this->resp = *pMsg; + } + SRpcMsg *Resp() { return &this->resp; } + + void Restart(CB cb) { + rpcClose(this->transCli); + rpcInit_.cfp = cb; + taosVersionStrToInt(td_version, &(rpcInit_.compatibilityVer)); + this->transCli = rpcOpen(&rpcInit_); + } + void Stop() { + rpcClose(this->transCli); + this->transCli = NULL; + } + + void SendAndRecv(SRpcMsg *req, SRpcMsg *resp) { + SEpSet epSet = {0}; + epSet.inUse = 0; + addEpIntoEpSet(&epSet, "127.0.0.1", 7000); + + rpcSendRequest(this->transCli, &epSet, req, NULL); + SemWait(); + *resp = this->resp; + } + void sendReq(SRpcMsg *req) { + SEpSet epSet = {0}; + epSet.inUse = 0; + addEpIntoEpSet(&epSet, "127.0.0.1", 7000); + + rpcSendRequest(this->transCli, &epSet, req, NULL); + } + void SendAndRecvNoHandle(SRpcMsg *req, SRpcMsg *resp) { + if (req->info.handle != NULL) { + rpcReleaseHandle(req->info.handle, TAOS_CONN_CLIENT); + req->info.handle = NULL; + } + SendAndRecv(req, resp); + } + + void SemWait() { tsem_wait(&this->sem); } + void SemPost() { tsem_post(&this->sem); } + void Reset() {} + + ~Client() { + if (this->transCli) rpcClose(this->transCli); + } + + private: + tsem_t sem; + SRpcInit rpcInit_; + void *transCli; + SRpcMsg resp; +}; +class Server { + public: + Server() { + memcpy(tsTempDir, TD_TMP_DIR_PATH, strlen(TD_TMP_DIR_PATH)); + memset(&rpcInit_, 0, sizeof(rpcInit_)); + + memcpy(rpcInit_.localFqdn, "localhost", strlen("localhost")); + rpcInit_.localPort = port; + + rpcInit_.label = (char *)svrLabel; + rpcInit_.numOfThreads = 1; + rpcInit_.cfp = processReq; + rpcInit_.user = (char *)user; + rpcInit_.connType = TAOS_CONN_SERVER; + taosVersionStrToInt(td_version, &(rpcInit_.compatibilityVer)); + } + void Start() { + this->transSrv = rpcOpen(&this->rpcInit_); + taosMsleep(1000); + } + void SetSrvContinueSend(CB cb) { + this->Stop(); + rpcInit_.cfp = cb; + this->Start(); + } + void Stop() { + if (this->transSrv == NULL) return; + rpcClose(this->transSrv); + this->transSrv = NULL; + } + void SetSrvSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { + this->Stop(); + rpcInit_.cfp = cfp; + this->Start(); + } + void Restart() { + this->Stop(); + this->Start(); + } + ~Server() { + if (this->transSrv) rpcClose(this->transSrv); + this->transSrv = NULL; + } + + private: + SRpcInit rpcInit_; + void *transSrv; +}; +static void processReq(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.info = pMsg->info; + rpcMsg.code = 0; + rpcFreeCont(pMsg->pCont); + rpcSendResponse(&rpcMsg); +} + +static void processContinueSend(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + for (int i = 0; i < 10; i++) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.info = pMsg->info; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + } +} +static void processReleaseHandleCb(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.info = pMsg->info; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); + + rpcReleaseHandle(&pMsg->info, TAOS_CONN_SERVER); +} +static void processRegisterFailure(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + { + SRpcMsg rpcMsg1 = {0}; + rpcMsg1.pCont = rpcMallocCont(100); + rpcMsg1.contLen = 100; + rpcMsg1.info = pMsg->info; + rpcMsg1.code = 0; + rpcRegisterBrokenLinkArg(&rpcMsg1); + } + taosMsleep(10); + + SRpcMsg rpcMsg = {0}; + rpcMsg.pCont = rpcMallocCont(100); + rpcMsg.contLen = 100; + rpcMsg.info = pMsg->info; + rpcMsg.code = 0; + rpcSendResponse(&rpcMsg); +} +// client process; +static void processResp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { + Client *client = (Client *)parent; + client->SetResp(pMsg); + client->SemPost(); + tDebug("received resp"); +} + +static void initEnv() { + dDebugFlag = 143; + vDebugFlag = 0; + mDebugFlag = 143; + cDebugFlag = 0; + jniDebugFlag = 0; + tmrDebugFlag = 143; + uDebugFlag = 143; + rpcDebugFlag = 143; + qDebugFlag = 0; + wDebugFlag = 0; + sDebugFlag = 0; + tsdbDebugFlag = 0; + tsLogEmbedded = 1; + tsAsyncLog = 0; + + std::string path = TD_TMP_DIR_PATH "transport"; + // taosRemoveDir(path.c_str()); + taosMkDir(path.c_str()); + + tstrncpy(tsLogDir, path.c_str(), PATH_MAX); + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { + printf("failed to init log file\n"); + } +} + +class TransObj { + public: + TransObj() { + initEnv(); + cli = new Client; + cli->Init(1); + srv = new Server; + srv->Start(); + } + + void RestartCli(CB cb) { + // + cli->Restart(cb); + } + void StopSrv() { + // + srv->Stop(); + } + // call when link broken, and notify query or fetch stop + void SetSrvContinueSend(void (*cfp)(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet)) { + /////// + srv->SetSrvContinueSend(cfp); + } + void RestartSrv() { srv->Restart(); } + void StopCli() { + /////// + cli->Stop(); + } + void cliSendAndRecv(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecv(req, resp); } + void cliSendReq(SRpcMsg *req) { cli->sendReq(req); } + void cliSendAndRecvNoHandle(SRpcMsg *req, SRpcMsg *resp) { cli->SendAndRecvNoHandle(req, resp); } + + ~TransObj() { + delete cli; + delete srv; + } + + private: + Client *cli; + Server *srv; +}; +class TransEnv : public ::testing::Test { + protected: + virtual void SetUp() { + // set up trans obj + tr = new TransObj(); + } + virtual void TearDown() { + // tear down + delete tr; + } + + TransObj *tr = NULL; +}; + +TEST_F(TransEnv, 01sendAndReq) { + for (int i = 0; i < 10000; i++) { + SRpcMsg req = {0}, resp = {0}; + req.msgType = 1; + req.pCont = rpcMallocCont(10); + req.contLen = 10; + tr->cliSendReq(&req); + //taosMsleep(100000); + assert(resp.code == 0); + } + taosMsleep(20000); +} diff --git a/source/libs/transport/test/transUT.cpp b/source/libs/transport/test/transUT.cpp index 8e396d59d774..92a09f821c1b 100644 --- a/source/libs/transport/test/transUT.cpp +++ b/source/libs/transport/test/transUT.cpp @@ -240,7 +240,7 @@ static void initEnv() { taosMkDir(path.c_str()); tstrncpy(tsLogDir, path.c_str(), PATH_MAX); - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_TAOSD) != 0) { printf("failed to init log file\n"); } } diff --git a/source/libs/transport/test/transUT2.cpp b/source/libs/transport/test/transUT2.cpp index 54d23b1f64fa..262cd1089bc1 100644 --- a/source/libs/transport/test/transUT2.cpp +++ b/source/libs/transport/test/transUT2.cpp @@ -250,7 +250,7 @@ static void initEnv() { taosMkDir(path.c_str()); tstrncpy(tsLogDir, path.c_str(), PATH_MAX); - if (taosInitLog("taosdlog", 1, false) != 0) { + if (taosInitLog("taosdlog", 1, LOG_MODE_BOTH ) != 0) { printf("failed to init log file\n"); } } diff --git a/source/os/src/osString.c b/source/os/src/osString.c index 0ee4f1c496a9..2c5d0f1225c4 100644 --- a/source/os/src/osString.c +++ b/source/os/src/osString.c @@ -16,6 +16,7 @@ #define ALLOW_FORBID_FUNC #define _DEFAULT_SOURCE #include "os.h" +#include "tutil.h" #ifndef DISALLOW_NCHAR_WITHOUT_ICONV #include "iconv.h" @@ -229,6 +230,7 @@ int32_t tasoUcs4Copy(TdUcs4 *target_ucs4, TdUcs4 *source_ucs4, int32_t len_ucs4) return TSDB_CODE_SUCCESS; } +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV typedef struct { iconv_t conv; int8_t inUse; @@ -236,45 +238,74 @@ typedef struct { // 0: Mbs --> Ucs4 // 1: Ucs4--> Mbs -SConv *gConv[2] = {NULL, NULL}; -int32_t convUsed[2] = {0, 0}; -int32_t gConvMaxNum[2] = {0, 0}; - +static SConv *gConv[2] = {NULL, NULL}; +static int32_t convUsed[2] = {0, 0}; +static int32_t gConvMaxNum[2] = {0, 0}; +static int8_t convInited = 0; +#endif int32_t taosConvInit(void) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV + int32_t code = 0, lino = 0; + int8_t old; + int32_t nLoops = 0; + while (1) { + old = atomic_val_compare_exchange_8(&convInited, 0, 2); + if (old != 2) break; + if (++nLoops > 1000) { + TAOS_UNUSED(sched_yield()); + nLoops = 0; + } + } + if (old == 1) { + goto _exit; + } + int8_t M2C = 0; gConvMaxNum[M2C] = 512; gConvMaxNum[1 - M2C] = 512; gConv[M2C] = taosMemoryCalloc(gConvMaxNum[M2C], sizeof(SConv)); if (gConv[M2C] == NULL) { - return terrno; + atomic_store_8(&convInited, 0); + TAOS_CHECK_EXIT(terrno); } gConv[1 - M2C] = taosMemoryCalloc(gConvMaxNum[1 - M2C], sizeof(SConv)); if (gConv[1 - M2C] == NULL) { taosMemoryFree(gConv[M2C]); - return terrno; + atomic_store_8(&convInited, 0); + TAOS_CHECK_EXIT(terrno); } for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { gConv[M2C][i].conv = iconv_open(DEFAULT_UNICODE_ENCODEC, tsCharset); if ((iconv_t)-1 == gConv[M2C][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + code = TAOS_SYSTEM_ERROR(errno); + atomic_store_8(&convInited, 1); + TAOS_CHECK_EXIT(code); } } for (int32_t i = 0; i < gConvMaxNum[1 - M2C]; ++i) { gConv[1 - M2C][i].conv = iconv_open(tsCharset, DEFAULT_UNICODE_ENCODEC); if ((iconv_t)-1 == gConv[1 - M2C][i].conv) { - terrno = TAOS_SYSTEM_ERROR(errno); - return terrno; + code = TAOS_SYSTEM_ERROR(errno); + atomic_store_8(&convInited, 1); + TAOS_CHECK_EXIT(code); } } + atomic_store_8(&convInited, 1); +_exit: + if (code) { + uError("failed to init taosConvInit at line %d since %s", lino, tstrerror(code)); + } +#endif return 0; } + void taosConvDestroy() { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV int8_t M2C = 0; for (int32_t i = 0; i < gConvMaxNum[M2C]; ++i) { (void)iconv_close(gConv[M2C][i].conv); @@ -286,9 +317,11 @@ void taosConvDestroy() { taosMemoryFreeClear(gConv[1 - M2C]); gConvMaxNum[M2C] = -1; gConvMaxNum[1 - M2C] = -1; +#endif } iconv_t taosAcquireConv(int32_t *idx, ConvType type) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if(idx == NULL) { terrno = TSDB_CODE_INVALID_PARA; return (iconv_t)-1; @@ -340,9 +373,14 @@ iconv_t taosAcquireConv(int32_t *idx, ConvType type) { } else { return gConv[type][startId].conv; } +#else + terrno = TSDB_CODE_APP_ERROR; + return (iconv_t)-1; +#endif } void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { +#ifndef DISALLOW_NCHAR_WITHOUT_ICONV if (idx < 0) { (void)iconv_close(conv); return; @@ -350,6 +388,7 @@ void taosReleaseConv(int32_t idx, iconv_t conv, ConvType type) { atomic_store_8(&gConv[type][idx].inUse, 0); (void)atomic_sub_fetch_32(&convUsed[type], 1); + #endif } bool taosMbsToUcs4(const char *mbs, size_t mbsLength, TdUcs4 *ucs4, int32_t ucs4_max_len, int32_t *len) { diff --git a/source/util/src/tcompare.c b/source/util/src/tcompare.c index ebc379897f5b..d1f08dc59b79 100644 --- a/source/util/src/tcompare.c +++ b/source/util/src/tcompare.c @@ -1211,11 +1211,12 @@ typedef struct UsingRegex { typedef UsingRegex* HashRegexPtr; typedef struct RegexCache { - SHashObj *regexHash; - void *regexCacheTmr; - void *timer; - SRWLatch mutex; - bool exit; + SHashObj *regexHash; + void *regexCacheTmr; + void *timer; + SRWLatch mutex; + bool exit; + int8_t inited; } RegexCache; static RegexCache sRegexCache; #define MAX_REGEX_CACHE_SIZE 20 @@ -1260,9 +1261,10 @@ void regexCacheFree(void *ppUsingRegex) { } int32_t InitRegexCache() { - #ifdef WINDOWS - return 0; - #endif +#ifdef WINDOWS + return 0; +#endif + if (atomic_val_compare_exchange_8(&sRegexCache.inited, 0, 1) != 0) return TSDB_CODE_SUCCESS; sRegexCache.regexHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_ENTRY_LOCK); if (sRegexCache.regexHash == NULL) { uError("failed to create RegexCache"); @@ -1299,7 +1301,9 @@ void DestroyRegexCache(){ taosWLockLatch(&sRegexCache.mutex); sRegexCache.exit = true; taosHashCleanup(sRegexCache.regexHash); + sRegexCache.regexHash = NULL; taosTmrCleanUp(sRegexCache.regexCacheTmr); + sRegexCache.regexCacheTmr = NULL; taosWUnLockLatch(&sRegexCache.mutex); } diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 693935575aeb..e314ac448a44 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -296,7 +296,7 @@ int32_t taosInitLogOutput(const char **ppLogName) { return 0; } -int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { +int32_t taosInitLog(const char *logName, int32_t maxFiles, ELogMode mode) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; int32_t code = osUpdate(); if (code != 0) { @@ -309,7 +309,7 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles, bool tsc) { } TAOS_CHECK_RETURN(taosInitNormalLog(logName, maxFiles)); - if (tsc) { + if (mode & LOG_MODE_TAOSC) { TAOS_CHECK_RETURN(taosInitSlowLog()); } TAOS_CHECK_RETURN(taosStartLog()); @@ -749,6 +749,8 @@ static inline void taosPrintLogImp(ELogLevel level, int32_t dflag, const char *b } } + if (!tsLogEmbedded) return; + int fd = 0; if (tsLogObj.outputType == LOG_OUTPUT_FILE) { if (dflag & DEBUG_SCREEN) fd = 1; diff --git a/source/util/test/log.cpp b/source/util/test/log.cpp index ba32d2d63949..4f254d20dcd2 100644 --- a/source/util/test/log.cpp +++ b/source/util/test/log.cpp @@ -15,7 +15,7 @@ TEST(log, check_log_refactor) { tsAsyncLog = 0; // idxDebugFlag = 143; strcpy(tsLogDir, (char *)logDir); - taosInitLog(tsLogDir, 10, false); + taosInitLog(tsLogDir, 10, LOG_MODE_BOTH); tsAsyncLog = 0; uDebugFlag = 143; diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 7abdef7c78e8..6734d5605c44 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -154,7 +154,7 @@ int main(int argc, char *argv[]) { } } - taosInitLog("tref.log", 10, false); + taosInitLog("tref.log", 10, LOG_MODE_TAOSD); SRefSpace *pSpaceList = (SRefSpace *)taosMemoryCalloc(sizeof(SRefSpace), threads); TdThread *pThreadList = (TdThread *)taosMemoryCalloc(sizeof(TdThread), threads); diff --git a/tools/shell/CMakeLists.txt b/tools/shell/CMakeLists.txt index 2301f3380326..2652a326df55 100644 --- a/tools/shell/CMakeLists.txt +++ b/tools/shell/CMakeLists.txt @@ -26,6 +26,10 @@ ELSE() SET(LINK_WEBSOCKET "") ENDIF() +if(${TD_ACORE}) + add_definitions(-DTD_ACORE) +endif(${TD_ACORE}) + IF(TD_LINUX AND TD_ALPINE) SET(LINK_ARGP "/usr/lib/libargp.a") ELSE() @@ -38,10 +42,17 @@ else() target_link_libraries(shell PUBLIC ${TAOS_LIB} ${LINK_WEBSOCKET} ${LINK_JEMALLOC} ${LINK_ARGP}) endif() -target_link_libraries( - shell - PRIVATE os common transport geometry util -) +if(${TD_ACORE}) + target_link_libraries( + shell + PRIVATE os common transport geometry util taosd + ) +else() + target_link_libraries( + shell + PRIVATE os common transport geometry util + ) +endif() target_include_directories( shell diff --git a/tools/shell/inc/shellInt.h b/tools/shell/inc/shellInt.h index b1f09d5161a0..aa04bdd33959 100644 --- a/tools/shell/inc/shellInt.h +++ b/tools/shell/inc/shellInt.h @@ -112,6 +112,13 @@ typedef struct { #endif } SShellObj; +#ifdef TD_ACORE +typedef struct { + TdThread pid; + int32_t stat; // < 0: start failed, 0: init(not start), 1: start successfully +} SDaemonObj; +#endif + typedef struct { char *buffer; char *command; @@ -129,7 +136,7 @@ int32_t shellParseArgs(int32_t argc, char* argv[]); int32_t shellReadCommand(char* command); // shellEngine.c -int32_t shellExecute(); +int32_t shellExecute(int argc, char *argv[]); int32_t shellCalcColWidth(TAOS_FIELD *field, int32_t precision); void shellPrintHeader(TAOS_FIELD *fields, int32_t *width, int32_t num_fields); void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t length, int32_t precision); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 6fcebd667d7c..7c1f70c0b7ce 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -22,6 +22,13 @@ #include "shellAuto.h" #include "shellInt.h" +#ifdef TD_ACORE +extern int dmStartDaemon(int argc, char const *argv[]); +extern void dmStopDaemon(); + +SDaemonObj daemonObj = {0}; +#endif + SShellObj shell = {0}; typedef struct { @@ -1299,17 +1306,118 @@ void *shellThreadLoop(void *arg) { return NULL; } -int32_t shellExecute() { +#ifdef TD_ACORE +typedef struct { + int32_t argc; + char **argv; +} SExecArgs; + +static void *dmStartDaemonFunc(void *param) { + int32_t code = 0; + SExecArgs *pArgs = (SExecArgs *)param; + int32_t argc = pArgs->argc; + char **argv = pArgs->argv; + + if (argc < 1) { + code = TSDB_CODE_INVALID_PARA; + printf("failed to start taosd since Invalid parameter, argc: %d\r\n", argc); + goto _exit; + } + + code = dmStartDaemon(argc, (const char **)argv); + if (code != 0) { + printf("failed to start taosd since %s\r\n", tstrerror(code)); + goto _exit; + } + +_exit: + if (code != 0) { + atomic_store_32(&daemonObj.stat, code); + } + return NULL; +} + +static int32_t shellStartDaemon(int argc, char *argv[]) { + int32_t code = 0, lino = 0; + SExecArgs *pArgs = NULL; + int64_t startMs = taosGetTimestampMs(), endMs = startMs; + + TdThreadAttr thAttr; + (void)taosThreadAttrInit(&thAttr); + (void)taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_JOINABLE); + + pArgs = (SExecArgs *)taosMemoryCalloc(1, sizeof(SExecArgs)); + if (pArgs == NULL) { + code = terrno; + TAOS_CHECK_EXIT(code); + } + pArgs->argc = argc; + pArgs->argv = argv; + + tsLogEmbedded = 1; + + TAOS_CHECK_EXIT(taosThreadCreate(&daemonObj.pid, &thAttr, dmStartDaemonFunc, pArgs)); + + while (true) { + if (atomic_load_64(&tsDndStarted)) { + atomic_store_32(&daemonObj.stat, 1); + break; + } + int32_t daemonstat = atomic_load_32(&daemonObj.stat); + if (daemonstat < 0) { + code = daemonstat; + TAOS_CHECK_EXIT(code); + } + + if (daemonstat > 1) { + code = TSDB_CODE_APP_ERROR; + TAOS_CHECK_EXIT(code); + } + taosMsleep(100); + } + +_exit: + endMs = taosGetTimestampMs(); + (void)taosThreadAttrDestroy(&thAttr); + taosMemoryFreeClear(pArgs); + if (code) { + printf("\r\n The daemon start failed at line %d since %s, cost %" PRIi64 " ms\r\n", lino, tstrerror(code), + endMs - startMs); + } else { + printf("\r\n The daemon started successfully, cost %" PRIi64 " ms\r\n", endMs - startMs); + } + tsLogEmbedded = 0; + return code; +} + +static void shellStopDaemon() { + tsLogEmbedded = 1; + dmStopDaemon(); + if (taosCheckPthreadValid(daemonObj.pid)) { + (void)taosThreadJoin(daemonObj.pid, NULL); + taosThreadClear(&daemonObj.pid); + } +} +#endif + +int32_t shellExecute(int argc, char *argv[]) { + int32_t code = 0, lino = 0; printf(shell.info.clientVersion, shell.info.cusName, taos_get_client_info(), shell.info.cusName); fflush(stdout); +#ifdef TD_ACORE + if ((code = shellStartDaemon(argc, argv)) != 0) { + TAOS_CHECK_GOTO(code, &lino, _exit_half); + } +#endif + SShellArgs *pArgs = &shell.args; #ifdef WEBSOCKET if (shell.args.restful || shell.args.cloud) { if (shell_conn_ws_server(1)) { printf("failed to connect to server, reason: %s[0x%08X]\n%s", ws_errstr(NULL), ws_errno(NULL), ERROR_CODE_DETAIL); fflush(stdout); - return -1; + TAOS_CHECK_GOTO(-1, &lino, _exit_half); } } else { #endif @@ -1322,7 +1430,8 @@ int32_t shellExecute() { if (shell.conn == NULL) { printf("failed to connect to server, reason: %s[0x%08X]\n%s", taos_errstr(NULL), taos_errno(NULL), ERROR_CODE_DETAIL); fflush(stdout); - return -1; + code = terrno == 0 ? TSDB_CODE_INTERNAL_ERROR : terrno; + TAOS_CHECK_GOTO(code, &lino, _exit_half); } #ifdef WEBSOCKET } @@ -1363,12 +1472,12 @@ int32_t shellExecute() { shellWriteHistory(); shellCleanupHistory(); - return 0; + goto _exit_half; } - if (tsem_init(&shell.cancelSem, 0, 0) != 0) { + if ((code = tsem_init(&shell.cancelSem, 0, 0)) != 0) { printf("failed to create cancel semaphore\r\n"); - return -1; + TAOS_CHECK_GOTO(code, &lino, _exit_half); } TdThread spid = {0}; @@ -1416,12 +1525,21 @@ int32_t shellExecute() { showAD(true); } #endif - taosThreadJoin(spid, NULL); + goto _exit; // normal exit +_exit_half: +#ifdef TD_ACORE + shellStopDaemon(); +#endif + TAOS_RETURN(code); +_exit: +#ifdef TD_ACORE + shellStopDaemon(); +#endif shellCleanupHistory(); taos_kill_query(shell.conn); taos_close(shell.conn); - return 0; + TAOS_RETURN(code); } diff --git a/tools/shell/src/shellMain.c b/tools/shell/src/shellMain.c index fc6ba0f7d897..6ae20c0082d3 100644 --- a/tools/shell/src/shellMain.c +++ b/tools/shell/src/shellMain.c @@ -48,6 +48,9 @@ int main(int argc, char *argv[]) { shell.args.cloud = true; shell.args.local = false; #endif +#ifdef TD_ACORE + tsAcoreOS = true; +#endif #if 0 #if !defined(WINDOWS) @@ -110,7 +113,7 @@ int main(int argc, char *argv[]) { // support port feature shellAutoInit(); - int32_t ret = shellExecute(); + int32_t ret = shellExecute(argc, argv); shellAutoExit(); return ret; } diff --git a/utils/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c index dcf5d6ab1237..88252a21c557 100644 --- a/utils/tsim/src/simSystem.c +++ b/utils/tsim/src/simSystem.c @@ -27,7 +27,7 @@ char simScriptDir[PATH_MAX] = {0}; extern bool simExecSuccess; int32_t simInitCfg() { - taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, 1); + taosCreateLog("simlog", 1, configDir, NULL, NULL, NULL, NULL, LOG_MODE_TAOSC); taosInitCfg(configDir, NULL, NULL, NULL, NULL, 1); SConfig *pCfg = taosGetCfg();