From 1ed7ec1d2eccb8990d02a8bd9ad76ec8237bf086 Mon Sep 17 00:00:00 2001 From: liqiang Date: Wed, 20 Mar 2024 16:05:38 +0800 Subject: [PATCH 1/2] rexec inherit process attributes and improve log logic Signed-off-by: liqiang --- qtfs/rexec/rexec.c | 105 ++++++++++++++++++++++++++++++++++---- qtfs/rexec/rexec.h | 40 +++++++++++---- qtfs/rexec/rexec_server.c | 2 + qtfs/rexec/rexec_shim.c | 97 +++++++++++++++++++++++++++++++++-- 4 files changed, 222 insertions(+), 22 deletions(-) diff --git a/qtfs/rexec/rexec.c b/qtfs/rexec/rexec.c index eb8b9bd..9455ebb 100644 --- a/qtfs/rexec/rexec.c +++ b/qtfs/rexec/rexec.c @@ -33,6 +33,8 @@ #include #include #include +#include + #include #include "dirent.h" @@ -418,20 +420,21 @@ static int rexec_get_fdinfo(struct dirent *fdentry, struct rexec_fdinfo *fdinfo) } return 0; } + + + // 返回一个拼装好的json格式字符串，内存在内部申请好 // 由调用者释放 // 内容是本进程所有REG类型文件的信息 -static char *rexec_get_fds_jsonstr() +int rexec_get_fds_jsonstr(struct json_object *root) { - struct json_object *root = json_object_new_object(); - char *json_str; int len; DIR *fddir = NULL; struct dirent *fdentry; struct rexec_fdinfo *fdinfo; if (root == NULL) { rexec_err("create json-c root failed."); - return NULL; + return -1; } fdinfo = (struct rexec_fdinfo *)malloc(sizeof(struct rexec_fdinfo)); if (fdinfo == NULL) { @@ -476,17 +479,99 @@ static char *rexec_get_fds_jsonstr() free(fdinfo); json_object_object_add(root, "Files", files_arr); - json_str = strdup(json_object_get_string(root)); - json_object_put(root); - - return json_str; + return 0; json_err: closedir(fddir); free(fdinfo); err_end: + return -1; +} + +rlim_t get_rlimit(unsigned int flag) +{ + char str[32] = {0}; + struct rlimit rlim; + if (getrlimit(flag, &rlim) != 0) { + rexec_err("get rlimit flag:%u failed.", flag); + return -1; + } + return rlim.rlim_cur; +} + +void put_str(char *str) +{ + return; +} + +rlim_t get_prlimit_handle(unsigned int flag) +{ + return get_rlimit(flag); +} + +// { +// "Attrs": [ +// {"key" : "maxMemLock", "value" : "1048576"}, +// {"key" : "maxProcesses", "value" : "10"}, +// {"key" : "maxFiles", "value" : "1024"}, +// {"key" : "maxCoreSize", "value" : "1048576"}, +// ] +// } +struct rexec_pattr_tab { + unsigned int flag; + char *key; + rlim_t (*get_value_handler)(unsigned int); +}; + +static struct rexec_pattr_tab pattr_table[] = { + {RLIMIT_MEMLOCK, "maxMemLock", get_prlimit_handle}, + {RLIMIT_NPROC, "maxProcesses", get_prlimit_handle}, + {RLIMIT_NOFILE, "maxFiles", get_prlimit_handle}, + {RLIMIT_CORE, "maxCoreSize", get_prlimit_handle}, + }; + +static int rexec_get_process_attr(struct json_object *root) +{ + struct json_object *attr_arr = json_object_new_array(); + + for (int i = 0; i < sizeof(pattr_table)/sizeof(pattr_table[0]); i++) { + struct json_object *cap = json_object_new_object(); + struct json_object *item = NULL; + + item = json_object_new_string(pattr_table[i].key); + json_object_object_add(cap, "key", item); + item = json_object_new_int64(pattr_table[i].get_value_handler(pattr_table[i].flag)); + json_object_object_add(cap, "value", item); + json_object_array_add(attr_arr, cap); + } + json_object_object_add(root, "Attrs", attr_arr); + return 0; +} + +static char *rexec_inherit_jsonstr() +{ + struct json_object *root = json_object_new_object(); + char *json_str; + + if (root == NULL) { + rexec_err("create json root failed. errno:%d", errno); + return NULL; + } + + if (rexec_get_fds_jsonstr(root) == -1) { + rexec_err("get fds json str failed and continue."); + } + + if (rexec_get_process_attr(root) == -1) { + rexec_err("get process attribute failed and continue."); + } + + json_str = strdup(json_object_get_string(root)); json_object_put(root); - return NULL; + + rexec_log("json get:%s", json_str); + + return json_str; } // 将rexec进程从parent继承到的匿名pipe继承给远端进程 @@ -735,7 +820,7 @@ int main(int argc, char *argv[]) rexec_err("argv is invalid."); return -1; } - char *fds_json = rexec_get_fds_jsonstr(); + char *fds_json = rexec_inherit_jsonstr(); if (fds_json == NULL) { rexec_err("Get fds info json string failed."); return -1; diff --git a/qtfs/rexec/rexec.h b/qtfs/rexec/rexec.h index 7ab23a4..fb0733b 100644 --- a/qtfs/rexec/rexec.h +++ b/qtfs/rexec/rexec.h @@ -74,24 +74,43 @@ static inline void rexec_log_init() char deflog[REXEC_LOG_PATH_LEN] = "/dev/null"; char *logfile = getenv("REXEC_LOG_FILE"); if (logfile == NULL) { - logfile = deflog; + rexec_logfile = NULL; + return; } else if (strcmp(logfile, "std") == 0) { rexec_logfile = stderr; return; } -retry: + rexec_logfile = fopen(logfile, "a"); - if (rexec_logfile == NULL) { - if (strcmp(logfile, "/dev/null") == 0) { - return; - } - // 输入的文件打开失败则回退到无日志模式 - logfile = deflog; - goto retry; + return; +} + +static inline void rexec_log_son_rebuild() +{ + char deflog[REXEC_LOG_PATH_LEN] = "/dev/null"; + char *logfile = getenv("REXEC_LOG_FILE"); + if (logfile == NULL) { + rexec_logfile = NULL; + return; + } else if (strcmp(logfile, "std") == 0) { + rexec_logfile = stderr; + return; } + fclose(rexec_logfile); + + int son_loglen = strlen(logfile) + 16; + char *son_logfile = (char *)malloc(son_loglen); + if (son_logfile == NULL) + return; + memset(son_logfile, 0, son_loglen); + sprintf(son_logfile, "%s_%d", logfile, getpid()); + + rexec_logfile = fopen(son_logfile, "a"); + free(son_logfile); return; } + // flag: 1--nonblock; 0--block static inline int rexec_set_nonblock(int fd, int block) { @@ -130,6 +149,7 @@ static inline int rexec_set_inherit(int fd, bool inherit) fprintf(rexec_logfile, "[%d/%02d/%02d %02d:%02d:%02d][LOG:%s:%3d]"info"\n", \ p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + fflush(rexec_logfile);\ } #define rexec_log2(info, ...) \ @@ -141,6 +161,7 @@ static inline int rexec_set_inherit(int fd, bool inherit) fprintf(rexec_logfile, "[%d/%02d/%02d %02d:%02d:%02d][LOG:%s:%3d]"info"\n", \ p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + fflush(rexec_logfile);\ } #define rexec_err(info, ...) \ @@ -152,6 +173,7 @@ static inline int rexec_set_inherit(int fd, bool inherit) fprintf(rexec_logfile, "[%d/%02d/%02d %02d:%02d:%02d][ERROR:%s:%3d]"info"\n", \ p.tm_year + 1900, p.tm_mon+1, p.tm_mday, \ p.tm_hour, p.tm_min, p.tm_sec, __func__, __LINE__, ##__VA_ARGS__); \ + fflush(rexec_logfile);\ } static inline unsigned int rexec_fd_mode(int fd) diff --git a/qtfs/rexec/rexec_server.c b/qtfs/rexec/rexec_server.c index 5e571fd..871f80f 100644 --- a/qtfs/rexec/rexec_server.c +++ b/qtfs/rexec/rexec_server.c @@ -312,7 +312,9 @@ static int rexec_start_new_process(int newconnfd) close(pipefd[PIPE_WRITE]); return 0; } + // son + rexec_log_son_rebuild(); close(pipefd[PIPE_READ]); struct rexec_msg head; diff --git a/qtfs/rexec/rexec_shim.c b/qtfs/rexec/rexec_shim.c index 6c62d3d..241db07 100644 --- a/qtfs/rexec/rexec_shim.c +++ b/qtfs/rexec/rexec_shim.c @@ -21,6 +21,7 @@ #include #include #include +#include #include #include @@ -107,7 +108,7 @@ void rshim_reg_file_resume(const char * const json_buf) } obj_files = json_object_object_get(fd_json, "Files"); int arraylen = json_object_array_length(obj_files); - for (int i=0; i< arraylen; i++){ + for (int i=0; i< arraylen; i++) { obj_file = json_object_array_get_idx(obj_files, i); obj_fd = json_object_object_get(obj_file, "Fd"); fd = json_object_get_int(obj_fd); @@ -126,6 +127,95 @@ void rshim_reg_file_resume(const char * const json_buf) return; } +int set_rlimit(unsigned int flag, rlim_t value) +{ + char str[32] = {0}; + struct rlimit rlim; + rlim.rlim_cur = rlim.rlim_max = value; + if (setrlimit(flag, &rlim) < 0) { + rexec_err("set rlimit flag:%u failed.", flag); + return -1; + } + rexec_log("set process rlimit flag:%u value:%lu", flag, value); + return 0; +} + +int set_prlimit_handle(unsigned int flag, rlim_t value) +{ + return set_rlimit(flag, value); +} + +// { +// "Attrs": [ +// {"key" : "maxMemLock", "value" : "1048576"}, +// {"key" : "maxProcesses", "value" : "10"}, +// {"key" : "maxFiles", "value" : "1024"}, +// {"key" : "maxCoreSize", "value" : "1048576"}, +// ] +// } + +struct rexec_pattr_tab { + unsigned int flag; + char *key; + int (*set_value_handler)(unsigned int, rlim_t); +}; + +// 这里排列要和另一端一模一样 +static struct rexec_pattr_tab pattr_table[] = { + {RLIMIT_MEMLOCK, "maxMemLock", set_prlimit_handle}, + {RLIMIT_NPROC, "maxProcesses", set_prlimit_handle}, + {RLIMIT_NOFILE, "maxFiles", set_prlimit_handle}, + {RLIMIT_CORE, "maxCoreSize", set_prlimit_handle}, +}; + +void rshim_process_attr_resume(const char * const json_buf) +{ + struct json_object *attrs; + struct json_object *cap; + struct json_object *key; + struct json_object *value; + struct json_object *attr_json = json_tokener_parse(json_buf); + if (attr_json == NULL) { + rshim_err("parse json error when get process attr resume"); + return; + } + + attrs = json_object_object_get(attr_json, "Attrs"); + if (attrs == NULL) { + rshim_err("get Attrs from json failed, ignore process attributes inherit."); + goto end; + } + int arraylen = json_object_array_length(attrs); + for (int i=0; i< arraylen; i++) { + const char *n; + rlim_t v; + cap = json_object_array_get_idx(attrs, i); + if (cap == NULL) { + rshim_err("get inherit process rlimit index:%d name:%s value:%lu failed.", i, pattr_table[i].key, v); + goto end; + } + + key = json_object_object_get(cap, "key"); + n = json_object_get_string(key); + value = json_object_object_get(cap, "value"); + v = json_object_get_int64(value); + + if (strcmp(n, pattr_table[i].key) != 0) { + rshim_err("attr inherit json item not match index:%d key:%s jsonkey:%s value:%lu", i, n, pattr_table[i].key, v); + continue; + } + if (pattr_table[i].set_value_handler(pattr_table[i].flag, v) != 0) { + rshim_err("inherit process rlimit :%s value:%lu failed.", pattr_table[i].key, v); + } else { + rshim_log("process attr set success, key:%s value:%lu", n, v); + } + } + +end: + json_object_put(attr_json); + return; +} + /* param list: 1) -f xxx.json binary param1 param2 ... @@ -143,11 +233,12 @@ int rexec_shim_entry(int argc, char *argv[]) newarg = argv; } - rshim_close_all_fd(); - rshim_log("Get json str:%s", json_str); + rshim_close_all_fd(); rshim_reg_file_resume(json_str); + rexec_log_son_rebuild(); + rshim_process_attr_resume(json_str); execvp(newarg[0], newarg); perror("execvp failed."); -- Gitee From ca35e18820efec2a3e3dddb613bf26a6a2df6ca1 Mon Sep 17 00:00:00 2001 From: liqiang Date: Wed, 20 Mar 2024 16:20:12 +0800 Subject: [PATCH 2/2] change rexec_logfile to CLOEXEC in son process Signed-off-by: liqiang --- qtfs/rexec/rexec.h | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/qtfs/rexec/rexec.h b/qtfs/rexec/rexec.h index fb0733b..6511c95 100644 --- a/qtfs/rexec/rexec.h +++ b/qtfs/rexec/rexec.h @@ -68,6 +68,20 @@ struct rexec_msg { char msg[0]; }; +static inline int rexec_set_inherit(int fd, bool inherit) +{ + int fflags; + if ((fflags = fcntl(fd, F_GETFD)) < 0) + return -1; + if (inherit) + fflags &= ~FD_CLOEXEC; + else + fflags |= FD_CLOEXEC; + if ((fcntl(fd, F_SETFD, fflags)) < 0) + return -1; + return 0; +} + extern FILE *rexec_logfile; static inline void rexec_log_init() { @@ -107,6 +121,7 @@ static inline void rexec_log_son_rebuild() rexec_logfile = fopen(son_logfile, "a"); free(son_logfile); + rexec_set_inherit(fileno(rexec_logfile), 0); return; } @@ -126,20 +141,6 @@ static inline int rexec_set_nonblock(int fd, int block) return 0; } -static inline int rexec_set_inherit(int fd, bool inherit) -{ - int fflags; - if ((fflags = fcntl(fd, F_GETFD)) < 0) - return -1; - if (inherit) - fflags &= ~FD_CLOEXEC; - else - fflags |= FD_CLOEXEC; - if ((fcntl(fd, F_SETFD, fflags)) < 0) - return -1; - return 0; -} - #define rexec_log(info, ...) \ if (rexec_logfile != NULL) {\ time_t t; \ -- Gitee