diff --git a/qtfs/qtfs_server/fsops.c b/qtfs/qtfs_server/fsops.c index 7a8496b160dee3750931d8bd1eacf5a2e2b79dda..55130ccece32dcb94b98e5ee36ac578b16fe4bca 100644 --- a/qtfs/qtfs_server/fsops.c +++ b/qtfs/qtfs_server/fsops.c @@ -426,7 +426,7 @@ static int handle_readiter(struct qtserver_arg *arg) rsp->d.errno = -ENOENT; goto end; } - fullname = file_path(file, pathbuf, PATH_MAX); + fullname = file_path(file, pathbuf, PATH_MAX); if (IS_ERR_OR_NULL(fullname) || !in_white_list(fullname, QTFS_WHITELIST_READ)) { qtfs_err("read iter path not in whitelist.\n"); __putname(pathbuf); @@ -493,7 +493,7 @@ early_end: static int handle_write(struct qtserver_arg *arg) { struct file *file = NULL; - char *pathbuf, *fullname; + char *pathbuf, *fullname; int block_size; struct qtreq_write *req = (struct qtreq_write *)REQ(arg); struct qtrsp_write *rsp = (struct qtrsp_write *)RSP(arg); diff --git a/qtfs/rexec/rexec.c b/qtfs/rexec/rexec.c index c1acb8d2d387fee0f5a55ad8563dc2d9be1ce8b3..1858e0b0d48642fa8dea4139dc3226a2512e4389 100644 --- a/qtfs/rexec/rexec.c +++ b/qtfs/rexec/rexec.c @@ -43,18 +43,18 @@ FILE *rexec_logfile = NULL; struct rexec_global_var { - int rexec_hs_fd[2]; + int rexec_hs_fd[2]; }; struct rexec_global_var g_rexec; struct rexec_client_event { - int fd; - int outfd; // for stdin out err and other pipe - int (*handler)(struct rexec_client_event *); - int *exit_status; - int *pidfd; + int fd; + int outfd; // for stdin out err and other pipe + int (*handler)(struct rexec_client_event *); + int *exit_status; + int *pidfd; }; #define REXEC_PIDMAP_PATH "/var/run/rexec/pids" @@ -62,604 +62,604 @@ struct rexec_client_event { #define REXEC_PID_LEN 16 static int rexec_conn_to_server() { - struct rexec_conn_arg arg; - char *ret = strncpy(arg.sun_path, REXEC_UDS_CONN, sizeof(arg.sun_path)); - if (ret == NULL) { - rexec_err("strncpy sun path failed"); - return -1; - } - arg.cs = REXEC_SOCK_CLIENT; - arg.udstype = SOCK_STREAM; - if (rexec_build_unix_connection(&arg) != 0) - return -1; - return arg.connfd; + struct rexec_conn_arg arg; + char *ret = strncpy(arg.sun_path, REXEC_UDS_CONN, sizeof(arg.sun_path)); + if (ret == NULL) { + rexec_err("strncpy sun path failed"); + return -1; + } + arg.cs = REXEC_SOCK_CLIENT; + arg.udstype = SOCK_STREAM; + if (rexec_build_unix_connection(&arg) != 0) + return -1; + return arg.connfd; } static int rexec_calc_argv_len(int argc, char *argv[]) { - int len = 0; - for (int i = 0; i < argc; i++) { - if (argv[i] == NULL) { - rexec_err("Invalid argv index:%d", i); - return -1; - } - len += strlen(argv[i]); - len++; - } - return len; + int len = 0; + for (int i = 0; i < argc; i++) { + if (argv[i] == NULL) { + rexec_err("Invalid argv index:%d", i); + return -1; + } + len += strlen(argv[i]); + len++; + } + return len; } static int rexec_msg_fill_argv(int argc, char *argv[], char *msg) { - int offset = 0; - for (int i = 0; i < argc; i++) { - strcpy(&msg[offset], argv[i]); //此处msg已经在前面通过计算出的len预先分配内存，保证这里不会越界 - offset += (strlen(argv[i]) + 1); - } - return offset; + int offset = 0; + for (int i = 0; i < argc; i++) { + strcpy(&msg[offset], argv[i]); //此处msg已经在前面通过计算出的len预先分配内存，保证这里不会越界 + offset += (strlen(argv[i]) + 1); + } + return offset; } static int rexec_io(struct rexec_client_event *evt) { #define MAX_MSG_LEN 256 - char buf[MAX_MSG_LEN]; - int len; - int ret; - while ((len = read(evt->fd, buf, MAX_MSG_LEN)) > 0) { - ret = write(evt->outfd, buf, len); - if (ret <= 0) { - rexec_err("Read from fd:%d len:%d write to fd:%d failed ret:%d", evt->fd, len, evt->outfd, ret); - return REXEC_EVENT_DEL; - } - if (ret != len) { - rexec_err("Read from fd:%d len:%d but write to fd:%d ret:%d", evt->fd, len, evt->outfd, ret); - } - } - return REXEC_EVENT_OK; + char buf[MAX_MSG_LEN]; + int len; + int ret; + while ((len = read(evt->fd, buf, MAX_MSG_LEN)) > 0) { + ret = write(evt->outfd, buf, len); + if (ret <= 0) { + rexec_err("Read from fd:%d len:%d write to fd:%d failed ret:%d", evt->fd, len, evt->outfd, ret); + return REXEC_EVENT_DEL; + } + if (ret != len) { + rexec_err("Read from fd:%d len:%d but write to fd:%d ret:%d", evt->fd, len, evt->outfd, ret); + } + } + return REXEC_EVENT_OK; } // return -1 means process exit. static int rexec_conn_msg(struct rexec_client_event *evt) { - struct rexec_msg head; - int ret = recv(evt->fd, &head, sizeof(struct rexec_msg), MSG_WAITALL); - if (ret <= 0) { - rexec_err("Rexec conn recv err:%d errno:%d", ret, errno); - return REXEC_EVENT_EXIT; - } - switch (head.msgtype) { - case REXEC_KILL: - *evt->exit_status = head.exit_status; - rexec_err("Rexec conn recv kill msg, exit:%d now.", head.exit_status); - return REXEC_EVENT_EXIT; - case REXEC_PIDMAP: { - int mypid = getpid(); - int peerpid = head.pid; - char path[REXEC_PIDMAP_PATH_LEN] = {0}; - char buf[REXEC_PID_LEN] = {0}; - int fd; - int err; - if (*evt->pidfd > 0) { - rexec_err("Rexec pidmap msg > 1 error."); - return REXEC_EVENT_OK; - } - sprintf(path, "%s/%d", REXEC_PIDMAP_PATH, mypid); - fd = open(path, O_CREAT|O_WRONLY, 0600); - if (fd < 0) { - rexec_err("Rexec create pidmap:%d-%d failed, path:%s open failed:%d", - mypid, peerpid, path, fd); - break; - } - *evt->pidfd = fd; - if ((err = flock(fd, LOCK_EX)) != 0) { - rexec_err("Rexec flock file:%s failed, errno:%d rexec exit.", path, err); - return REXEC_EVENT_EXIT; - } - if ((err = ftruncate(fd, 0)) != 0) { - rexec_err("Rexec pidmap file:%s clear failed errno:%d rexec exit.", path, err); - return REXEC_EVENT_EXIT; - } - if ((err = lseek(fd, 0, SEEK_SET)) < 0) { - rexec_err("Rexec pidmap file:%s lseek 0 failed errno:%d rexec exit", path, err); - return REXEC_EVENT_EXIT; - } - sprintf(buf, "%d", peerpid); - if ((err = write(fd, buf, strlen(buf))) <= 0) { - rexec_err("Rexec pidmap file:%s write pid:%d failed errno:%d rexec exit.", path, peerpid, err); - return REXEC_EVENT_EXIT; - } - if (g_rexec.rexec_hs_fd[PIPE_WRITE] != -1 && g_rexec.rexec_hs_fd[PIPE_READ] != -1) { - err = write(g_rexec.rexec_hs_fd[PIPE_WRITE], "1", 1); - if (err <= 0) { - rexec_err("rexec handshake write 1 failed, hs write:%d.", g_rexec.rexec_hs_fd[PIPE_WRITE]); - return REXEC_EVENT_ERR; - } - } else { - char msg[sizeof(struct rexec_msg) + 1]; - struct rexec_msg *hs = msg; - char *ok = hs->msg; - hs->msgtype = REXEC_HANDSHAKE; - hs->msglen = 1; - *ok = '1'; - if (write(evt->fd, hs, sizeof(struct rexec_msg) + 1) <= 0) { - rexec_err("send handshake failed, remote process will die"); - return REXEC_EVENT_EXIT; - } - } - break; - } - default: - break; - } - - rexec_log("Rexec conn recv msgtype:%d argc:%d stdno:%d msglen:%d", - head.msgtype, head.argc, head.stdno, head.msglen); - return REXEC_EVENT_OK; + struct rexec_msg head; + int ret = recv(evt->fd, &head, sizeof(struct rexec_msg), MSG_WAITALL); + if (ret <= 0) { + rexec_err("Rexec conn recv err:%d errno:%d", ret, errno); + return REXEC_EVENT_EXIT; + } + switch (head.msgtype) { + case REXEC_KILL: + *evt->exit_status = head.exit_status; + rexec_err("Rexec conn recv kill msg, exit:%d now.", head.exit_status); + return REXEC_EVENT_EXIT; + case REXEC_PIDMAP: { + int mypid = getpid(); + int peerpid = head.pid; + char path[REXEC_PIDMAP_PATH_LEN] = {0}; + char buf[REXEC_PID_LEN] = {0}; + int fd; + int err; + if (*evt->pidfd > 0) { + rexec_err("Rexec pidmap msg > 1 error."); + return REXEC_EVENT_OK; + } + sprintf(path, "%s/%d", REXEC_PIDMAP_PATH, mypid); + fd = open(path, O_CREAT|O_WRONLY, 0600); + if (fd < 0) { + rexec_err("Rexec create pidmap:%d-%d failed, path:%s open failed:%d", + mypid, peerpid, path, fd); + break; + } + *evt->pidfd = fd; + if ((err = flock(fd, LOCK_EX)) != 0) { + rexec_err("Rexec flock file:%s failed, errno:%d rexec exit.", path, err); + return REXEC_EVENT_EXIT; + } + if ((err = ftruncate(fd, 0)) != 0) { + rexec_err("Rexec pidmap file:%s clear failed errno:%d rexec exit.", path, err); + return REXEC_EVENT_EXIT; + } + if ((err = lseek(fd, 0, SEEK_SET)) < 0) { + rexec_err("Rexec pidmap file:%s lseek 0 failed errno:%d rexec exit", path, err); + return REXEC_EVENT_EXIT; + } + sprintf(buf, "%d", peerpid); + if ((err = write(fd, buf, strlen(buf))) <= 0) { + rexec_err("Rexec pidmap file:%s write pid:%d failed errno:%d rexec exit.", path, peerpid, err); + return REXEC_EVENT_EXIT; + } + if (g_rexec.rexec_hs_fd[PIPE_WRITE] != -1 && g_rexec.rexec_hs_fd[PIPE_READ] != -1) { + err = write(g_rexec.rexec_hs_fd[PIPE_WRITE], "1", 1); + if (err <= 0) { + rexec_err("rexec handshake write 1 failed, hs write:%d.", g_rexec.rexec_hs_fd[PIPE_WRITE]); + return REXEC_EVENT_ERR; + } + } else { + char msg[sizeof(struct rexec_msg) + 1]; + struct rexec_msg *hs = msg; + char *ok = hs->msg; + hs->msgtype = REXEC_HANDSHAKE; + hs->msglen = 1; + *ok = '1'; + if (write(evt->fd, hs, sizeof(struct rexec_msg) + 1) <= 0) { + rexec_err("send handshake failed, remote process will die"); + return REXEC_EVENT_EXIT; + } + } + break; + } + default: + break; + } + + rexec_log("Rexec conn recv msgtype:%d argc:%d stdno:%d msglen:%d", + head.msgtype, head.argc, head.stdno, head.msglen); + return REXEC_EVENT_OK; } static struct rexec_client_event *rexec_add_event(int efd, int fd, int outfd, int (*handler)(struct rexec_client_event *)) { - struct rexec_client_event *event = (struct rexec_client_event *)malloc(sizeof(struct rexec_client_event)); - if (event == NULL) { - rexec_err("malloc failed."); - return NULL; - } - event->fd = fd; - event->outfd = outfd; - event->handler = handler; - struct epoll_event evt; - evt.data.ptr = (void *)event; - evt.events = EPOLLIN; - if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, event->fd, &evt)) { - rexec_err("epoll ctl add fd:%d event failed.", event->fd); - free(event); - return NULL; - } - return event; + struct rexec_client_event *event = (struct rexec_client_event *)malloc(sizeof(struct rexec_client_event)); + if (event == NULL) { + rexec_err("malloc failed."); + return NULL; + } + event->fd = fd; + event->outfd = outfd; + event->handler = handler; + struct epoll_event evt; + evt.data.ptr = (void *)event; + evt.events = EPOLLIN; + if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, event->fd, &evt)) { + rexec_err("epoll ctl add fd:%d event failed.", event->fd); + free(event); + return NULL; + } + return event; } static int rexec_del_event(struct rexec_client_event *event) { - // close will del fd in epoll list - close(event->fd); - free(event); - return 0; + // close will del fd in epoll list + close(event->fd); + free(event); + return 0; } enum { - REPOL_IN_INDEX = 0, - REPOL_OUT_INDEX, - REPOL_ERR_INDEX, - REPOL_INV_INDEX, + REPOL_IN_INDEX = 0, + REPOL_OUT_INDEX, + REPOL_ERR_INDEX, + REPOL_INV_INDEX, }; static int rexec_std_event(int efd, int rstdin, int rstdout, int rstderr) { - #define REXEC_MAX_EVENTS 4 - int infds[REPOL_INV_INDEX] = {STDIN_FILENO, rstdout, rstderr}; - int outfds[REPOL_INV_INDEX] = {rstdin, STDOUT_FILENO, STDERR_FILENO}; - - for (int i = 0; i < REPOL_INV_INDEX; i++) { - if (NULL == rexec_add_event(efd, infds[i], outfds[i], rexec_io)) { - rexec_err("epoll ctl add fd:%d event failed and ignore this mistake.", infds[i]); - continue; - } else { - if (rexec_set_nonblock(infds[i], 1) != 0) { - rexec_err("rexec set fd:%d i:%d non block failed.", infds[i], i); - return -1; - } - } - } - return 0; + #define REXEC_MAX_EVENTS 4 + int infds[REPOL_INV_INDEX] = {STDIN_FILENO, rstdout, rstderr}; + int outfds[REPOL_INV_INDEX] = {rstdin, STDOUT_FILENO, STDERR_FILENO}; + + for (int i = 0; i < REPOL_INV_INDEX; i++) { + if (NULL == rexec_add_event(efd, infds[i], outfds[i], rexec_io)) { + rexec_err("epoll ctl add fd:%d event failed and ignore this mistake.", infds[i]); + continue; + } else { + if (rexec_set_nonblock(infds[i], 1) != 0) { + rexec_err("rexec set fd:%d i:%d non block failed.", infds[i], i); + return -1; + } + } + } + return 0; } static int rexec_run(int efd, int connfd, char *argv[]) { - int pidfd = -1; - int exit_status = EXIT_FAILURE; - - struct rexec_client_event *connevt = rexec_add_event(efd, connfd, -1, rexec_conn_msg); - if (NULL == connevt || rexec_set_nonblock(connfd, 1) != 0) { - // process will exit, fd or mem resource will free by kernel soon - rexec_err("rexec add connfd event failed"); - return exit_status; - } - // 这两个指针只能在当前函数上下文使用，是当前函数栈指针 - connevt->exit_status = &exit_status; - connevt->pidfd = &pidfd; - - struct epoll_event *evts = calloc(REXEC_MAX_EVENTS, sizeof(struct epoll_event)); - if (evts == NULL) { - rexec_err("init calloc evts failed."); - goto end; - } - rexec_log("Rexec process start run, as proxy of remote %s", argv[1]); - while (1) { - int n = epoll_wait(efd, evts, REXEC_MAX_EVENTS, 1000); - int process_exit = 0; - if (n == 0) - continue; - if (n < 0) { - rexec_err("epoll wait return errcode:%d", n); - continue; - } - for (int i = 0; i < n; i++) { - struct rexec_client_event *evt = (struct rexec_client_event *)evts[i].data.ptr; - int ret = evt->handler(evt); - if (ret == REXEC_EVENT_EXIT) { - process_exit = 1; - } - if (ret == REXEC_EVENT_DEL) { - rexec_del_event(evt); - } - } - // process will exit, and free all resource and exit - if (process_exit) { - rexec_log("Rexec process %s exit.", argv[1]); - break; - } - } - - // clear pidmap file - if (pidfd > 0) { - char path[32] = {0}; - sprintf(path, "%s/%d", REXEC_PIDMAP_PATH, getpid()); - close(pidfd); - remove(path); - } + int pidfd = -1; + int exit_status = EXIT_FAILURE; + + struct rexec_client_event *connevt = rexec_add_event(efd, connfd, -1, rexec_conn_msg); + if (NULL == connevt || rexec_set_nonblock(connfd, 1) != 0) { + // process will exit, fd or mem resource will free by kernel soon + rexec_err("rexec add connfd event failed"); + return exit_status; + } + // 这两个指针只能在当前函数上下文使用，是当前函数栈指针 + connevt->exit_status = &exit_status; + connevt->pidfd = &pidfd; + + struct epoll_event *evts = calloc(REXEC_MAX_EVENTS, sizeof(struct epoll_event)); + if (evts == NULL) { + rexec_err("init calloc evts failed."); + goto end; + } + rexec_log("Rexec process start run, as proxy of remote %s", argv[1]); + while (1) { + int n = epoll_wait(efd, evts, REXEC_MAX_EVENTS, 1000); + int process_exit = 0; + if (n == 0) + continue; + if (n < 0) { + rexec_err("epoll wait return errcode:%d", n); + continue; + } + for (int i = 0; i < n; i++) { + struct rexec_client_event *evt = (struct rexec_client_event *)evts[i].data.ptr; + int ret = evt->handler(evt); + if (ret == REXEC_EVENT_EXIT) { + process_exit = 1; + } + if (ret == REXEC_EVENT_DEL) { + rexec_del_event(evt); + } + } + // process will exit, and free all resource and exit + if (process_exit) { + rexec_log("Rexec process %s exit.", argv[1]); + break; + } + } + + // clear pidmap file + if (pidfd > 0) { + char path[32] = {0}; + sprintf(path, "%s/%d", REXEC_PIDMAP_PATH, getpid()); + close(pidfd); + remove(path); + } free_end: - free(evts); + free(evts); end: - close(efd); - return exit_status; + close(efd); + return exit_status; } void rexec_create_pidmap_path() { - if (access(REXEC_RUN_PATH, F_OK) != 0) { - mkdir(REXEC_RUN_PATH, 0700); - } - mkdir(REXEC_PIDMAP_PATH, 0700); - return; + if (access(REXEC_RUN_PATH, F_OK) != 0) { + mkdir(REXEC_RUN_PATH, 0700); + } + mkdir(REXEC_PIDMAP_PATH, 0700); + return; } void rexec_clear_pids() { - char path[REXEC_PIDMAP_PATH_LEN] = {0}; - DIR *dir = NULL; - struct dirent *entry; - if (access(REXEC_PIDMAP_PATH, F_OK) != 0) { - rexec_create_pidmap_path(); - return; - } - dir = opendir(REXEC_PIDMAP_PATH); - if (dir == NULL) { - rexec_err("open path:%s failed", REXEC_PIDMAP_PATH); - return; - } - while (entry = readdir(dir)) { - int fd; - - if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0 || - strlen(entry->d_name) >= REXEC_PID_LEN) - continue; - - memset(path, 0, sizeof(path)); - sprintf(path, "%s/%s", REXEC_PIDMAP_PATH, entry->d_name); - fd = open(path, O_RDONLY); - if (fd <= 0) { - rexec_err("open pid file:%s failed", path); - continue; - } - if (flock(fd, LOCK_EX|LOCK_NB) != 0) { - close(fd); - continue; - } - close(fd); - if (remove(path) != 0) { - rexec_err("remove unuse pidmap file:%s failed", path); - } - } - - closedir(dir); - return; + char path[REXEC_PIDMAP_PATH_LEN] = {0}; + DIR *dir = NULL; + struct dirent *entry; + if (access(REXEC_PIDMAP_PATH, F_OK) != 0) { + rexec_create_pidmap_path(); + return; + } + dir = opendir(REXEC_PIDMAP_PATH); + if (dir == NULL) { + rexec_err("open path:%s failed", REXEC_PIDMAP_PATH); + return; + } + while (entry = readdir(dir)) { + int fd; + + if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0 || + strlen(entry->d_name) >= REXEC_PID_LEN) + continue; + + memset(path, 0, sizeof(path)); + sprintf(path, "%s/%s", REXEC_PIDMAP_PATH, entry->d_name); + fd = open(path, O_RDONLY); + if (fd <= 0) { + rexec_err("open pid file:%s failed", path); + continue; + } + if (flock(fd, LOCK_EX|LOCK_NB) != 0) { + close(fd); + continue; + } + close(fd); + if (remove(path) != 0) { + rexec_err("remove unuse pidmap file:%s failed", path); + } + } + + closedir(dir); + return; } #define REXEC_PATH_MAX 4096 struct rexec_fdinfo { - int fd; - char path[REXEC_PATH_MAX]; - unsigned int perm; - int offset; + int fd; + char path[REXEC_PATH_MAX]; + unsigned int perm; + int offset; }; static inline unsigned int rexec_fd_mode(int fd) { - struct stat st; - char path[32] = {0}; - if (fstat(fd, &st) != 0) { - rexec_err("get fd:%d fstat failed, errno:%d", fd, errno); - return 0; - } - return st.st_mode; + struct stat st; + char path[32] = {0}; + if (fstat(fd, &st) != 0) { + rexec_err("get fd:%d fstat failed, errno:%d", fd, errno); + return 0; + } + return st.st_mode; } static inline int rexec_is_reg_file(int fd) { - if (S_ISREG(rexec_fd_mode(fd))) - return 1; - return 0; + if (S_ISREG(rexec_fd_mode(fd))) + return 1; + return 0; } static int rexec_get_fdinfo(struct dirent *fdentry, struct rexec_fdinfo *fdinfo) { - char path[32] = {0}; - int ret; - int fd = atoi(fdentry->d_name); - if (fd <= STDERR_FILENO || fd == fileno(rexec_logfile)) - return -1; - if (!rexec_is_reg_file(fd)) - return -1; - sprintf(path, "/proc/self/fd/%s", fdentry->d_name); - ret = readlink(path, fdinfo->path, REXEC_PATH_MAX); - if (ret < 0) { - rexec_err("Get fd:%d link failed.", fd); - return -1; - } - fdinfo->fd = fd; - fdinfo->offset = lseek(fd, 0, SEEK_CUR); - - fdinfo->perm = fcntl(fd, F_GETFL, NULL); - if (fdinfo->perm == -1) { - rexec_err("Get fd:%d flags failed", fd); - return -1; - } - return 0; + char path[32] = {0}; + int ret; + int fd = atoi(fdentry->d_name); + if (fd <= STDERR_FILENO || fd == fileno(rexec_logfile)) + return -1; + if (!rexec_is_reg_file(fd)) + return -1; + sprintf(path, "/proc/self/fd/%s", fdentry->d_name); + ret = readlink(path, fdinfo->path, REXEC_PATH_MAX); + if (ret < 0) { + rexec_err("Get fd:%d link failed.", fd); + return -1; + } + fdinfo->fd = fd; + fdinfo->offset = lseek(fd, 0, SEEK_CUR); + + fdinfo->perm = fcntl(fd, F_GETFL, NULL); + if (fdinfo->perm == -1) { + rexec_err("Get fd:%d flags failed", fd); + return -1; + } + return 0; } // 返回一个拼装好的json格式字符串，内存在内部申请好 // 由调用者释放 // 内容是本进程所有REG类型文件的信息 static char *rexec_get_fds_jsonstr() { - struct json_object *root = json_object_new_object(); - char *json_str; - int len; - DIR *fddir = NULL; - struct dirent *fdentry; - struct rexec_fdinfo *fdinfo; - if (root == NULL) { - rexec_err("create json-c root failed."); - return NULL; - } - fdinfo = (struct rexec_fdinfo *)malloc(sizeof(struct rexec_fdinfo)); - if (fdinfo == NULL) { - rexec_err("malloc failed."); - goto err_end; - } - - fddir = opendir("/proc/self/fd"); - if (fddir == NULL) { - free(fdinfo); - rexec_err("open path:%s failed", REXEC_PIDMAP_PATH); - goto err_end; - } - - struct json_object *files_arr = json_object_new_array(); - - while (fdentry = readdir(fddir)) { - struct json_object *fd_obj = json_object_new_object(); - struct json_object *item = NULL; - - if (fd_obj == NULL) { - rexec_err("json c new object failed."); - goto json_err; - } - memset(fdinfo, 0, sizeof(struct rexec_fdinfo)); - if (rexec_get_fdinfo(fdentry, fdinfo) != 0) { - json_object_put(fd_obj); - continue; - } - item = json_object_new_int(fdinfo->fd); - json_object_object_add(fd_obj, "Fd", item); - item = json_object_new_string(fdinfo->path); - json_object_object_add(fd_obj, "Path", item); - item = json_object_new_int(fdinfo->perm); - json_object_object_add(fd_obj, "Perm", item); - item = json_object_new_int(fdinfo->offset); - json_object_object_add(fd_obj, "Offset", item); - - json_object_array_add(files_arr, fd_obj); - } - closedir(fddir); - free(fdinfo); - - json_object_object_add(root, "Files", files_arr); - json_str = strdup(json_object_get_string(root)); - json_object_put(root); - - return json_str; + struct json_object *root = json_object_new_object(); + char *json_str; + int len; + DIR *fddir = NULL; + struct dirent *fdentry; + struct rexec_fdinfo *fdinfo; + if (root == NULL) { + rexec_err("create json-c root failed."); + return NULL; + } + fdinfo = (struct rexec_fdinfo *)malloc(sizeof(struct rexec_fdinfo)); + if (fdinfo == NULL) { + rexec_err("malloc failed."); + goto err_end; + } + + fddir = opendir("/proc/self/fd"); + if (fddir == NULL) { + free(fdinfo); + rexec_err("open path:%s failed", REXEC_PIDMAP_PATH); + goto err_end; + } + + struct json_object *files_arr = json_object_new_array(); + + while (fdentry = readdir(fddir)) { + struct json_object *fd_obj = json_object_new_object(); + struct json_object *item = NULL; + + if (fd_obj == NULL) { + rexec_err("json c new object failed."); + goto json_err; + } + memset(fdinfo, 0, sizeof(struct rexec_fdinfo)); + if (rexec_get_fdinfo(fdentry, fdinfo) != 0) { + json_object_put(fd_obj); + continue; + } + item = json_object_new_int(fdinfo->fd); + json_object_object_add(fd_obj, "Fd", item); + item = json_object_new_string(fdinfo->path); + json_object_object_add(fd_obj, "Path", item); + item = json_object_new_int(fdinfo->perm); + json_object_object_add(fd_obj, "Perm", item); + item = json_object_new_int(fdinfo->offset); + json_object_object_add(fd_obj, "Offset", item); + + json_object_array_add(files_arr, fd_obj); + } + closedir(fddir); + free(fdinfo); + + json_object_object_add(root, "Files", files_arr); + json_str = strdup(json_object_get_string(root)); + json_object_put(root); + + return json_str; json_err: - closedir(fddir); - free(fdinfo); + closedir(fddir); + free(fdinfo); err_end: - json_object_put(root); - return NULL; + json_object_put(root); + return NULL; } static int rexec_handshake_proc(struct rexec_client_event *evt) { - char msg[sizeof(struct rexec_msg) + 1]; - struct rexec_msg *hs = msg; - int ret = read(evt->fd, hs->msg, 1); - if (ret <= 0) { - rexec_err("read from handshake pipe failed, ret:%d err:%d", ret, errno); - return REXEC_EVENT_DEL; - } - hs->msgtype = REXEC_HANDSHAKE; - hs->msglen = 1; - ret = write(evt->outfd, hs, sizeof(struct rexec_msg) + 1); - if (ret < 0) { - rexec_err("send handshake failed, connfd:%d.", evt->outfd); - } - return REXEC_EVENT_OK; + char msg[sizeof(struct rexec_msg) + 1]; + struct rexec_msg *hs = msg; + int ret = read(evt->fd, hs->msg, 1); + if (ret <= 0) { + rexec_err("read from handshake pipe failed, ret:%d err:%d", ret, errno); + return REXEC_EVENT_DEL; + } + hs->msgtype = REXEC_HANDSHAKE; + hs->msglen = 1; + ret = write(evt->outfd, hs, sizeof(struct rexec_msg) + 1); + if (ret < 0) { + rexec_err("send handshake failed, connfd:%d.", evt->outfd); + } + return REXEC_EVENT_OK; } static int rexec_handshake_init(int efd, int connfd) { - char *hs_read = getenv("REXEC_HANDSHAKE_RD"); - if (hs_read == NULL) { - rexec_log("handshake not in effect, read:%lx", hs_read); - return 0; - } - g_rexec.rexec_hs_fd[PIPE_READ] = atoi(hs_read); - - char *hs_write = getenv("REXEC_HANDSHAKE_WR"); - if (hs_write == NULL) { - rexec_log("handshake not in effect, wirte:%lx", hs_write); - g_rexec.rexec_hs_fd[PIPE_READ] = -1; - return 0; - } - g_rexec.rexec_hs_fd[PIPE_WRITE] = atoi(hs_write); - - if (g_rexec.rexec_hs_fd[PIPE_READ] <= STDERR_FILENO || g_rexec.rexec_hs_fd[PIPE_WRITE] <= STDERR_FILENO) { - rexec_log("handshake invalid fd read:%d write:%d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); - goto err_end; - } - if (!S_ISFIFO(rexec_fd_mode(g_rexec.rexec_hs_fd[PIPE_READ])) || !S_ISFIFO(rexec_fd_mode(g_rexec.rexec_hs_fd[PIPE_WRITE]))) { - rexec_err("handshake fd mode not fifo:%d %d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); - goto err_end; - } - if (rexec_add_event(efd, g_rexec.rexec_hs_fd[PIPE_READ], connfd, rexec_handshake_proc) == NULL) { - rexec_err("add handshake pipe read fd:%d to epoll failed", g_rexec.rexec_hs_fd[PIPE_READ]); - goto err_end; - } - rexec_log("handshake effect read:%d write:%d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); - return 0; + char *hs_read = getenv("REXEC_HANDSHAKE_RD"); + if (hs_read == NULL) { + rexec_log("handshake not in effect, read:%lx", hs_read); + return 0; + } + g_rexec.rexec_hs_fd[PIPE_READ] = atoi(hs_read); + + char *hs_write = getenv("REXEC_HANDSHAKE_WR"); + if (hs_write == NULL) { + rexec_log("handshake not in effect, wirte:%lx", hs_write); + g_rexec.rexec_hs_fd[PIPE_READ] = -1; + return 0; + } + g_rexec.rexec_hs_fd[PIPE_WRITE] = atoi(hs_write); + + if (g_rexec.rexec_hs_fd[PIPE_READ] <= STDERR_FILENO || g_rexec.rexec_hs_fd[PIPE_WRITE] <= STDERR_FILENO) { + rexec_log("handshake invalid fd read:%d write:%d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); + goto err_end; + } + if (!S_ISFIFO(rexec_fd_mode(g_rexec.rexec_hs_fd[PIPE_READ])) || !S_ISFIFO(rexec_fd_mode(g_rexec.rexec_hs_fd[PIPE_WRITE]))) { + rexec_err("handshake fd mode not fifo:%d %d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); + goto err_end; + } + if (rexec_add_event(efd, g_rexec.rexec_hs_fd[PIPE_READ], connfd, rexec_handshake_proc) == NULL) { + rexec_err("add handshake pipe read fd:%d to epoll failed", g_rexec.rexec_hs_fd[PIPE_READ]); + goto err_end; + } + rexec_log("handshake effect read:%d write:%d", g_rexec.rexec_hs_fd[PIPE_READ], g_rexec.rexec_hs_fd[PIPE_WRITE]); + return 0; err_end: - g_rexec.rexec_hs_fd[PIPE_READ] = -1; - g_rexec.rexec_hs_fd[PIPE_WRITE] = -1; - return -1; + g_rexec.rexec_hs_fd[PIPE_READ] = -1; + g_rexec.rexec_hs_fd[PIPE_WRITE] = -1; + return -1; } static void rexec_global_var_init() { - memset(&g_rexec, 0, sizeof(g_rexec)); - g_rexec.rexec_hs_fd[PIPE_READ] = -1; - g_rexec.rexec_hs_fd[PIPE_WRITE] = -1; - return; + memset(&g_rexec, 0, sizeof(g_rexec)); + g_rexec.rexec_hs_fd[PIPE_READ] = -1; + g_rexec.rexec_hs_fd[PIPE_WRITE] = -1; + return; } int main(int argc, char *argv[]) { - rexec_log_init(); - rexec_clear_pids(); - - int efd = epoll_create1(0); - if (efd == -1) { - rexec_err("epoll create1 failed, errno:%d.", errno); - return -1; - } - rexec_global_var_init(); - - int connfd = rexec_conn_to_server(); - if (connfd < 0) { - rexec_err("Rexec connect to server failed, errno:%d", errno); - return -1; - } - - if (rexec_handshake_init(efd, connfd) != 0) { - rexec_err("Rexec handshake environment set but get error."); - return -1; - } - rexec_log("Remote exec binary:%s", argv[1]); - - int arglen = rexec_calc_argv_len(argc - 1, &argv[1]); - if (arglen <= 0) { - rexec_err("argv is invalid."); - return -1; - } - char *fds_json = rexec_get_fds_jsonstr(); - if (fds_json == NULL) { - rexec_err("Get fds info json string failed."); - return -1; - } - arglen += sizeof(struct rexec_msg); - arglen += strlen(fds_json); - arglen = ((arglen / REXEC_MSG_LEN) + 1) * REXEC_MSG_LEN; - if (arglen <= 0) { - rexec_err("invalid arguments length:%d.", arglen); + rexec_log_init(); + rexec_clear_pids(); + + int efd = epoll_create1(0); + if (efd == -1) { + rexec_err("epoll create1 failed, errno:%d.", errno); + return -1; + } + rexec_global_var_init(); + + int connfd = rexec_conn_to_server(); + if (connfd < 0) { + rexec_err("Rexec connect to server failed, errno:%d", errno); + return -1; + } + + if (rexec_handshake_init(efd, connfd) != 0) { + rexec_err("Rexec handshake environment set but get error."); + return -1; + } + rexec_log("Remote exec binary:%s", argv[1]); + + int arglen = rexec_calc_argv_len(argc - 1, &argv[1]); + if (arglen <= 0) { + rexec_err("argv is invalid."); + return -1; + } + char *fds_json = rexec_get_fds_jsonstr(); + if (fds_json == NULL) { + rexec_err("Get fds info json string failed."); + return -1; + } + arglen += sizeof(struct rexec_msg); + arglen += strlen(fds_json); + arglen = ((arglen / REXEC_MSG_LEN) + 1) * REXEC_MSG_LEN; + if (arglen <= 0) { + rexec_err("invalid arguments length:%d.", arglen); free(fds_json); - return -1; - } - - struct rexec_msg *pmsg = (struct rexec_msg *)malloc(arglen); - if (pmsg == NULL) { - rexec_err("malloc failed"); - free(fds_json); - return -1; - } - char *bufmsg = pmsg->msg; - memset(pmsg, 0, arglen); - pmsg->msgtype = REXEC_EXEC; - pmsg->argc = argc - 1; // for remote binary's argc is argc-1 - // pmsg->msg is like: "binary"\0"argv[1]"\0"argv[2]"\0"..." - pmsg->msglen = rexec_msg_fill_argv(pmsg->argc, &argv[1], bufmsg); - strcpy(&bufmsg[pmsg->msglen], fds_json); - pmsg->msglen += strlen(fds_json); - free(fds_json); - - // pipefd[0] -- for read; pipefd[1] -- for write. - // rexec stdin --> rstdin[1] ------> rstdin[0] as stdin - // rexec stdout <-- rstdout[0] <------ rstdout[1] as stdout - // rexec stderr <-- rstderr[0] <------ rstderr[1] as stderr - int rstdin[2]; - int rstdout[2]; - int rstderr[2]; - - if (pipe(rstdin) == -1 || pipe(rstdout) == -1 || pipe(rstderr) == -1) { - rexec_err("Rexec create pipe failed."); - goto err_end; - } - pmsg->stdno = REXEC_STDIN; - if (rexec_sendmsg(connfd, (char *)pmsg, sizeof(struct rexec_msg) + pmsg->msglen, rstdin[0]) < 0) { - rexec_err("Rexec send exec msg failed, errno:%d", errno); - goto err_end; - } - rexec_log("Normal msg send len:%d head:%d", sizeof(struct rexec_msg) + pmsg->msglen, sizeof(struct rexec_msg)); - pmsg->msgtype = REXEC_PIPE; - pmsg->argc = 0; - pmsg->msglen = 0; - pmsg->stdno = REXEC_STDOUT; - if (rexec_sendmsg(connfd, (char *)pmsg, sizeof(struct rexec_msg), rstdout[1]) < 0) { - rexec_err("Rexec send exec msg failed, errno:%d", errno); - goto err_end; - } - pmsg->stdno = REXEC_STDERR; - if (rexec_sendmsg(connfd, (char *)pmsg, sizeof(struct rexec_msg), rstderr[1]) < 0) { - rexec_err("Rexec send exec msg failed, errno:%d", errno); - goto err_end; - } - free(pmsg); - - int exit_status; - close(rstdin[0]); - close(rstdout[1]); - close(rstderr[1]); - if (rexec_std_event(efd, rstdin[1], rstdout[0], rstderr[0]) != 0) { - rexec_err("add std event failed"); - goto err_end; - } - exit_status = rexec_run(efd, connfd, argv); - close(rstdin[1]); - close(rstdout[0]); - close(rstderr[0]); - fclose(rexec_logfile); - - exit(exit_status); + return -1; + } + + struct rexec_msg *pmsg = (struct rexec_msg *)malloc(arglen); + if (pmsg == NULL) { + rexec_err("malloc failed"); + free(fds_json); + return -1; + } + char *bufmsg = pmsg->msg; + memset(pmsg, 0, arglen); + pmsg->msgtype = REXEC_EXEC; + pmsg->argc = argc - 1; // for remote binary's argc is argc-1 + // pmsg->msg is like: "binary"\0"argv[1]"\0"argv[2]"\0"..." + pmsg->msglen = rexec_msg_fill_argv(pmsg->argc, &argv[1], bufmsg); + strcpy(&bufmsg[pmsg->msglen], fds_json); + pmsg->msglen += strlen(fds_json); + free(fds_json); + + // pipefd[0] -- for read; pipefd[1] -- for write. + // rexec stdin --> rstdin[1] ------> rstdin[0] as stdin + // rexec stdout <-- rstdout[0] <------ rstdout[1] as stdout + // rexec stderr <-- rstderr[0] <------ rstderr[1] as stderr + int rstdin[2]; + int rstdout[2]; + int rstderr[2]; + + if (pipe(rstdin) == -1 || pipe(rstdout) == -1 || pipe(rstderr) == -1) { + rexec_err("Rexec create pipe failed."); + goto err_end; + } + pmsg->stdno = REXEC_STDIN; + if (rexec_sendmsg(connfd, (char *)pmsg, sizeof(struct rexec_msg) + pmsg->msglen, rstdin[0]) < 0) { + rexec_err("Rexec send exec msg failed, errno:%d", errno); + goto err_end; + } + rexec_log("Normal msg send len:%d head:%d", sizeof(struct rexec_msg) + pmsg->msglen, sizeof(struct rexec_msg)); + pmsg->msgtype = REXEC_PIPE; + pmsg->argc = 0; + pmsg->msglen = 0; + pmsg->stdno = REXEC_STDOUT; + if (rexec_sendmsg(connfd, (char *)pmsg, sizeof(struct rexec_msg), rstdout[1]) < 0) { + rexec_err("Rexec send exec msg failed, errno:%d", errno); + goto err_end; + } + pmsg->stdno = REXEC_STDERR; + if (rexec_sendmsg(connfd, (char *)pmsg, sizeof(struct rexec_msg), rstderr[1]) < 0) { + rexec_err("Rexec send exec msg failed, errno:%d", errno); + goto err_end; + } + free(pmsg); + + int exit_status; + close(rstdin[0]); + close(rstdout[1]); + close(rstderr[1]); + if (rexec_std_event(efd, rstdin[1], rstdout[0], rstderr[0]) != 0) { + rexec_err("add std event failed"); + goto err_end; + } + exit_status = rexec_run(efd, connfd, argv); + close(rstdin[1]); + close(rstdout[0]); + close(rstderr[0]); + fclose(rexec_logfile); + + exit(exit_status); err_end: - fclose(rexec_logfile); - free(pmsg); - return -1; + fclose(rexec_logfile); + free(pmsg); + return -1; } diff --git a/qtfs/rexec/rexec.h b/qtfs/rexec/rexec.h index ce1280a02fea6ea798469134b0a0a84a4a40b366..e9182a29caeec09cf3c815532d2c6e63e3f1482e 100644 --- a/qtfs/rexec/rexec.h +++ b/qtfs/rexec/rexec.h @@ -20,22 +20,22 @@ #include enum { - PIPE_READ = 0, - PIPE_WRITE, + PIPE_READ = 0, + PIPE_WRITE, }; enum { - REXEC_EVENT_OK, - REXEC_EVENT_DEL, // del this event - REXEC_EVENT_EXIT, // exit process - REXEC_EVENT_ERR, + REXEC_EVENT_OK, + REXEC_EVENT_DEL, // del this event + REXEC_EVENT_EXIT, // exit process + REXEC_EVENT_ERR, }; enum { - REXEC_STDIN = 0x5a, - REXEC_STDOUT, - REXEC_STDERR, - REXEC_NONE, + REXEC_STDIN = 0x5a, + REXEC_STDOUT, + REXEC_STDERR, + REXEC_NONE, }; #define REXEC_MSG_1K 1024 @@ -48,78 +48,78 @@ enum { #define REXEC_RUN_PATH "/var/run/rexec/" enum rexec_msgtype { - REXEC_EXEC = 0x5a5a, // exec process - REXEC_KILL, // kill process - REXEC_PIPE, // client send a pipefd as stdin/out/err to server - REXEC_PIDMAP, // server send remote process's pid to client - REXEC_HANDSHAKE, + REXEC_EXEC = 0x5a5a, // exec process + REXEC_KILL, // kill process + REXEC_PIPE, // client send a pipefd as stdin/out/err to server + REXEC_PIDMAP, // server send remote process's pid to client + REXEC_HANDSHAKE, }; struct rexec_msg { - int msgtype; - // client to server - int argc; - int stdno; - int msglen; - // server to client - int exit_status; - int pid; // for pidmap - char msg[0]; + int msgtype; + // client to server + int argc; + int stdno; + int msglen; + // server to client + int exit_status; + int pid; // for pidmap + char msg[0]; }; extern FILE *rexec_logfile; static inline void rexec_log_init() { - char deflog[REXEC_LOG_PATH_LEN] = "/dev/null"; - char *logfile = getenv("REXEC_LOG_FILE"); - if (logfile == NULL) { - logfile = deflog; - } else if (strcmp(logfile, "std") == 0) { - rexec_logfile = stderr; - return; - } else { - printf("REXEC_LOG_FILE cannot be set to any value other than std\n"); - } + char deflog[REXEC_LOG_PATH_LEN] = "/dev/null"; + char *logfile = getenv("REXEC_LOG_FILE"); + if (logfile == NULL) { + logfile = deflog; + } else if (strcmp(logfile, "std") == 0) { + rexec_logfile = stderr; + return; + } else { + printf("REXEC_LOG_FILE cannot be set to any value other than std\n"); + } retry: - rexec_logfile = fopen(logfile, "a"); - if (rexec_logfile == NULL) { - if (strcmp(logfile, "/dev/null") == 0) { - return; - } - // 输入的文件打开失败则回退到无日志模式 - logfile = deflog; - goto retry; - } - return; + rexec_logfile = fopen(logfile, "a"); + if (rexec_logfile == NULL) { + if (strcmp(logfile, "/dev/null") == 0) { + return; + } + // 输入的文件打开失败则回退到无日志模式 + logfile = deflog; + goto retry; + } + return; } // flag: 1--nonblock; 0--block static inline int rexec_set_nonblock(int fd, int block) { - int fflags; - if ((fflags = fcntl(fd, F_GETFL)) < 0) - return -1; - if (block == 0) - fflags &= ~O_NONBLOCK; - else - fflags |= O_NONBLOCK; - if ((fcntl(fd, F_SETFL, fflags)) < 0) - return -1; - return 0; + int fflags; + if ((fflags = fcntl(fd, F_GETFL)) < 0) + return -1; + if (block == 0) + fflags &= ~O_NONBLOCK; + else + fflags |= O_NONBLOCK; + if ((fcntl(fd, F_SETFL, fflags)) < 0) + return -1; + return 0; } static inline int rexec_set_inherit(int fd, bool inherit) { - int fflags; - if ((fflags = fcntl(fd, F_GETFD)) < 0) - return -1; - if (inherit) - fflags &= ~FD_CLOEXEC; - else - fflags |= FD_CLOEXEC; - if ((fcntl(fd, F_SETFD, fflags)) < 0) - return -1; - return 0; + int fflags; + if ((fflags = fcntl(fd, F_GETFD)) < 0) + return -1; + if (inherit) + fflags &= ~FD_CLOEXEC; + else + fflags |= FD_CLOEXEC; + if ((fcntl(fd, F_SETFD, fflags)) < 0) + return -1; + return 0; } #define rexec_log(info, ...) \ diff --git a/qtfs/rexec/rexec_server.c b/qtfs/rexec/rexec_server.c index 0c4b3c193398136716c37e1b11a9e1371a6e6501..c8b686b374eaad9867fcf0acb52943475616d4fc 100644 --- a/qtfs/rexec/rexec_server.c +++ b/qtfs/rexec/rexec_server.c @@ -45,8 +45,8 @@ static GHashTable *child_hash = NULL; #define REXEC_WHITELIST_MAX_ITEMS 256 struct rexec_white_list_str { - int wl_nums; - char *wl[REXEC_WHITELIST_MAX_ITEMS]; + int wl_nums; + char *wl[REXEC_WHITELIST_MAX_ITEMS]; }; static struct rexec_white_list_str rexec_wl; @@ -57,501 +57,501 @@ int rexec_hash_lookup_direct(GHashTable *table, int key); void rexec_hash_remove_direct(GHashTable *table, int key); struct rexec_event { - int fd; - union { - int pid; - int connfd; - }; - int (*handler)(struct rexec_event *); + int fd; + union { + int pid; + int connfd; + }; + int (*handler)(struct rexec_event *); }; static int rexec_add_event(int efd, int fd, int pid, int (*handler)(struct rexec_event *)) { - struct rexec_event *event = (struct rexec_event *)malloc(sizeof(struct rexec_event)); - if (event == NULL) { - rexec_err("malloc failed."); - return -1; - } - event->fd = fd; - event->pid = pid; - event->handler = handler; - struct epoll_event evt; - evt.data.ptr = (void *)event; - evt.events = EPOLLIN; - if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, event->fd, &evt)) { - rexec_err("epoll ctl add fd:%d event failed.", event->fd); - free(event); - return -1; - } - return 0; + struct rexec_event *event = (struct rexec_event *)malloc(sizeof(struct rexec_event)); + if (event == NULL) { + rexec_err("malloc failed."); + return -1; + } + event->fd = fd; + event->pid = pid; + event->handler = handler; + struct epoll_event evt; + evt.data.ptr = (void *)event; + evt.events = EPOLLIN; + if (-1 == epoll_ctl(efd, EPOLL_CTL_ADD, event->fd, &evt)) { + rexec_err("epoll ctl add fd:%d event failed.", event->fd); + free(event); + return -1; + } + return 0; } static int rexec_del_event(int efd, struct rexec_event *event) { - int ret = epoll_ctl(efd, EPOLL_CTL_DEL, event->fd, NULL); - if (ret != 0) { - rexec_err("failed to delete event fd:%d.", event->fd); - } else { - rexec_log("event fd:%d deleted.", event->fd); - } - close(event->fd); - free(event); - return 0; + int ret = epoll_ctl(efd, EPOLL_CTL_DEL, event->fd, NULL); + if (ret != 0) { + rexec_err("failed to delete event fd:%d.", event->fd); + } else { + rexec_log("event fd:%d deleted.", event->fd); + } + close(event->fd); + free(event); + return 0; } static int rexec_event_process_manage(struct rexec_event *event) { - struct rexec_msg head; - int ret = recv(event->fd, &head, sizeof(struct rexec_msg), MSG_WAITALL); - if (ret <= 0) { - rexec_log("Event fd:%d recv ret:%d errno:%d, peer rexec closed, kill the associated process:%d.", - event->fd, ret, errno, event->pid); - kill(event->pid, SIGKILL); - return REXEC_EVENT_DEL; - } - rexec_err("Recv msg from client, msgtype:%d msglen:%d argc:%d stdno:%d", - head.msgtype, head.msglen, head.argc, head.stdno); - return REXEC_EVENT_OK; + struct rexec_msg head; + int ret = recv(event->fd, &head, sizeof(struct rexec_msg), MSG_WAITALL); + if (ret <= 0) { + rexec_log("Event fd:%d recv ret:%d errno:%d, peer rexec closed, kill the associated process:%d.", + event->fd, ret, errno, event->pid); + kill(event->pid, SIGKILL); + return REXEC_EVENT_DEL; + } + rexec_err("Recv msg from client, msgtype:%d msglen:%d argc:%d stdno:%d", + head.msgtype, head.msglen, head.argc, head.stdno); + return REXEC_EVENT_OK; } static int rexec_event_handshake(struct rexec_event *event) { - int sonpid = 0; - int ret = read(event->fd, &sonpid, sizeof(int)); - if (ret <= 0) { - rexec_err("Rexec read from pipe ret:%d errno:%d", ret, errno); - return REXEC_EVENT_DEL; - } - int connfd = event->connfd; - if (sonpid == -1) { - rexec_err("Handshake recv -1, dont add to process manage"); - close(connfd); - return REXEC_EVENT_DEL; - } - rexec_log("Rexec recv son pid:%d, connfd:%d", sonpid, connfd); - - rexec_hash_insert_direct(child_hash, sonpid, connfd); - rexec_add_event(main_epoll_fd, connfd, sonpid, rexec_event_process_manage); - - // 成功后同样要删除这个pipe监听事件，删除时会close掉fd - return REXEC_EVENT_DEL; + int sonpid = 0; + int ret = read(event->fd, &sonpid, sizeof(int)); + if (ret <= 0) { + rexec_err("Rexec read from pipe ret:%d errno:%d", ret, errno); + return REXEC_EVENT_DEL; + } + int connfd = event->connfd; + if (sonpid == -1) { + rexec_err("Handshake recv -1, dont add to process manage"); + close(connfd); + return REXEC_EVENT_DEL; + } + rexec_log("Rexec recv son pid:%d, connfd:%d", sonpid, connfd); + + rexec_hash_insert_direct(child_hash, sonpid, connfd); + rexec_add_event(main_epoll_fd, connfd, sonpid, rexec_event_process_manage); + + // 成功后同样要删除这个pipe监听事件，删除时会close掉fd + return REXEC_EVENT_DEL; } static void rexec_dup_std(int fd, int stdno) { - if (stdno < REXEC_STDIN || stdno > REXEC_STDERR) { - return; - } - dup2(fd, stdno - REXEC_STDIN); - close(fd); - return; + if (stdno < REXEC_STDIN || stdno > REXEC_STDERR) { + return; + } + dup2(fd, stdno - REXEC_STDIN); + close(fd); + return; } // argv list: [0]binary,[1]-f,[2]*json_str,[3]arg1,[4]arg2,... static int rexec_parse_argv(int argc, char *argv_str, char *argv[]) { - int offset = 0; - for (int i = 0; i < argc; i++) { - argv[i] = &argv_str[offset]; - offset += strlen(argv[i]) + 1; - } - argv[argc] = NULL; - return offset; + int offset = 0; + for (int i = 0; i < argc; i++) { + argv[i] = &argv_str[offset]; + offset += strlen(argv[i]) + 1; + } + argv[argc] = NULL; + return offset; } static inline void rexec_clear_string_tail(char *str, int len) { - while (len >= 0 && str[len] < 0x20) { - str[len] = '\0'; - len--; - } - return; + while (len >= 0 && str[len] < 0x20) { + str[len] = '\0'; + len--; + } + return; } #define REXEC_WHITELIST_FILE "/etc/rexec/whitelist" static int rexec_whitelist_build(struct rexec_white_list_str *wl) { - if (access(REXEC_WHITELIST_FILE, F_OK) != 0) { - rexec_err("Please configure the white list(%s).", REXEC_WHITELIST_FILE); - return -1; - } + if (access(REXEC_WHITELIST_FILE, F_OK) != 0) { + rexec_err("Please configure the white list(%s).", REXEC_WHITELIST_FILE); + return -1; + } - wl->wl_nums = 0; - memset(wl->wl, 0, sizeof(uintptr_t) * REXEC_WHITELIST_MAX_ITEMS); + wl->wl_nums = 0; + memset(wl->wl, 0, sizeof(uintptr_t) * REXEC_WHITELIST_MAX_ITEMS); #define MAX_CMD_LEN 256 - char cmd[MAX_CMD_LEN]; - FILE *fwl = fopen(REXEC_WHITELIST_FILE, "r"); - if (fwl == NULL) { - rexec_err("open white list file:%s failed.", REXEC_WHITELIST_FILE); - return -1; - } - struct stat stats; - int ret = fstat(fileno(fwl), &stats); - if (ret != 0) { - rexec_err("fstat white list file:%s failed.", REXEC_WHITELIST_FILE); - goto err_end; - } - if (stats.st_mode & 0777 != 0400) { - rexec_err("white list file:%s permissions(%o) error, must be read-only(0400)", stats.st_mode, REXEC_WHITELIST_FILE); - goto err_end; - } - while (!feof(fwl) && wl->wl_nums < REXEC_WHITELIST_MAX_ITEMS) { - int len; - char *fstr; - memset(cmd, 0, MAX_CMD_LEN); - fstr = fgets(cmd, MAX_CMD_LEN - 1, fwl); - if (fstr == NULL) - continue; - rexec_clear_string_tail(cmd, strlen(cmd)); - len = strlen(cmd); - fstr = (char *)malloc(len + 1); - if (fstr == NULL) { - rexec_err("Malloc failed"); - goto err_end; - } - memset(fstr, 0, len + 1); - memcpy(fstr, cmd, len); - wl->wl[wl->wl_nums] = fstr; - wl->wl_nums++; - rexec_log("Cmd:<%s> added to white list.", cmd); - } - fclose(fwl); - if (wl->wl_nums == 0) { - rexec_err("White list file [%s] has no valid content.", REXEC_WHITELIST_FILE); - return -1; - } - return 0; + char cmd[MAX_CMD_LEN]; + FILE *fwl = fopen(REXEC_WHITELIST_FILE, "r"); + if (fwl == NULL) { + rexec_err("open white list file:%s failed.", REXEC_WHITELIST_FILE); + return -1; + } + struct stat stats; + int ret = fstat(fileno(fwl), &stats); + if (ret != 0) { + rexec_err("fstat white list file:%s failed.", REXEC_WHITELIST_FILE); + goto err_end; + } + if (stats.st_mode & 0777 != 0400) { + rexec_err("white list file:%s permissions(%o) error, must be read-only(0400)", stats.st_mode, REXEC_WHITELIST_FILE); + goto err_end; + } + while (!feof(fwl) && wl->wl_nums < REXEC_WHITELIST_MAX_ITEMS) { + int len; + char *fstr; + memset(cmd, 0, MAX_CMD_LEN); + fstr = fgets(cmd, MAX_CMD_LEN - 1, fwl); + if (fstr == NULL) + continue; + rexec_clear_string_tail(cmd, strlen(cmd)); + len = strlen(cmd); + fstr = (char *)malloc(len + 1); + if (fstr == NULL) { + rexec_err("Malloc failed"); + goto err_end; + } + memset(fstr, 0, len + 1); + memcpy(fstr, cmd, len); + wl->wl[wl->wl_nums] = fstr; + wl->wl_nums++; + rexec_log("Cmd:<%s> added to white list.", cmd); + } + fclose(fwl); + if (wl->wl_nums == 0) { + rexec_err("White list file [%s] has no valid content.", REXEC_WHITELIST_FILE); + return -1; + } + return 0; err_end: - for (int i = 0; i < wl->wl_nums; i++) { - free(wl->wl[i]); - } - fclose(fwl); - return -1; + for (int i = 0; i < wl->wl_nums; i++) { + free(wl->wl[i]); + } + fclose(fwl); + return -1; } static void rexec_white_list_free(struct rexec_white_list_str *wl) { - for (int i = 0; i < wl->wl_nums; i++) { - free(wl->wl[i]); - } - return; + for (int i = 0; i < wl->wl_nums; i++) { + free(wl->wl[i]); + } + return; } static int rexec_whitelist_check(char *binary) { - // white list file not exist, always ok - if (access(REXEC_WHITELIST_FILE, F_OK) != 0) - return -1; - for (int i = 0; i < rexec_wl.wl_nums; i++) { - if (strlen(binary) == strlen(rexec_wl.wl[i]) && strcmp(binary, rexec_wl.wl[i]) == 0) - return 0; - } - return -1; + // white list file not exist, always ok + if (access(REXEC_WHITELIST_FILE, F_OK) != 0) + return -1; + for (int i = 0; i < rexec_wl.wl_nums; i++) { + if (strlen(binary) == strlen(rexec_wl.wl[i]) && strcmp(binary, rexec_wl.wl[i]) == 0) + return 0; + } + return -1; } #define IS_VALID_FD(fd) (fd > STDERR_FILENO) static void rexec_server_sig_chld(int num) { - int status; - pid_t pid; - while ((pid = waitpid(0, &status, WNOHANG)) > 0) { - int exit_status = status; - if (WIFEXITED(status)) { - exit_status = WEXITSTATUS(status); - } else if (WIFSIGNALED(status)) { - exit_status = WTERMSIG(status) + 128; - } - int connfd = rexec_hash_lookup_direct(child_hash, pid); - if (IS_VALID_FD(connfd)) { - struct rexec_msg head; - head.msgtype = REXEC_KILL; - head.msglen = 0; - head.exit_status = exit_status; - rexec_sendmsg(connfd, (char *)&head, sizeof(struct rexec_msg), -1); - rexec_hash_remove_direct(child_hash, pid); - // don't close connfd - } - } - return; + int status; + pid_t pid; + while ((pid = waitpid(0, &status, WNOHANG)) > 0) { + int exit_status = status; + if (WIFEXITED(status)) { + exit_status = WEXITSTATUS(status); + } else if (WIFSIGNALED(status)) { + exit_status = WTERMSIG(status) + 128; + } + int connfd = rexec_hash_lookup_direct(child_hash, pid); + if (IS_VALID_FD(connfd)) { + struct rexec_msg head; + head.msgtype = REXEC_KILL; + head.msglen = 0; + head.exit_status = exit_status; + rexec_sendmsg(connfd, (char *)&head, sizeof(struct rexec_msg), -1); + rexec_hash_remove_direct(child_hash, pid); + // don't close connfd + } + } + return; } static void rexec_server_sig_pipe(int signum) { - return; + return; } #define REXEC_MSG_NORMAL (1 << 3) #define REXEC_MSG_OVER 0xf static int rexec_start_new_process(int newconnfd) { - int ret; - int pipefd[2]; - if (pipe(pipefd) == -1) { - rexec_err("pipe syscall error, errno:%d", errno); - return -1; - } - // handshake阶段，联合体里面记录newconnfd - // 等到handshake成功后，新的事件监听这个newconnfd，联合体改为记录son pid - rexec_add_event(main_epoll_fd, pipefd[PIPE_READ], newconnfd, rexec_event_handshake); - - int pid = fork(); - // parent - if (pid != 0) { - close(pipefd[PIPE_WRITE]); - return 0; - } - // son - close(pipefd[PIPE_READ]); - - struct rexec_msg head; - int argc; - char *msgbuf = NULL; - char msg_bit = 0; - while (msg_bit != REXEC_MSG_OVER) { - int scmfd = -1; - int len = sizeof(struct rexec_msg); - memset(&head, 0, sizeof(struct rexec_msg)); - ret = rexec_recvmsg(newconnfd, (char *)&head, len, &scmfd, MSG_WAITALL); - if (ret <= 0) { - rexec_log("recvmsg ret:%d, errno:%d", ret, errno); - goto err_to_parent; - } - // 将管道与自己的标准输入输出关联 - rexec_dup_std(scmfd, head.stdno); - if (head.stdno >= REXEC_STDIN && head.stdno <= REXEC_STDERR) { - msg_bit |= (1 << (head.stdno - REXEC_STDIN)); - } - if (head.msglen == 0) - continue; - // 普通消息，代码暂时没考虑多个普通消息的，先直接过滤 - if (msgbuf != NULL || head.msgtype != REXEC_EXEC) { - rexec_err("not support multi normal msg or msgtype:%d msglen:%d invalid", head.msgtype, head.msglen); - continue; - } - msg_bit |= REXEC_MSG_NORMAL; - // exec msg - rexec_log("Exec msgtype:0x%x msglen:%d argc:%d stdno:%d", - head.msgtype, head.msglen, head.argc, head.stdno); - argc = head.argc; - if (head.msglen > REXEC_MSG_MAX || argc > REXEC_MSG_MAX / sizeof(uintptr_t) || - head.msglen <= 0 || argc < 0) { - rexec_err("msg len:%d or argc:%d is too large", head.msglen, argc); - goto err_to_parent; - } - msgbuf = (char *)malloc(head.msglen + 1); - if (msgbuf == NULL) { - rexec_err("malloc failed"); - goto err_to_parent; - } - memset(msgbuf, 0, head.msglen + 1); - ret = recv(newconnfd, msgbuf, head.msglen, MSG_WAITALL); - if (ret <= 0) { - rexec_err("recv failed, ret:%d errno:%d", ret, errno); - goto err_free; - } - rexec_log("recv normal msg len:%d headlen:%d real recv:%d msg:%s", - head.msglen, sizeof(struct rexec_msg), ret, msgbuf); - } - - // msg is like: "binary"\0"argv[1]"\0"argv[2]"\0"..." - char *binary = msgbuf; - if (rexec_whitelist_check(binary) != 0) { - rexec_err("Cmd:<%s> not in white list.", binary); - goto err_free; - } - - char *ack; - int mypid = getpid(); - char msg[sizeof(struct rexec_msg) + 1]; - struct rexec_msg *pm = msg; - pm->msgtype = REXEC_PIDMAP; - pm->msglen = 0; - pm->pid = mypid; - ret = write(newconnfd, pm, sizeof(struct rexec_msg)); - if (ret <= 0) { - rexec_err("Rexec send son pid:%d to client failed, ret:%d errno:%d", mypid, ret, errno); - } else { + int ret; + int pipefd[2]; + if (pipe(pipefd) == -1) { + rexec_err("pipe syscall error, errno:%d", errno); + return -1; + } + // handshake阶段，联合体里面记录newconnfd + // 等到handshake成功后，新的事件监听这个newconnfd，联合体改为记录son pid + rexec_add_event(main_epoll_fd, pipefd[PIPE_READ], newconnfd, rexec_event_handshake); + + int pid = fork(); + // parent + if (pid != 0) { + close(pipefd[PIPE_WRITE]); + return 0; + } + // son + close(pipefd[PIPE_READ]); + + struct rexec_msg head; + int argc; + char *msgbuf = NULL; + char msg_bit = 0; + while (msg_bit != REXEC_MSG_OVER) { + int scmfd = -1; + int len = sizeof(struct rexec_msg); + memset(&head, 0, sizeof(struct rexec_msg)); + ret = rexec_recvmsg(newconnfd, (char *)&head, len, &scmfd, MSG_WAITALL); + if (ret <= 0) { + rexec_log("recvmsg ret:%d, errno:%d", ret, errno); + goto err_to_parent; + } + // 将管道与自己的标准输入输出关联 + rexec_dup_std(scmfd, head.stdno); + if (head.stdno >= REXEC_STDIN && head.stdno <= REXEC_STDERR) { + msg_bit |= (1 << (head.stdno - REXEC_STDIN)); + } + if (head.msglen == 0) + continue; + // 普通消息，代码暂时没考虑多个普通消息的，先直接过滤 + if (msgbuf != NULL || head.msgtype != REXEC_EXEC) { + rexec_err("not support multi normal msg or msgtype:%d msglen:%d invalid", head.msgtype, head.msglen); + continue; + } + msg_bit |= REXEC_MSG_NORMAL; + // exec msg + rexec_log("Exec msgtype:0x%x msglen:%d argc:%d stdno:%d", + head.msgtype, head.msglen, head.argc, head.stdno); + argc = head.argc; + if (head.msglen > REXEC_MSG_MAX || argc > REXEC_MSG_MAX / sizeof(uintptr_t) || + head.msglen <= 0 || argc < 0) { + rexec_err("msg len:%d or argc:%d is too large", head.msglen, argc); + goto err_to_parent; + } + msgbuf = (char *)malloc(head.msglen + 1); + if (msgbuf == NULL) { + rexec_err("malloc failed"); + goto err_to_parent; + } + memset(msgbuf, 0, head.msglen + 1); + ret = recv(newconnfd, msgbuf, head.msglen, MSG_WAITALL); + if (ret <= 0) { + rexec_err("recv failed, ret:%d errno:%d", ret, errno); + goto err_free; + } + rexec_log("recv normal msg len:%d headlen:%d real recv:%d msg:%s", + head.msglen, sizeof(struct rexec_msg), ret, msgbuf); + } + + // msg is like: "binary"\0"argv[1]"\0"argv[2]"\0"..." + char *binary = msgbuf; + if (rexec_whitelist_check(binary) != 0) { + rexec_err("Cmd:<%s> not in white list.", binary); + goto err_free; + } + + char *ack; + int mypid = getpid(); + char msg[sizeof(struct rexec_msg) + 1]; + struct rexec_msg *pm = msg; + pm->msgtype = REXEC_PIDMAP; + pm->msglen = 0; + pm->pid = mypid; + ret = write(newconnfd, pm, sizeof(struct rexec_msg)); + if (ret <= 0) { + rexec_err("Rexec send son pid:%d to client failed, ret:%d errno:%d", mypid, ret, errno); + } else { retry: - rexec_log("Waiting for rexec client handshake..."); - ret = read(newconnfd, pm, sizeof(struct rexec_msg) + 1); - if (ret <= 0) { - rexec_err("Recv handshake failed, ret:%d err:%d", ret, errno); - goto err_to_parent; - } - if (pm->msgtype != REXEC_HANDSHAKE) { - rexec_err("Recv unexpected msg:%d", pm->msgtype); - goto retry; - } - ack = pm->msg; - if (*ack != '1') { - rexec_err("recv error handshake ack from client:%c, exit now", *ack); - goto err_to_parent; - } - } - // 写会PID必须放在基于newconnfd接收完所有消息之后， - // 后面newconnfd的控制权交回父进程rexec server服务进程 - if (write(pipefd[PIPE_WRITE], &mypid, sizeof(int)) <= 0) { - rexec_err("write pid to parent failed, pipefd:%d.", pipefd[PIPE_WRITE]); - } - // 子进程不再使用pipe write和connfd - close(pipefd[PIPE_WRITE]); - close(newconnfd); - - rexec_log("handshake over normaly, continue to exec new process:%s.", binary); - - // rexec_shim_entry argv like: - // argv[0]: binary - // argv[1]: -f - // argv[2]: *json_str - // argv[3]: param list 1 - // argv[4]: ... - char **argv = (char **)malloc(sizeof(uintptr_t) * (argc + 3)); - if (argv == NULL) { - rexec_err("malloc failed, argc:%d.", argc); - goto err_free; - } - int offset = rexec_parse_argv(argc, msgbuf, &argv[2]); - argv[0] = "-f"; - argv[1] = &msgbuf[offset]; - - rexec_log("Parse argv result argc:%d", argc); - for (int i = 2; i < argc + 2; i++) { - rexec_log(" argv[%d]:%s", i - 2, argv[i]); - } - ret = rexec_shim_entry(argc + 2, argv); - perror("rexec shim entry"); + rexec_log("Waiting for rexec client handshake..."); + ret = read(newconnfd, pm, sizeof(struct rexec_msg) + 1); + if (ret <= 0) { + rexec_err("Recv handshake failed, ret:%d err:%d", ret, errno); + goto err_to_parent; + } + if (pm->msgtype != REXEC_HANDSHAKE) { + rexec_err("Recv unexpected msg:%d", pm->msgtype); + goto retry; + } + ack = pm->msg; + if (*ack != '1') { + rexec_err("recv error handshake ack from client:%c, exit now", *ack); + goto err_to_parent; + } + } + // 写会PID必须放在基于newconnfd接收完所有消息之后， + // 后面newconnfd的控制权交回父进程rexec server服务进程 + if (write(pipefd[PIPE_WRITE], &mypid, sizeof(int)) <= 0) { + rexec_err("write pid to parent failed, pipefd:%d.", pipefd[PIPE_WRITE]); + } + // 子进程不再使用pipe write和connfd + close(pipefd[PIPE_WRITE]); + close(newconnfd); + + rexec_log("handshake over normaly, continue to exec new process:%s.", binary); + + // rexec_shim_entry argv like: + // argv[0]: binary + // argv[1]: -f + // argv[2]: *json_str + // argv[3]: param list 1 + // argv[4]: ... + char **argv = (char **)malloc(sizeof(uintptr_t) * (argc + 3)); + if (argv == NULL) { + rexec_err("malloc failed, argc:%d.", argc); + goto err_free; + } + int offset = rexec_parse_argv(argc, msgbuf, &argv[2]); + argv[0] = "-f"; + argv[1] = &msgbuf[offset]; + + rexec_log("Parse argv result argc:%d", argc); + for (int i = 2; i < argc + 2; i++) { + rexec_log(" argv[%d]:%s", i - 2, argv[i]); + } + ret = rexec_shim_entry(argc + 2, argv); + perror("rexec shim entry"); err_free: - free(msgbuf); + free(msgbuf); err_to_parent: - do { - int errpid = -1; - write(pipefd[PIPE_WRITE], &errpid, sizeof(int)); - } while (0); + do { + int errpid = -1; + write(pipefd[PIPE_WRITE], &errpid, sizeof(int)); + } while (0); - exit(0); + exit(0); } // 道生一 static int rexec_event_new_process(struct rexec_event *event) { - int newconnfd = rexec_sock_step_accept(event->fd); - if (newconnfd < 0) { - rexec_err("Accept failed, ret:%d errno:%d", newconnfd, errno); - return REXEC_EVENT_ERR; - } - // 主进程只负责接收新链接，基于newconnfd的新消息由子进程自己去接收，但是最 - // 后父进程要进入监听此链接的状态，是为了联动kill（对端杀死client进程则本端 - // 也杀死，或者本端杀死子进程后消息通知对端也杀死） - // 这个fd不能同时被父子进程监听，所以先建立一个pipe，等子进程完全接收完 - // 初始消息后，通过pipe告知父进程再由父进程接管newconnfd，在这之前，父进程 - // 监听pipe的read端 - // 白名单也在子进程里做，在fork之后，rexec代码控制范围 - rexec_log("Start new process new conn fd:%d", newconnfd); - rexec_start_new_process(newconnfd); - return REXEC_EVENT_OK; + int newconnfd = rexec_sock_step_accept(event->fd); + if (newconnfd < 0) { + rexec_err("Accept failed, ret:%d errno:%d", newconnfd, errno); + return REXEC_EVENT_ERR; + } + // 主进程只负责接收新链接，基于newconnfd的新消息由子进程自己去接收，但是最 + // 后父进程要进入监听此链接的状态，是为了联动kill（对端杀死client进程则本端 + // 也杀死，或者本端杀死子进程后消息通知对端也杀死） + // 这个fd不能同时被父子进程监听，所以先建立一个pipe，等子进程完全接收完 + // 初始消息后，通过pipe告知父进程再由父进程接管newconnfd，在这之前，父进程 + // 监听pipe的read端 + // 白名单也在子进程里做，在fork之后，rexec代码控制范围 + rexec_log("Start new process new conn fd:%d", newconnfd); + rexec_start_new_process(newconnfd); + return REXEC_EVENT_OK; } static void rexec_server_mainloop() { #define REXEC_MAX_EVENTS 16 - main_epoll_fd = epoll_create1(0); - if (main_epoll_fd == -1) { - rexec_err("epoll create1 failed, errno:%d.", errno); - return; - } - if (rexec_set_inherit(main_epoll_fd, false) < 0) { - rexec_err("epoll fd set inherit to false failed."); - } - struct rexec_conn_arg ser = { - .cs = REXEC_SOCK_SERVER, - .udstype = SOCK_STREAM, - }; - strncpy(ser.sun_path, REXEC_UDS_CONN, strlen(REXEC_UDS_CONN) + 1); - int buildret = rexec_build_unix_connection(&ser); - if (buildret != 0) { - rexec_err("faild to build main sock:%d errno:%d", buildret, errno); - close(main_epoll_fd); - return; - } - if (chmod(REXEC_UDS_CONN, 0600) < 0) { - rexec_err("failed to set uds sock file mode:%s errno:%d", REXEC_UDS_CONN, errno); - close(main_epoll_fd); - close(ser.sockfd); - return; - } - if (rexec_set_inherit(ser.sockfd, false) < 0) { - rexec_err("cs conn fd fd set inherit to false failed."); - } - rexec_add_event(main_epoll_fd, ser.sockfd, 0, rexec_event_new_process); - - struct epoll_event *evts = calloc(REXEC_MAX_EVENTS, sizeof(struct epoll_event)); - if (evts == NULL) { - rexec_err("init calloc evts failed."); - goto end; - } - while (1) { - int n = epoll_wait(main_epoll_fd, evts, REXEC_MAX_EVENTS, 1000); - if (n == 0) - continue; - if (n < 0) { - rexec_err("epoll wait return errcode:%d", n); - continue; - } - rexec_log("epoll wait trigger %d events.", n); - for (int i = 0; i < n; i++) { - struct rexec_event *event = (struct rexec_event *)evts[i].data.ptr; - int ret = event->handler(event); - if (ret == REXEC_EVENT_DEL) - rexec_del_event(main_epoll_fd, event); - } - } - free(evts); + main_epoll_fd = epoll_create1(0); + if (main_epoll_fd == -1) { + rexec_err("epoll create1 failed, errno:%d.", errno); + return; + } + if (rexec_set_inherit(main_epoll_fd, false) < 0) { + rexec_err("epoll fd set inherit to false failed."); + } + struct rexec_conn_arg ser = { + .cs = REXEC_SOCK_SERVER, + .udstype = SOCK_STREAM, + }; + strncpy(ser.sun_path, REXEC_UDS_CONN, strlen(REXEC_UDS_CONN) + 1); + int buildret = rexec_build_unix_connection(&ser); + if (buildret != 0) { + rexec_err("faild to build main sock:%d errno:%d", buildret, errno); + close(main_epoll_fd); + return; + } + if (chmod(REXEC_UDS_CONN, 0600) < 0) { + rexec_err("failed to set uds sock file mode:%s errno:%d", REXEC_UDS_CONN, errno); + close(main_epoll_fd); + close(ser.sockfd); + return; + } + if (rexec_set_inherit(ser.sockfd, false) < 0) { + rexec_err("cs conn fd fd set inherit to false failed."); + } + rexec_add_event(main_epoll_fd, ser.sockfd, 0, rexec_event_new_process); + + struct epoll_event *evts = calloc(REXEC_MAX_EVENTS, sizeof(struct epoll_event)); + if (evts == NULL) { + rexec_err("init calloc evts failed."); + goto end; + } + while (1) { + int n = epoll_wait(main_epoll_fd, evts, REXEC_MAX_EVENTS, 1000); + if (n == 0) + continue; + if (n < 0) { + rexec_err("epoll wait return errcode:%d", n); + continue; + } + rexec_log("epoll wait trigger %d events.", n); + for (int i = 0; i < n; i++) { + struct rexec_event *event = (struct rexec_event *)evts[i].data.ptr; + int ret = event->handler(event); + if (ret == REXEC_EVENT_DEL) + rexec_del_event(main_epoll_fd, event); + } + } + free(evts); end: - close(main_epoll_fd); - close(ser.sockfd); - return; + close(main_epoll_fd); + close(ser.sockfd); + return; } // hash map for child pid and conn fd int rexec_pid_hashmap_init(GHashTable **table) { - *table = g_hash_table_new(g_direct_hash, g_direct_equal); - if (*table == NULL) { - rexec_err("Init child pid hashmap failed."); - return -1; - } - return 0; + *table = g_hash_table_new(g_direct_hash, g_direct_equal); + if (*table == NULL) { + rexec_err("Init child pid hashmap failed."); + return -1; + } + return 0; } void rexec_pid_hashmap_destroy(GHashTable *table) { - g_hash_table_destroy(table); - return; + g_hash_table_destroy(table); + return; } #pragma GCC diagnostic ignored "-Wint-to-pointer-cast" #pragma GCC diagnostic ignored "-Wpointer-to-int-cast" int rexec_hash_insert_direct(GHashTable *table, int key, int value) { - if (g_hash_table_insert(table, (gpointer)key, (gpointer)value) == 0) { - rexec_err("Hash table key:%d value:%d is already exist, update it.", key, value); - } - return 0; + if (g_hash_table_insert(table, (gpointer)key, (gpointer)value) == 0) { + rexec_err("Hash table key:%d value:%d is already exist, update it.", key, value); + } + return 0; } int rexec_hash_lookup_direct(GHashTable *table, int key) { - return (int)g_hash_table_lookup(table, (gpointer)key); + return (int)g_hash_table_lookup(table, (gpointer)key); } void rexec_hash_remove_direct(GHashTable *table, int key) { - g_hash_table_remove(table, (gpointer)key); - return; + g_hash_table_remove(table, (gpointer)key); + return; } #pragma GCC diagnostic pop @@ -566,30 +566,30 @@ int check_socket_lock(void) int main(int argc, char *argv[]) { - mode_t newmask = 0077; - rexec_log("Change umask from:%o to %o", umask(newmask), newmask); - rexec_log_init(); - signal(SIGCHLD, rexec_server_sig_chld); - signal(SIGPIPE, rexec_server_sig_pipe); - if (rexec_whitelist_build(&rexec_wl) != 0) { - return -1; - } - if (access(REXEC_RUN_PATH, F_OK) != 0) { - mkdir(REXEC_RUN_PATH, 0700); - } - - if (check_socket_lock() < 0) { + mode_t newmask = 0077; + rexec_log("Change umask from:%o to %o", umask(newmask), newmask); + rexec_log_init(); + signal(SIGCHLD, rexec_server_sig_chld); + signal(SIGPIPE, rexec_server_sig_pipe); + if (rexec_whitelist_build(&rexec_wl) != 0) { + return -1; + } + if (access(REXEC_RUN_PATH, F_OK) != 0) { + mkdir(REXEC_RUN_PATH, 0700); + } + + if (check_socket_lock() < 0) { return -1; - } - - if (rexec_pid_hashmap_init(&child_hash) != 0) { - rexec_white_list_free(&rexec_wl); - return -1; - } - rexec_server_mainloop(); - rexec_pid_hashmap_destroy(child_hash); - fclose(rexec_logfile); - rexec_logfile = NULL; - return 0; + } + + if (rexec_pid_hashmap_init(&child_hash) != 0) { + rexec_white_list_free(&rexec_wl); + return -1; + } + rexec_server_mainloop(); + rexec_pid_hashmap_destroy(child_hash); + fclose(rexec_logfile); + rexec_logfile = NULL; + return 0; } diff --git a/qtfs/rexec/rexec_shim.c b/qtfs/rexec/rexec_shim.c index d1633015a05db7041859ebe2f8d2830c26d5a1f3..cc7fc9e10e8b78263035b130c7ded894149e3da3 100644 --- a/qtfs/rexec/rexec_shim.c +++ b/qtfs/rexec/rexec_shim.c @@ -32,126 +32,126 @@ void rshim_close_all_fd() { - DIR *dir = NULL; - struct dirent *entry; - dir = opendir("/proc/self/fd/"); - if (dir == NULL) { - rshim_err("open path:/proc/self/fd/ failed"); - return; - } - while (entry = readdir(dir)) { - int fd = atoi(entry->d_name); - if (fd <= 2) - continue; - close(fd); - } - closedir(dir); - return; + DIR *dir = NULL; + struct dirent *entry; + dir = opendir("/proc/self/fd/"); + if (dir == NULL) { + rshim_err("open path:/proc/self/fd/ failed"); + return; + } + while (entry = readdir(dir)) { + int fd = atoi(entry->d_name); + if (fd <= 2) + continue; + close(fd); + } + closedir(dir); + return; } int rshim_get_file_size(char *file) { - int size = 0; - FILE *f = fopen(file, "rb"); - if (f == NULL) { - rshim_err("File:%s fopen failed.", file); - return -1; - } - fseek(f, 0, SEEK_END); - size = ftell(f); - fclose(f); - return size; + int size = 0; + FILE *f = fopen(file, "rb"); + if (f == NULL) { + rshim_err("File:%s fopen failed.", file); + return -1; + } + fseek(f, 0, SEEK_END); + size = ftell(f); + fclose(f); + return size; } void rshim_reg_file_open(int fd_target, const char *path, int perm, int offset) { - int fd = open(path, perm); - int fd2 = -1; - if (fd < 0) { - rshim_err("Open file:%s failed, fd:%d errno:%d", path, fd, errno); - return; - } - if (fd != fd_target) { - fd2 = dup2(fd, fd_target); - if (fd2 != fd_target) { - rshim_err("Failed to open file:%s by fd:%d", path, fd_target); - close(fd2); - close(fd); - return; - } - close(fd); - } - int off = lseek(fd_target, offset, SEEK_SET); - if (off < 0) { - rshim_err("Failed to set offset:%d to file:%s, fd:%d, fd2:%d", offset, path, fd, fd2); - return; - } - rshim_log("Successed to set offset:%d to file:%s, fd:%d, fd2:%d", offset, path, fd, fd2); - return; + int fd = open(path, perm); + int fd2 = -1; + if (fd < 0) { + rshim_err("Open file:%s failed, fd:%d errno:%d", path, fd, errno); + return; + } + if (fd != fd_target) { + fd2 = dup2(fd, fd_target); + if (fd2 != fd_target) { + rshim_err("Failed to open file:%s by fd:%d", path, fd_target); + close(fd2); + close(fd); + return; + } + close(fd); + } + int off = lseek(fd_target, offset, SEEK_SET); + if (off < 0) { + rshim_err("Failed to set offset:%d to file:%s, fd:%d, fd2:%d", offset, path, fd, fd2); + return; + } + rshim_log("Successed to set offset:%d to file:%s, fd:%d, fd2:%d", offset, path, fd, fd2); + return; } void rshim_reg_file_resume(const char * const json_buf) { - struct json_object *obj_files; - struct json_object *obj_file; - struct json_object *obj_fd; - struct json_object *obj_path; - struct json_object *obj_perm; - struct json_object *obj_offset; - int fd, perm, offset; - const char *path = NULL; - int curfd = 3; // begin from 3 - struct json_object *fd_json = json_tokener_parse(json_buf); - if (fd_json == NULL) { - fprintf(stderr, "parse json error\n"); - return; - } - obj_files = json_object_object_get(fd_json, "Files"); - int arraylen = json_object_array_length(obj_files); - for (int i=0; i< arraylen; i++){ - obj_file = json_object_array_get_idx(obj_files, i); - obj_fd = json_object_object_get(obj_file, "Fd"); - fd = json_object_get_int(obj_fd); - obj_path = json_object_object_get(obj_file, "Path"); - path = json_object_get_string(obj_path); - obj_perm = json_object_object_get(obj_file, "Perm"); - perm = json_object_get_int(obj_perm); - obj_offset = json_object_object_get(obj_file, "Offset"); - offset = json_object_get_int(obj_offset); - rshim_log("Get file from json fd:%d path:%s perm:%d offset:%d", - fd, path, perm, offset); - rshim_reg_file_open(fd, path, perm, offset); - } + struct json_object *obj_files; + struct json_object *obj_file; + struct json_object *obj_fd; + struct json_object *obj_path; + struct json_object *obj_perm; + struct json_object *obj_offset; + int fd, perm, offset; + const char *path = NULL; + int curfd = 3; // begin from 3 + struct json_object *fd_json = json_tokener_parse(json_buf); + if (fd_json == NULL) { + fprintf(stderr, "parse json error\n"); + return; + } + obj_files = json_object_object_get(fd_json, "Files"); + int arraylen = json_object_array_length(obj_files); + for (int i=0; i< arraylen; i++){ + obj_file = json_object_array_get_idx(obj_files, i); + obj_fd = json_object_object_get(obj_file, "Fd"); + fd = json_object_get_int(obj_fd); + obj_path = json_object_object_get(obj_file, "Path"); + path = json_object_get_string(obj_path); + obj_perm = json_object_object_get(obj_file, "Perm"); + perm = json_object_get_int(obj_perm); + obj_offset = json_object_object_get(obj_file, "Offset"); + offset = json_object_get_int(obj_offset); + rshim_log("Get file from json fd:%d path:%s perm:%d offset:%d", + fd, path, perm, offset); + rshim_reg_file_open(fd, path, perm, offset); + } - json_object_put(fd_json); - return; + json_object_put(fd_json); + return; } /* - param list: - 1) -f xxx.json binary param1 param2 ... - 2) binary param1 param2... + param list: + 1) -f xxx.json binary param1 param2 ... + 2) binary param1 param2... */ int rexec_shim_entry(int argc, char *argv[]) { - char *json_str = NULL; - char **newarg = NULL; + char *json_str = NULL; + char **newarg = NULL; - if (strcmp(argv[0], "-f") == 0) { - json_str = argv[1]; - newarg = &argv[2]; - } else { - newarg = argv; - } + if (strcmp(argv[0], "-f") == 0) { + json_str = argv[1]; + newarg = &argv[2]; + } else { + newarg = argv; + } - rshim_close_all_fd(); + rshim_close_all_fd(); - rshim_log("Get json str:%s", json_str); + rshim_log("Get json str:%s", json_str); - rshim_reg_file_resume(json_str); - execvp(newarg[0], newarg); - perror("execvp failed."); + rshim_reg_file_resume(json_str); + execvp(newarg[0], newarg); + perror("execvp failed."); - return -1; + return -1; } diff --git a/qtfs/rexec/rexec_sock.c b/qtfs/rexec/rexec_sock.c index 888eb4b47d89d20debf5f173d18b3a5efbcb7c16..e8750c51eb0d102d5913305bfe6b3459abb84b8a 100644 --- a/qtfs/rexec/rexec_sock.c +++ b/qtfs/rexec/rexec_sock.c @@ -102,76 +102,76 @@ int rexec_sock_step_accept(int sock_fd) // or a normal msg and a SCM_RIGHTS fd int rexec_sendmsg(int sockfd, char *msgbuf, int msglen, int scmfd) { - struct msghdr msg; - struct cmsghdr *cmsg = NULL; - struct iovec iov; - char buf[CMSG_SPACE(sizeof(scmfd))]; - int ret; - - memset(&msg, 0, sizeof(msg)); - iov.iov_base = (void *)msgbuf; - iov.iov_len = msglen; - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - msg.msg_name = NULL; - msg.msg_namelen = 0; - - msg.msg_control = buf; - msg.msg_controllen = sizeof(buf); - if (scmfd > 0) { - cmsg = CMSG_FIRSTHDR(&msg); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(scmfd)); - /* Initialize the payload: */ - memcpy(CMSG_DATA(cmsg), &scmfd, sizeof(scmfd)); - msg.msg_controllen = cmsg->cmsg_len; - } else { - msg.msg_controllen = 0; - } - if ((ret = sendmsg(sockfd, &msg, 0)) != iov.iov_len) { - return ret; - } - return ret; + struct msghdr msg; + struct cmsghdr *cmsg = NULL; + struct iovec iov; + char buf[CMSG_SPACE(sizeof(scmfd))]; + int ret; + + memset(&msg, 0, sizeof(msg)); + iov.iov_base = (void *)msgbuf; + iov.iov_len = msglen; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + if (scmfd > 0) { + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(scmfd)); + /* Initialize the payload: */ + memcpy(CMSG_DATA(cmsg), &scmfd, sizeof(scmfd)); + msg.msg_controllen = cmsg->cmsg_len; + } else { + msg.msg_controllen = 0; + } + if ((ret = sendmsg(sockfd, &msg, 0)) != iov.iov_len) { + return ret; + } + return ret; } int rexec_recvmsg(int sockfd, char *msgbuf, int len, int *scmfd, int flags) { - struct iovec iov; - struct msghdr msg; - int fd = -1; - struct cmsghdr *cmsg; - char buf[CMSG_SPACE(sizeof(fd))]; - - /* send at least one char */ - memset(&msg, 0, sizeof(msg)); - iov.iov_base = msgbuf; - iov.iov_len = len; - msg.msg_iov = &iov; - msg.msg_iovlen = 1; - msg.msg_name = NULL; - msg.msg_namelen = 0; - - msg.msg_control = buf; - msg.msg_controllen = sizeof(buf); - cmsg = CMSG_FIRSTHDR(&msg); - cmsg->cmsg_level = SOL_SOCKET; - cmsg->cmsg_type = SCM_RIGHTS; - cmsg->cmsg_len = CMSG_LEN(sizeof(fd)); - /* Initialize the payload: */ - memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd)); - msg.msg_controllen = cmsg->cmsg_len; - - int ret = recvmsg(sockfd, &msg, flags); - if (ret < 0) - return ret; - - cmsg = CMSG_FIRSTHDR(&msg); - if (cmsg != NULL) { - memcpy(&fd, CMSG_DATA(cmsg), sizeof(fd)); - *scmfd = fd; - } - return ret; + struct iovec iov; + struct msghdr msg; + int fd = -1; + struct cmsghdr *cmsg; + char buf[CMSG_SPACE(sizeof(fd))]; + + /* send at least one char */ + memset(&msg, 0, sizeof(msg)); + iov.iov_base = msgbuf; + iov.iov_len = len; + msg.msg_iov = &iov; + msg.msg_iovlen = 1; + msg.msg_name = NULL; + msg.msg_namelen = 0; + + msg.msg_control = buf; + msg.msg_controllen = sizeof(buf); + cmsg = CMSG_FIRSTHDR(&msg); + cmsg->cmsg_level = SOL_SOCKET; + cmsg->cmsg_type = SCM_RIGHTS; + cmsg->cmsg_len = CMSG_LEN(sizeof(fd)); + /* Initialize the payload: */ + memcpy(CMSG_DATA(cmsg), &fd, sizeof(fd)); + msg.msg_controllen = cmsg->cmsg_len; + + int ret = recvmsg(sockfd, &msg, flags); + if (ret < 0) + return ret; + + cmsg = CMSG_FIRSTHDR(&msg); + if (cmsg != NULL) { + memcpy(&fd, CMSG_DATA(cmsg), sizeof(fd)); + *scmfd = fd; + } + return ret; } diff --git a/qtfs/rexec/rexec_sock.h b/qtfs/rexec/rexec_sock.h index ea0a377c514fef97babfe3ade795607821c7ac29..c61d6bcf170946ffdf47d459f262d6ecc6079d61 100644 --- a/qtfs/rexec/rexec_sock.h +++ b/qtfs/rexec/rexec_sock.h @@ -17,8 +17,8 @@ #define __REXEC_SOCK_H__ enum { - REXEC_SOCK_CLIENT = 1, - REXEC_SOCK_SERVER, + REXEC_SOCK_CLIENT = 1, + REXEC_SOCK_SERVER, }; #define UDS_SUN_PATH_LEN 108