From 626b5ea2c506fcc31affc753ed5827e65704d2fe Mon Sep 17 00:00:00 2001 From: liqiang Date: Fri, 12 Aug 2022 10:16:54 +0800 Subject: [PATCH] qtfs baseline version Signed-off-by: liqiang --- qtfs/License | 340 +++++ qtfs/README.en.md | 36 + qtfs/README.md | 64 + qtfs/comm.h | 227 ++++ qtfs/conn.c | 908 +++++++++++++ qtfs/conn.h | 167 +++ qtfs/demo/Makefile | 12 + qtfs/demo/cfifo_r.c | 94 ++ qtfs/demo/cfifo_w.c | 30 + qtfs/doc/ Overall_architecture_diagram.png | Bin 0 -> 49089 bytes qtfs/log.h | 106 ++ qtfs/misc.c | 154 +++ qtfs/qtfs/Makefile | 14 + qtfs/qtfs/miss.c | 66 + qtfs/qtfs/ops.h | 21 + qtfs/qtfs/proc.c | 221 ++++ qtfs/qtfs/qtfs-mod.c | 269 ++++ qtfs/qtfs/qtfs-mod.h | 172 +++ qtfs/qtfs/sb.c | 1376 ++++++++++++++++++++ qtfs/qtfs/syscall.c | 526 ++++++++ qtfs/qtfs/syscall.h | 7 + qtfs/qtfs/xattr.c | 169 +++ qtfs/qtfs_server/Makefile | 18 + qtfs/qtfs_server/fsops.c | 1070 +++++++++++++++ qtfs/qtfs_server/fsops.h | 12 + qtfs/qtfs_server/qtfs-server.c | 287 ++++ qtfs/qtfs_server/qtfs-server.h | 27 + qtfs/qtfs_server/user_engine.c | 236 ++++ qtfs/qtinfo/Makefile | 12 + qtfs/qtinfo/qtinfo.c | 343 +++++ qtfs/qtinfo/qtinfo.h | 54 + qtfs/req.h | 519 ++++++++ qtfs/test/cgroup.go | 53 + 33 files changed, 7610 insertions(+) create mode 100644 qtfs/License create mode 100644 qtfs/README.en.md create mode 100644 qtfs/README.md create mode 100644 qtfs/comm.h create mode 100644 qtfs/conn.c create mode 100644 qtfs/conn.h create mode 100644 qtfs/demo/Makefile create mode 100644 qtfs/demo/cfifo_r.c create mode 100644 qtfs/demo/cfifo_w.c create mode 100644 qtfs/doc/ Overall_architecture_diagram.png create mode 100644 qtfs/log.h create mode 100644 qtfs/misc.c create mode 100644 qtfs/qtfs/Makefile create mode 100644 qtfs/qtfs/miss.c create mode 100644 qtfs/qtfs/ops.h create mode 100644 qtfs/qtfs/proc.c create mode 100644 qtfs/qtfs/qtfs-mod.c create mode 100644 qtfs/qtfs/qtfs-mod.h create mode 100644 qtfs/qtfs/sb.c create mode 100644 qtfs/qtfs/syscall.c create mode 100644 qtfs/qtfs/syscall.h create mode 100644 qtfs/qtfs/xattr.c create mode 100644 qtfs/qtfs_server/Makefile create mode 100644 qtfs/qtfs_server/fsops.c create mode 100644 qtfs/qtfs_server/fsops.h create mode 100644 qtfs/qtfs_server/qtfs-server.c create mode 100644 qtfs/qtfs_server/qtfs-server.h create mode 100644 qtfs/qtfs_server/user_engine.c create mode 100644 qtfs/qtinfo/Makefile create mode 100644 qtfs/qtinfo/qtinfo.c create mode 100644 qtfs/qtinfo/qtinfo.h create mode 100644 qtfs/req.h create mode 100644 qtfs/test/cgroup.go diff --git a/qtfs/License b/qtfs/License new file mode 100644 index 0000000..3912109 --- /dev/null +++ b/qtfs/License @@ -0,0 +1,340 @@ + GNU GENERAL PUBLIC LICENSE + Version 2, June 1991 + + Copyright (C) 1989, 1991 Free Software Foundation, Inc. + 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The licenses for most software are designed to take away your +freedom to share and change it. By contrast, the GNU General Public +License is intended to guarantee your freedom to share and change free +software--to make sure the software is free for all its users. This +General Public License applies to most of the Free Software +Foundation's software and to any other program whose authors commit to +using it. (Some other Free Software Foundation software is covered by +the GNU Library General Public License instead.) You can apply it to +your programs, too. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +this service if you wish), that you receive source code or can get it +if you want it, that you can change the software or use pieces of it +in new free programs; and that you know you can do these things. + + To protect your rights, we need to make restrictions that forbid +anyone to deny you these rights or to ask you to surrender the rights. +These restrictions translate to certain responsibilities for you if you +distribute copies of the software, or if you modify it. + + For example, if you distribute copies of such a program, whether +gratis or for a fee, you must give the recipients all the rights that +you have. You must make sure that they, too, receive or can get the +source code. And you must show them these terms so they know their +rights. + + We protect your rights with two steps: (1) copyright the software, and +(2) offer you this license which gives you legal permission to copy, +distribute and/or modify the software. + + Also, for each author's protection and ours, we want to make certain +that everyone understands that there is no warranty for this free +software. If the software is modified by someone else and passed on, we +want its recipients to know that what they have is not the original, so +that any problems introduced by others will not reflect on the original +authors' reputations. + + Finally, any free program is threatened constantly by software +patents. We wish to avoid the danger that redistributors of a free +program will individually obtain patent licenses, in effect making the +program proprietary. To prevent this, we have made it clear that any +patent must be licensed for everyone's free use or not licensed at all. + + The precise terms and conditions for copying, distribution and +modification follow. + + GNU GENERAL PUBLIC LICENSE + TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION + + 0. This License applies to any program or other work which contains +a notice placed by the copyright holder saying it may be distributed +under the terms of this General Public License. The "Program", below, +refers to any such program or work, and a "work based on the Program" +means either the Program or any derivative work under copyright law: +that is to say, a work containing the Program or a portion of it, +either verbatim or with modifications and/or translated into another +language. (Hereinafter, translation is included without limitation in +the term "modification".) Each licensee is addressed as "you". + +Activities other than copying, distribution and modification are not +covered by this License; they are outside its scope. The act of +running the Program is not restricted, and the output from the Program +is covered only if its contents constitute a work based on the +Program (independent of having been made by running the Program). +Whether that is true depends on what the Program does. + + 1. You may copy and distribute verbatim copies of the Program's +source code as you receive it, in any medium, provided that you +conspicuously and appropriately publish on each copy an appropriate +copyright notice and disclaimer of warranty; keep intact all the +notices that refer to this License and to the absence of any warranty; +and give any other recipients of the Program a copy of this License +along with the Program. + +You may charge a fee for the physical act of transferring a copy, and +you may at your option offer warranty protection in exchange for a fee. + + 2. You may modify your copy or copies of the Program or any portion +of it, thus forming a work based on the Program, and copy and +distribute such modifications or work under the terms of Section 1 +above, provided that you also meet all of these conditions: + + a) You must cause the modified files to carry prominent notices + stating that you changed the files and the date of any change. + + b) You must cause any work that you distribute or publish, that in + whole or in part contains or is derived from the Program or any + part thereof, to be licensed as a whole at no charge to all third + parties under the terms of this License. + + c) If the modified program normally reads commands interactively + when run, you must cause it, when started running for such + interactive use in the most ordinary way, to print or display an + announcement including an appropriate copyright notice and a + notice that there is no warranty (or else, saying that you provide + a warranty) and that users may redistribute the program under + these conditions, and telling the user how to view a copy of this + License. (Exception: if the Program itself is interactive but + does not normally print such an announcement, your work based on + the Program is not required to print an announcement.) + +These requirements apply to the modified work as a whole. If +identifiable sections of that work are not derived from the Program, +and can be reasonably considered independent and separate works in +themselves, then this License, and its terms, do not apply to those +sections when you distribute them as separate works. But when you +distribute the same sections as part of a whole which is a work based +on the Program, the distribution of the whole must be on the terms of +this License, whose permissions for other licensees extend to the +entire whole, and thus to each and every part regardless of who wrote it. + +Thus, it is not the intent of this section to claim rights or contest +your rights to work written entirely by you; rather, the intent is to +exercise the right to control the distribution of derivative or +collective works based on the Program. + +In addition, mere aggregation of another work not based on the Program +with the Program (or with a work based on the Program) on a volume of +a storage or distribution medium does not bring the other work under +the scope of this License. + + 3. You may copy and distribute the Program (or a work based on it, +under Section 2) in object code or executable form under the terms of +Sections 1 and 2 above provided that you also do one of the following: + + a) Accompany it with the complete corresponding machine-readable + source code, which must be distributed under the terms of Sections + 1 and 2 above on a medium customarily used for software interchange; or, + + b) Accompany it with a written offer, valid for at least three + years, to give any third party, for a charge no more than your + cost of physically performing source distribution, a complete + machine-readable copy of the corresponding source code, to be + distributed under the terms of Sections 1 and 2 above on a medium + customarily used for software interchange; or, + + c) Accompany it with the information you received as to the offer + to distribute corresponding source code. (This alternative is + allowed only for noncommercial distribution and only if you + received the program in object code or executable form with such + an offer, in accord with Subsection b above.) + +The source code for a work means the preferred form of the work for +making modifications to it. For an executable work, complete source +code means all the source code for all modules it contains, plus any +associated interface definition files, plus the scripts used to +control compilation and installation of the executable. However, as a +special exception, the source code distributed need not include +anything that is normally distributed (in either source or binary +form) with the major components (compiler, kernel, and so on) of the +operating system on which the executable runs, unless that component +itself accompanies the executable. + +If distribution of executable or object code is made by offering +access to copy from a designated place, then offering equivalent +access to copy the source code from the same place counts as +distribution of the source code, even though third parties are not +compelled to copy the source along with the object code. + + 4. You may not copy, modify, sublicense, or distribute the Program +except as expressly provided under this License. Any attempt +otherwise to copy, modify, sublicense or distribute the Program is +void, and will automatically terminate your rights under this License. +However, parties who have received copies, or rights, from you under +this License will not have their licenses terminated so long as such +parties remain in full compliance. + + 5. You are not required to accept this License, since you have not +signed it. However, nothing else grants you permission to modify or +distribute the Program or its derivative works. These actions are +prohibited by law if you do not accept this License. Therefore, by +modifying or distributing the Program (or any work based on the +Program), you indicate your acceptance of this License to do so, and +all its terms and conditions for copying, distributing or modifying +the Program or works based on it. + + 6. Each time you redistribute the Program (or any work based on the +Program), the recipient automatically receives a license from the +original licensor to copy, distribute or modify the Program subject to +these terms and conditions. You may not impose any further +restrictions on the recipients' exercise of the rights granted herein. +You are not responsible for enforcing compliance by third parties to +this License. + + 7. If, as a consequence of a court judgment or allegation of patent +infringement or for any other reason (not limited to patent issues), +conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot +distribute so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you +may not distribute the Program at all. For example, if a patent +license would not permit royalty-free redistribution of the Program by +all those who receive copies directly or indirectly through you, then +the only way you could satisfy both it and this License would be to +refrain entirely from distribution of the Program. + +If any portion of this section is held invalid or unenforceable under +any particular circumstance, the balance of the section is intended to +apply and the section as a whole is intended to apply in other +circumstances. + +It is not the purpose of this section to induce you to infringe any +patents or other property right claims or to contest validity of any +such claims; this section has the sole purpose of protecting the +integrity of the free software distribution system, which is +implemented by public license practices. Many people have made +generous contributions to the wide range of software distributed +through that system in reliance on consistent application of that +system; it is up to the author/donor to decide if he or she is willing +to distribute software through any other system and a licensee cannot +impose that choice. + +This section is intended to make thoroughly clear what is believed to +be a consequence of the rest of this License. + + 8. If the distribution and/or use of the Program is restricted in +certain countries either by patents or by copyrighted interfaces, the +original copyright holder who places the Program under this License +may add an explicit geographical distribution limitation excluding +those countries, so that distribution is permitted only in or among +countries not thus excluded. In such case, this License incorporates +the limitation as if written in the body of this License. + + 9. The Free Software Foundation may publish revised and/or new versions +of the General Public License from time to time. Such new versions will +be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + +Each version is given a distinguishing version number. If the Program +specifies a version number of this License which applies to it and "any +later version", you have the option of following the terms and conditions +either of that version or of any later version published by the Free +Software Foundation. If the Program does not specify a version number of +this License, you may choose any version ever published by the Free Software +Foundation. + + 10. If you wish to incorporate parts of the Program into other free +programs whose distribution conditions are different, write to the author +to ask for permission. For software which is copyrighted by the Free +Software Foundation, write to the Free Software Foundation; we sometimes +make exceptions for this. Our decision will be guided by the two goals +of preserving the free status of all derivatives of our free software and +of promoting the sharing and reuse of software generally. + + NO WARRANTY + + 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY +FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN +OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES +PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED +OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS +TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE +PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, +REPAIR OR CORRECTION. + + 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR +REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, +INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING +OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED +TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY +YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER +PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +convey the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + + +Also add information on how to contact you by electronic and paper mail. + +If the program is interactive, make it output a short notice like this +when it starts in an interactive mode: + + Gnomovision version 69, Copyright (C) year name of author + Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. + This is free software, and you are welcome to redistribute it + under certain conditions; type `show c' for details. + +The hypothetical commands `show w' and `show c' should show the appropriate +parts of the General Public License. Of course, the commands you use may +be called something other than `show w' and `show c'; they could even be +mouse-clicks or menu items--whatever suits your program. + +You should also get your employer (if you work as a programmer) or your +school, if any, to sign a "copyright disclaimer" for the program, if +necessary. Here is a sample; alter the names: + + Yoyodyne, Inc., hereby disclaims all copyright interest in the program + `Gnomovision' (which makes passes at compilers) written by James Hacker. + + , 1 April 1989 + Ty Coon, President of Vice + +This General Public License does not permit incorporating your program into +proprietary programs. If your program is a subroutine library, you may +consider it more useful to permit linking proprietary applications with the +library. If this is what you want to do, use the GNU Library General +Public License instead of this License. diff --git a/qtfs/README.en.md b/qtfs/README.en.md new file mode 100644 index 0000000..15010d3 --- /dev/null +++ b/qtfs/README.en.md @@ -0,0 +1,36 @@ +# qtfs + +#### Description +{**When you're done, you can delete the content in this README and update the file with details for others getting started with your repository**} + +#### Software Architecture +Software architecture description + +#### Installation + +1. xxxx +2. xxxx +3. xxxx + +#### Instructions + +1. xxxx +2. xxxx +3. xxxx + +#### Contribution + +1. Fork the repository +2. Create Feat_xxx branch +3. Commit your code +4. Create Pull Request + + +#### Gitee Feature + +1. You can use Readme\_XXX.md to support different languages, such as Readme\_en.md, Readme\_zh.md +2. Gitee blog [blog.gitee.com](https://blog.gitee.com) +3. Explore open source project [https://gitee.com/explore](https://gitee.com/explore) +4. The most valuable open source project [GVP](https://gitee.com/gvp) +5. The manual of Gitee [https://gitee.com/help](https://gitee.com/help) +6. The most popular members [https://gitee.com/gitee-stars/](https://gitee.com/gitee-stars/) diff --git a/qtfs/README.md b/qtfs/README.md new file mode 100644 index 0000000..0cbc2e1 --- /dev/null +++ b/qtfs/README.md @@ -0,0 +1,64 @@ +# qtfs + +## 介绍 + +qtfs是一个共享文件系统项目，可部署在host-dpu的硬件架构上，也可以部署在2台服务器之间。以客户端服务器的模式工作，使客户端能通过qtfs访问服务端的指定文件系统，就像访问本地文件系统一样。 + +qtfs的特性： ++ 支持挂载点传播； ++ 支持proc、sys、cgroup等特殊文件系统的共享； ++ 客户端对qtfs目录下文件的操作都被转移到服务端，文件读写可共享； ++ 支持在客户端对服务端的文件系统进行远程挂载； ++ 可以定制化处理特殊文件； ++ 支持远端fifo、unix-socket等，并且支持epoll，使客户端和服务端像本地通信一样使用这些文件； ++ 基于host-dpu架构时，底层通信方式可以支持PCIe，性能大大优于网络； ++ 内核模块形式开发，无需对内核进行侵入式修改。 + +## 软件架构 + +软件大体框架图： + +![输入图片说明](doc/%20Overall_architecture_diagram.png) + + +## 安装教程 + +目录说明： ++ **qtfs**: 客户端内核模块相关代码，直接在该目录下编译客户端ko。 ++ **qtfs_server**: 服务端内核模块相关代码，直接在该目录下编译服务端ko和相关程序。 ++ **qtinfo**: 诊断工具，支持查询文件系统的工作状态以及修改log级别等。 ++ **demo**、**test**、**doc**: 测试程序、演示程序以及项目资料等。 ++ 根目录: 是客户端与服务端都能用到的公共模块代码。 + +首先找两台服务器（或虚拟机）配置内核编译环境： + + 1. 要求内核版本在5.10或更高版本。 + 2. 安装内核开发包：yum install kernel-devel。 + +服务端安装： + + 1. cd qtfs_server + 2. make clean && make + 3. insmod qtfs_server.ko qtfs_server_ip=x.x.x.x qtfs_server_port=12345 qtfs_log_level=WARN + 4. ./engine 4096 16 + +客户端安装： + + 1. cd qtfs + 2. make clean && make + 3. insmod qtfs.ko qtfs_server_ip=x.x.x.x qtfs_server_port=12345 qtfs_log_level=WARN + +## 使用说明 + +安装完成后，客户端通过挂载把服务端的文件系统让客户端可见，例如： + + mount -t qtfs / /root/mnt/ + +客户端进入"/root/mnt"后便可查看到server端的所有文件，以及对其进行相关操作。 + +## 参与贡献 + +1. Fork 本仓库 +2. 新建 Feat_xxx 分支 +3. 提交代码 +4. 新建 Pull Request diff --git a/qtfs/comm.h b/qtfs/comm.h new file mode 100644 index 0000000..56a3572 --- /dev/null +++ b/qtfs/comm.h @@ -0,0 +1,227 @@ +#ifndef __QTFS_SERVER_COMM_H__ +#define __QTFS_SERVER_COMM_H__ + +extern struct qtinfo *qtfs_diag_info; + +#define QTFS_IOCTL_MAGIC 'Q' +enum { + _QTFS_IOCTL_EXEC, + _QTFS_IOCTL_THREAD_RUN, + _QTFS_IOCTL_EPFDSET, + _QTFS_IOCTL_EPOLLT, + _QTFS_IOCTL_EPOLL_THREAD_RUN, + _QTFS_IOCTL_EXIT, + + _QTFS_IOCTL_ALLINFO, + _QTFS_IOCTL_CLEARALL, + + _QTFS_IOCTL_LOG_LEVEL, +}; + +#define QTFS_IOCTL_THREAD_INIT _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EXEC) +#define QTFS_IOCTL_THREAD_RUN _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_THREAD_RUN) +#define QTFS_IOCTL_EPFDSET _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EPFDSET) +#define QTFS_IOCTL_EPOLL_THREAD_INIT _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EPOLLT) +#define QTFS_IOCTL_EPOLL_THREAD_RUN _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EPOLL_THREAD_RUN) +#define QTFS_IOCTL_EXIT _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_EXIT) +#define QTFS_IOCTL_ALLINFO _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_ALLINFO) +#define QTFS_IOCTL_CLEARALL _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_CLEARALL) +#define QTFS_IOCTL_LOGLEVEL _IO(QTFS_IOCTL_MAGIC, _QTFS_IOCTL_LOG_LEVEL) + +#define QTINFO_MAX_EVENT_TYPE 36 // look qtreq_type at req.h +#define QTFS_FUNCTION_LEN 64 + +#define QTFS_MAX_THREADS 16 +#define QTFS_LOGLEVEL_STRLEN 6 + +struct qtfs_server_userp_s { + size_t size; + void *userp; + void *userp2; +}; + +struct qtfs_thread_init_s { + int thread_nums; + struct qtfs_server_userp_s *userp; +}; + +struct qtreq_epoll_event { + unsigned int events; + unsigned long data; +}; + +struct qtfs_server_epoll_s { + int epfd; + int event_nums; + struct epoll_event *events; + struct epoll_event *kevents; +}; + +enum qtfs_errcode { + QTOK = 0, + QTERROR = 1, + QTEXIT = 2, +}; + +// qtinfo start +#if (defined(QTFS_CLIENT) || defined(client)) +enum qtinfo_cnts { + QTINF_ACTIV_CONN, + QTINF_EPOLL_ADDFDS, + QTINF_EPOLL_DELFDS, + QTINF_EPOLL_FDERR, + QTINF_SEQ_ERR, + QTINF_RESTART_SYS, + QTINF_TYPE_MISMATCH, + QTINF_NUM, +}; +#endif + +#if defined(QTFS_SERVER) || defined(server) +enum qtinfo_cnts { + QTINF_ACTIV_CONN, + QTINF_EPOLL_ADDFDS, + QTINF_EPOLL_DELFDS, + QTINF_NUM, +}; +#endif + +// for connection state machine +typedef enum { + QTCONN_INIT, + QTCONN_CONNECTING, + QTCONN_ACTIVE, +} qtfs_conn_type_e; + +struct qtinfo_client { + unsigned long cnts[QTINF_NUM]; + unsigned long recv_err[QTINFO_MAX_EVENT_TYPE]; + unsigned long send_err[QTINFO_MAX_EVENT_TYPE]; + unsigned long i_events[QTINFO_MAX_EVENT_TYPE]; + unsigned long o_events[QTINFO_MAX_EVENT_TYPE]; +}; + +struct qtinfo_server { + unsigned long cnts[QTINF_NUM]; + unsigned long i_events[QTINFO_MAX_EVENT_TYPE]; + unsigned long o_events[QTINFO_MAX_EVENT_TYPE]; +}; + +struct qtinfo { + union { + struct qtinfo_client c; + struct qtinfo_server s; + }; + // all struct qtreq_xxx's size + unsigned int req_size[QTINFO_MAX_EVENT_TYPE]; + unsigned int rsp_size[QTINFO_MAX_EVENT_TYPE]; + int log_level; + int thread_state[QTFS_MAX_THREADS]; + char who_using[QTFS_MAX_THREADS][QTFS_FUNCTION_LEN]; + int epoll_state; + int pvar_vld; // valid param's number + int pvar_busy; // busy param's number +}; + +#define QTINFO_STATE(state) ((state == QTCONN_INIT) ? "INIT" : \ + ((state == QTCONN_CONNECTING) ? "CONNECTING" : \ + ((state == QTCONN_ACTIVE) ? "ACTIVE" : "UNKNOWN"))) + +//ko compile +#if (defined(QTFS_CLIENT) || defined(client)) +static inline void qtinfo_clear(void) +{ + int i; + for (i = QTINF_SEQ_ERR; i < QTINF_NUM; i++) + qtfs_diag_info->c.cnts[i] = 0; + memset(qtfs_diag_info->c.recv_err, 0, sizeof(qtfs_diag_info->c.recv_err)); + memset(qtfs_diag_info->c.send_err, 0, sizeof(qtfs_diag_info->c.send_err)); + memset(qtfs_diag_info->c.i_events, 0, sizeof(qtfs_diag_info->c.i_events)); + memset(qtfs_diag_info->c.o_events, 0, sizeof(qtfs_diag_info->c.o_events)); + return; +} +static inline void qtinfo_cntinc(enum qtinfo_cnts idx) +{ + if (idx >= QTINF_NUM) + return; + qtfs_diag_info->c.cnts[idx]++; + return; +} +static inline void qtinfo_cntdec(enum qtinfo_cnts idx) +{ + if (idx >= QTINF_NUM || qtfs_diag_info->c.cnts[idx] == 0) + return; + qtfs_diag_info->c.cnts[idx]--; + return; +} +static inline void qtinfo_recvinc(int idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->c.i_events[idx]++; + return; +} +static inline void qtinfo_sendinc(int idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->c.o_events[idx]++; + return; +} +static inline void qtinfo_recverrinc(int idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->c.recv_err[idx]++; + return; +} +static inline void qtinfo_senderrinc(int idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->c.send_err[idx]++; + return; +} +#endif + +// ko compile +#if defined(QTFS_SERVER) || defined(server) +static inline void qtinfo_clear(void) +{ + memset(qtfs_diag_info->s.i_events, 0, sizeof(qtfs_diag_info->s.i_events)); + memset(qtfs_diag_info->s.o_events, 0, sizeof(qtfs_diag_info->s.o_events)); + return; +} +static inline void qtinfo_cntinc(enum qtinfo_cnts idx) +{ + if (idx >= QTINF_NUM) + return; + qtfs_diag_info->s.cnts[idx]++; + return; +} +static inline void qtinfo_cntdec(enum qtinfo_cnts idx) +{ + if (idx >= QTINF_NUM || qtfs_diag_info->s.cnts[idx] == 0) + return; + qtfs_diag_info->s.cnts[idx]--; + return; +} +static inline void qtinfo_recvinc(int idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->s.i_events[idx]++; + return; +} +static inline void qtinfo_sendinc(int idx) +{ + if (idx >= QTINFO_MAX_EVENT_TYPE) + return; + qtfs_diag_info->s.o_events[idx]++; + return; +} +#endif +// QTINFO END + +#endif + diff --git a/qtfs/conn.c b/qtfs/conn.c new file mode 100644 index 0000000..4d30ee2 --- /dev/null +++ b/qtfs/conn.c @@ -0,0 +1,908 @@ + +#include +#include +#include + +#include "comm.h" +#include "conn.h" +#include "log.h" +#include "req.h" + +static struct kprobe kp = { + .symbol_name = "kallsyms_lookup_name" +}; + +char qtfs_log_level[QTFS_LOGLEVEL_STRLEN] = {0}; +char qtfs_server_ip[20] = "127.0.0.1"; +int log_level = LOG_ERROR; +int qtfs_server_port = 12345; +int qtfs_sock_max_conn = QTFS_MAX_THREADS; +struct qtinfo *qtfs_diag_info = NULL; + +static atomic_t g_qtfs_conn_num; +static struct list_head g_vld_lst; +static struct list_head g_busy_lst; +static struct llist_head g_lazy_put_llst; +static struct mutex g_param_mutex; +int qtfs_mod_exiting = false; +struct qtfs_sock_var_s *qtfs_thread_var[QTFS_MAX_THREADS] = {NULL}; +struct qtfs_sock_var_s *qtfs_epoll_var = NULL; +#ifdef QTFS_SERVER +struct socket *qtfs_server_main_sock = NULL; +struct qtfs_server_userp_s *qtfs_userps = NULL; +#endif +#define QTFS_EPOLL_THREADIDX (QTFS_MAX_THREADS + 4) + + +#define QTCONN_IS_EPOLL_CONN(pvar) (pvar->cur_threadidx == QTFS_EPOLL_THREADIDX) +#define QTSOCK_SET_KEEPX(sock, val) sock_set_keepalive(sock->sk); tcp_sock_set_keepcnt(sock->sk, val);\ + tcp_sock_set_keepidle(sock->sk, val); tcp_sock_set_keepintvl(sock->sk, val); + +#define QTFS_SERVER_MAXCONN 2 + +static int qtfs_conn_sock_recv(struct qtfs_sock_var_s *pvar, bool block); +static int qtfs_conn_sock_send(struct qtfs_sock_var_s *pvar); +static void qtfs_conn_sock_fini(struct qtfs_sock_var_s *pvar); + +#ifdef QTFS_SERVER +static int qtfs_conn_server_accept(struct qtfs_sock_var_s *pvar) +{ + struct socket *sock = NULL; + int ret; + + if (!QTCONN_IS_EPOLL_CONN(pvar)) { + sock = qtfs_server_main_sock; + } else { + sock = pvar->sock; + } + + if (sock == NULL) { + WARN_ON(1); + qtfs_err("qtfs server accept failed, main sock is NULL, threadidx:%d.", pvar->cur_threadidx); + return -EINVAL; + } + ret = kernel_accept(sock, &pvar->client_sock, SOCK_NONBLOCK); + if (ret < 0) { + return ret; + } + QTSOCK_SET_KEEPX(sock, 5); + + qtfs_info("qtfs accept a client connection.\n"); + return 0; +} + +static int qtfs_conn_sockserver_init(struct qtfs_sock_var_s *pvar) +{ + struct socket *sock; + int ret; + struct sockaddr_in saddr; + saddr.sin_family = AF_INET; + saddr.sin_port = htons(pvar->port); + saddr.sin_addr.s_addr = in_aton(pvar->addr); + + if (!QTCONN_IS_EPOLL_CONN(pvar) && qtfs_server_main_sock != NULL) { + qtfs_info("qtfs server main sock is %lx, valid or out-of-date?", (unsigned long)qtfs_server_main_sock); + return 0; + } + if (QTCONN_IS_EPOLL_CONN(pvar) && pvar->sock != NULL) { + qtfs_info("qtfs server epoll sock is %lx, valid or out-of-date?", (unsigned long)pvar->sock); + return 0; + } + qtfs_info("qtfs sock server init enter pvar:%lx, threadidx:%d mainsock:%lx pvarsock:%lx", (unsigned long)pvar, pvar->cur_threadidx, + (unsigned long)qtfs_server_main_sock, (unsigned long)pvar->sock); + + ret = sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &sock); + if (ret) { + qtfs_err("qtfs sock server init create sock failed.\n"); + goto err_end; + } + + sock_set_reuseaddr(sock->sk); + QTSOCK_SET_KEEPX(sock, 5); + + ret = sock->ops->bind(sock, (struct sockaddr *)&saddr, sizeof(struct sockaddr_in)); + if (ret < 0) { + qtfs_err("qtfs sock server bind error: %d.\n", ret); + goto err_end; + } + + ret = sock->ops->listen(sock, QTFS_SERVER_MAXCONN); + if (ret < 0) { + qtfs_err("qtfs sock server listen failed.\n"); + goto err_end; + } + + if (!QTCONN_IS_EPOLL_CONN(pvar)) { + qtfs_server_main_sock = sock; + qtfs_info("qtfs thread main sock get:%lx, threadidx:%d.", (unsigned long)qtfs_server_main_sock, pvar->cur_threadidx); + } else { + pvar->sock = sock; + qtfs_info("qtfs epoll main sock get:%lx, threadidx:%d.", (unsigned long)pvar->sock, pvar->cur_threadidx); + } + + return 0; + +err_end: + sock_release(sock); + return ret; +} +#endif +#ifdef QTFS_CLIENT +static int qtfs_conn_client_conn(struct qtfs_sock_var_s *pvar) +{ + struct socket *sock = pvar->client_sock; + int ret; + struct sockaddr_in saddr; + saddr.sin_family = AF_INET; + saddr.sin_port = htons(pvar->port); + saddr.sin_addr.s_addr = in_aton(pvar->addr); + + ret = sock->ops->connect(sock, (struct sockaddr *)&saddr, sizeof(saddr), SOCK_NONBLOCK); + if (ret < 0) { + qtfs_err("%s: sock(%llx) addr(%s): connect get ret: %d\n", __func__, (__u64)sock, pvar->addr, ret); + return ret; + } + QTSOCK_SET_KEEPX(sock, 5); + + return 0; +} +static int qtfs_conn_sockclient_init(struct qtfs_sock_var_s *pvar) +{ + struct socket *sock; + int ret; + struct sockaddr_in saddr; + saddr.sin_family = AF_INET; + saddr.sin_port = htons(pvar->port); + saddr.sin_addr.s_addr = in_aton(pvar->addr); + + ret = sock_create_kern(&init_net, AF_INET, SOCK_STREAM, 0, &sock); + if (ret) { + qtfs_err("qtfs sock client init create sock failed.\n"); + goto err_end; + } + QTSOCK_SET_KEEPX(sock, 5); + pvar->client_sock = sock; + + return 0; +err_end: + sock_release(sock); + return ret; +} +#endif + +int qtfs_conn_init(int msg_mode, struct qtfs_sock_var_s *pvar) +{ + int ret; + + switch (msg_mode) { + case QTFS_CONN_SOCKET: +#ifdef QTFS_SERVER + ret = qtfs_conn_sockserver_init(pvar); +#endif +#ifdef QTFS_CLIENT + ret = qtfs_conn_sockclient_init(pvar); +#endif + break; + + default: + qtfs_err("qtfs connection init failed, unknown mode:%d.\n", msg_mode); + break; + } + + return ret; +} + +void qtfs_conn_fini(int msg_mode, struct qtfs_sock_var_s *pvar) +{ + switch (msg_mode) { + case QTFS_CONN_SOCKET: + qtfs_conn_sock_fini(pvar); + break; + + default: + qtfs_err("qtfs connection fini failed, unknown mode:%d.\n", msg_mode); + break; + } + return; +} + +int qtfs_conn_send(int msg_mode, struct qtfs_sock_var_s *pvar) +{ + int ret; + switch (msg_mode) { + case QTFS_CONN_SOCKET: + ret = qtfs_conn_sock_send(pvar); + break; + default: + qtfs_err("qtfs connection send failed, unknown mode:%d.\n", msg_mode); + break; + } + return ret; +} + +int do_qtfs_conn_recv(int msg_mode, struct qtfs_sock_var_s *pvar, bool block) +{ + int ret; + switch (msg_mode) { + case QTFS_CONN_SOCKET: + ret = qtfs_conn_sock_recv(pvar, block); + break; + + default: + qtfs_err("qtfs connection recv failed, unknown mode:%d.\n", msg_mode); + break; + } + return ret; +} + +int qtfs_conn_recv_block(int msg_mode, struct qtfs_sock_var_s *pvar) +{ + return do_qtfs_conn_recv(msg_mode, pvar, true); +} + +int qtfs_conn_recv(int msg_mode, struct qtfs_sock_var_s *pvar) +{ + int ret = do_qtfs_conn_recv(msg_mode, pvar, false); + if (ret <= 0) { + msleep(1); + } + return ret; +} + +void qtfs_sock_recvtimeo_set(struct socket *sock, __s64 sec, __s64 usec) +{ + int error; + struct __kernel_sock_timeval tv; + sockptr_t optval = KERNEL_SOCKPTR((void *)&tv); + tv.tv_sec = sec; + tv.tv_usec = usec; + + if (sock == NULL) { + qtfs_err("qtfs sock recvtimeo set failed, sock is invalid."); + return; + } + error = sock_setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO_OLD, + optval, sizeof(struct __kernel_sock_timeval)); + if (error) { + qtfs_err("qtfs param setsockopt error, ret:%d.\n", error); + } +} + +static int qtfs_conn_sock_recv(struct qtfs_sock_var_s *pvar, bool block) +{ + int ret; + int headlen = 0; + int total = 0; + struct qtreq *rsp = NULL; + struct kvec load; + + memset(&pvar->msg_recv, 0, sizeof(pvar->msg_recv)); + + headlen = kernel_recvmsg(pvar->client_sock, &pvar->msg_recv, &pvar->vec_recv, 1, + QTFS_MSG_HEAD_LEN, (block == true) ? 0 : MSG_DONTWAIT); + if (headlen <= 0) { + return headlen; + } + + load.iov_base = pvar->vec_recv.iov_base + QTFS_MSG_HEAD_LEN; + load.iov_len = pvar->vec_recv.iov_len - QTFS_MSG_HEAD_LEN; + total = 0; + rsp = pvar->vec_recv.iov_base; + if (rsp->len > load.iov_len) { + qtfs_err("qtfs recv head invalid len is:%lu", rsp->len); + return -EINVAL; + } + while (total < rsp->len) { + ret = kernel_recvmsg(pvar->client_sock, &pvar->msg_recv, &load, 1, + rsp->len - total, (block == true) ? 0 : MSG_DONTWAIT); + if (ret <= 0) break; + total += ret; + load.iov_base += ret; + load.iov_len -= ret; + if (load.iov_base > (pvar->vec_recv.iov_base + pvar->vec_recv.iov_len)) { + qtfs_err("qtfs recv error, total:%d iov_base:%lx iovlen:%lu ret:%d rsplen:%lu", total, + (unsigned long)pvar->vec_recv.iov_base, pvar->vec_recv.iov_len, ret, rsp->len); + WARN_ON(1); + dump_stack(); + break; + } + } + if (total > rsp->len) { + qtfs_err("recv total:%d msg len:%lu\n", total, rsp->len); + BUG(); + } + + return total + headlen; +} + +static int qtfs_conn_sock_send(struct qtfs_sock_var_s *pvar) +{ + int ret = kernel_sendmsg(pvar->client_sock, &pvar->msg_send, &pvar->vec_send, 1, + pvar->vec_send.iov_len); + if (ret < 0) { + qtfs_err("qtfs sock send error, ret:%d.\n", ret); + } + return ret; +} + +static void qtfs_conn_sock_fini(struct qtfs_sock_var_s *pvar) +{ + if (pvar->client_sock != NULL) { + qtfs_err("qtfs conn sock finish threadidx:%d, client:%lx.", pvar->cur_threadidx, (unsigned long)pvar->client_sock); + sock_release(pvar->client_sock); + pvar->client_sock = NULL; + } + + return; +} + +int qtfs_sock_var_init(struct qtfs_sock_var_s *pvar) +{ + memset(pvar, 0, sizeof(struct qtfs_sock_var_s)); + pvar->vec_recv.iov_base = kmalloc(QTFS_MSG_LEN, GFP_KERNEL); + if (pvar->vec_recv.iov_base == NULL) { + qtfs_err("qtfs recv kmalloc failed, len:%lu.\n", QTFS_MSG_LEN); + return QTFS_ERR; + } + pvar->vec_send.iov_base = kmalloc(QTFS_MSG_LEN, GFP_KERNEL); + if (pvar->vec_send.iov_base == NULL) { + qtfs_err("qtfs send kmalloc failed, len:%lu.\n", QTFS_MSG_LEN); + kfree(pvar->vec_recv.iov_base); + pvar->vec_recv.iov_base = NULL; + return QTFS_ERR; + } + pvar->vec_recv.iov_len = QTFS_MSG_LEN; + pvar->vec_send.iov_len = 0; + memset(pvar->vec_recv.iov_base, 0, QTFS_MSG_LEN); + memset(pvar->vec_send.iov_base, 0, QTFS_MSG_LEN); + INIT_LIST_HEAD(&pvar->lst); + return QTFS_OK; +} + +void qtfs_sock_var_fini(struct qtfs_sock_var_s *pvar) +{ + if (pvar->vec_recv.iov_base != NULL) { + kfree(pvar->vec_recv.iov_base); + pvar->vec_recv.iov_base = NULL; + } + if (pvar->vec_send.iov_base != NULL) { + kfree(pvar->vec_send.iov_base); + pvar->vec_send.iov_base = NULL; + } + + return ; +} + +void qtfs_sock_msg_clear(struct qtfs_sock_var_s *pvar) +{ + memset(pvar->vec_recv.iov_base, 0, QTFS_MSG_LEN); + memset(pvar->vec_send.iov_base, 0, QTFS_MSG_LEN); + pvar->recv_valid = QTFS_MSG_LEN; + pvar->send_valid = QTFS_MSG_LEN; +#ifdef QTFS_CLIENT + memset(pvar->who_using, 0, QTFS_FUNCTION_LEN); +#endif + return; +} + +void *qtfs_sock_msg_buf(struct qtfs_sock_var_s *pvar, int dir) +{ + struct qtreq *req = (dir == QTFS_SEND) ? pvar->vec_send.iov_base : pvar->vec_recv.iov_base; + if (!req) { + WARN_ON(1); + return NULL; + } + return req->data; +} + +// state machine +#define QTCONN_CUR_STATE(pvar) ((pvar->state == QTCONN_INIT) ? "INIT" : \ + ((pvar->state == QTCONN_CONNECTING) ? "CONNECTING" : \ + ((pvar->state == QTCONN_ACTIVE) ? "ACTIVE" : "UNKNOWN"))) + +static int qtfs_sm_connecting(struct qtfs_sock_var_s *pvar) +{ + int ret = QTERROR; + +#ifdef QTFS_SERVER + ret = qtfs_conn_server_accept(pvar); + if (ret == 0) { + qtfs_info("qtfs sm connecting accept a new connection, addr:%s port:%u.", + pvar->addr, pvar->port); + qtfs_sock_recvtimeo_set(pvar->client_sock, QTFS_SOCK_RCVTIMEO, 0); + } else { + msleep(500); + } +#endif +#ifdef QTFS_CLIENT + int retry; + qtfs_info("qtfs sm connecting wait for server thread:%d, addr:%s port:%u", + pvar->cur_threadidx, pvar->addr, pvar->port); + retry = 3; + while (qtfs_mod_exiting == false && retry-- > 0) { + ret = qtfs_conn_client_conn(pvar); + if (ret == 0) { + qtfs_info("qtfs sm connecting connect to a new connection, addr:%s port:%u.", + pvar->addr, pvar->port); + qtfs_sock_recvtimeo_set(pvar->client_sock, QTFS_SOCK_RCVTIMEO, 0); + break; + } + msleep(1); + } +#endif + + return ret; +} + +int qtfs_sm_active(struct qtfs_sock_var_s *pvar) +{ + int ret = 0; + + switch (pvar->state) { + case QTCONN_ACTIVE: + // do nothing + break; + case QTCONN_INIT: + // create sock (server:bind listen) + if (pvar->client_sock != NULL) { + WARN_ON(1); + qtfs_err("qtfs sm active client sock not NULL!"); + } + ret = qtfs_conn_init(QTFS_CONN_SOCKET, pvar); + if (ret < 0) { + qtfs_err("qtfs sm active init failed, ret:%d.", ret); + break; + } + // dont break, just enter connecting state to process + pvar->state = QTCONN_CONNECTING; + qtfs_info("qtfs sm active connecting, threadidx:%d sock:%lx client_sock:%lx", + pvar->cur_threadidx, (unsigned long)pvar->sock, (unsigned long)pvar->client_sock); + // fall-through + + case QTCONN_CONNECTING: + // accept(server) or connect(client) + ret = qtfs_sm_connecting(pvar); + if (ret == 0) + pvar->state = QTCONN_ACTIVE; + break; + default: + qtfs_err("qtfs sm active unknown state:%s.", QTCONN_CUR_STATE(pvar)); + ret = -EINVAL; + break; + } + return ret; +} + +int qtfs_sm_reconnect(struct qtfs_sock_var_s *pvar) +{ + int ret = QTOK; + switch (pvar->state) { + case QTCONN_INIT: + WARN_ON(1); + qtfs_err("qtfs sm reconnect state error!"); + ret = QTERROR; + break; + case QTCONN_ACTIVE: + // release current socket and reconnect + if (pvar->client_sock == NULL) { + qtfs_err("qtfs sm reconnect client sock invalid?"); + WARN_ON(1); + } + sock_release(pvar->client_sock); + pvar->client_sock = NULL; + + ret = qtfs_conn_init(QTFS_CONN_SOCKET, pvar); + if (ret < 0) { + qtfs_err("qtfs sm active init failed, ret:%d.", ret); + break; + } + + pvar->state = QTCONN_CONNECTING; + qtfs_warn("qtfs sm reconnect thread:%d, state:%s.", pvar->cur_threadidx, QTCONN_CUR_STATE(pvar)); + // fall-through + case QTCONN_CONNECTING: + ret = qtfs_sm_connecting(pvar); + if (ret == 0) + pvar->state = QTCONN_ACTIVE; + break; + default: + qtfs_err("qtfs sm reconnect unknown state:%s.", QTCONN_CUR_STATE(pvar)); + ret = QTERROR; + break; + } + return ret; +} + +int qtfs_sm_exit(struct qtfs_sock_var_s *pvar) +{ + int ret = QTOK; + switch (pvar->state) { + case QTCONN_INIT: + // do nothing + break; + case QTCONN_ACTIVE: + case QTCONN_CONNECTING: + if (pvar->client_sock == NULL) { + qtfs_err("qtfs sm exit client sock invalid."); + break; + } + sock_release(pvar->client_sock); + pvar->client_sock = NULL; +#ifdef QTFS_SERVER + pvar->state = QTCONN_CONNECTING; +#endif +#ifdef QTFS_CLIENT + pvar->state = QTCONN_INIT; +#endif + qtfs_warn("qtfs sm exit thread:%d state:%s.", pvar->cur_threadidx, QTCONN_CUR_STATE(pvar)); + break; + + default: + qtfs_err("qtfs sm exit unknown state:%s.", QTCONN_CUR_STATE(pvar)); + ret = QTERROR; + break; + } + return ret; +} + +int qtfs_mutex_lock_interruptible(struct mutex *lock) +{ + int ret; + ret = mutex_lock_interruptible(lock); + if (ret == 0) { + // mutex lock successed, proc lazy put + while (1) { + struct llist_node *toput = llist_del_first(&g_lazy_put_llst); + struct qtfs_sock_var_s *pvar; + if (toput == NULL) + break; + pvar = llist_entry(toput, struct qtfs_sock_var_s, lazy_put); + qtfs_sock_msg_clear(pvar); + list_move_tail(&pvar->lst, &g_vld_lst); + qtfs_warn("qtfs pvar lazy put idx:%d.", pvar->cur_threadidx); + } + } + return ret; +} + +void qtfs_conn_param_init(void) +{ + INIT_LIST_HEAD(&g_vld_lst); + INIT_LIST_HEAD(&g_busy_lst); + init_llist_head(&g_lazy_put_llst); + atomic_set(&g_qtfs_conn_num, 0); + + mutex_init(&g_param_mutex); + return; +} + +void qtfs_conn_param_fini(void) +{ + struct list_head *plst; + struct list_head *n; + int ret; + int conn_num; + int i; + + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret < 0) { + qtfs_err("qtfs conn param finish mutex lock interrup failed, ret:%d.", ret); + WARN_ON(1); + return; + } + + list_for_each_safe(plst, n, &g_vld_lst) { + struct qtfs_sock_var_s *pvar = (struct qtfs_sock_var_s *)plst; + qtfs_sock_var_fini((struct qtfs_sock_var_s *)plst); + qtfs_sm_exit((struct qtfs_sock_var_s *)plst); + kfree(plst); + if (pvar->cur_threadidx < 0 || pvar->cur_threadidx >= QTFS_MAX_THREADS) { + qtfs_err("qtfs free unknown threadidx %d", pvar->cur_threadidx); + } else { + qtfs_thread_var[pvar->cur_threadidx] = NULL; + qtfs_info("qtfs free pvar idx:%d successed.", pvar->cur_threadidx); + } + } + conn_num = atomic_read(&g_qtfs_conn_num); + for (i = 0; i < conn_num; i++) { + if (qtfs_thread_var[i] != NULL) { + qtfs_err("qtfs param not free idx:%d holder:%s", + qtfs_thread_var[i]->cur_threadidx, + qtfs_thread_var[i]->who_using); + } + } + mutex_unlock(&g_param_mutex); +#ifdef QTFS_SERVER + if (qtfs_server_main_sock != NULL) { + sock_release(qtfs_server_main_sock); + qtfs_server_main_sock = NULL; + } +#endif +} + +struct qtfs_sock_var_s *_qtfs_conn_get_param(const char *func) +{ + struct qtfs_sock_var_s *pvar = NULL; + int ret; + int cnt = 0; + + if (qtfs_mod_exiting == true) { + qtfs_warn("qtfs module is exiting, good bye!"); + return NULL; + } + +retry: + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret < 0) { + qtfs_err("qtfs conn get param mutex lock interrup failed, ret:%d.", ret); + return NULL; + } + if (!list_empty(&g_vld_lst)) + pvar = list_last_entry(&g_vld_lst, struct qtfs_sock_var_s, lst); + if (pvar != NULL) { + list_move_tail(&pvar->lst, &g_busy_lst); + } + mutex_unlock(&g_param_mutex); + + if (pvar != NULL) { + int ret; + if (pvar->state == QTCONN_ACTIVE && qtfs_sock_connected(pvar) == false) { + qtfs_warn("qtfs get param thread:%d disconnected, try to reconnect.", pvar->cur_threadidx); + ret = qtfs_sm_reconnect(pvar); + } else { + ret = qtfs_sm_active(pvar); + } + if (ret != 0) { + qtfs_conn_put_param(pvar); + return NULL; + } + memcpy(pvar->who_using, func, (strlen(func) >= QTFS_FUNCTION_LEN - 1) ? (QTFS_FUNCTION_LEN - 1) : strlen(func)); + return pvar; + } + + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret < 0) { + qtfs_err("qtfs conn get param mutex lock interrup failed, ret:%d.", ret); + return NULL; + } + if (atomic_read(&g_qtfs_conn_num) >= qtfs_sock_max_conn) { + mutex_unlock(&g_param_mutex); + cnt++; + msleep(1); + if (cnt < 100000) + goto retry; + qtfs_err("qtfs get param failed, the concurrency specification has reached the upper limit"); + return NULL; + } + pvar = kmalloc(sizeof(struct qtfs_sock_var_s), GFP_KERNEL); + if (pvar == NULL) { + qtfs_err("qtfs get param kmalloc failed.\n"); + mutex_unlock(&g_param_mutex); + return NULL; + } + if (QTFS_OK != qtfs_sock_var_init(pvar)) { + qtfs_err("qtfs sock var init failed.\n"); + kfree(pvar); + mutex_unlock(&g_param_mutex); + return NULL; + } + + memcpy(pvar->who_using, func, (strlen(func) >= QTFS_FUNCTION_LEN - 1) ? (QTFS_FUNCTION_LEN - 1) : strlen(func)); + pvar->cur_threadidx = atomic_read(&g_qtfs_conn_num); + qtfs_info("qtfs create new param, cur conn num:%d\n", atomic_read(&g_qtfs_conn_num)); + + qtfs_thread_var[pvar->cur_threadidx] = pvar; + // add to busy list + atomic_inc(&g_qtfs_conn_num); + list_add(&pvar->lst, &g_busy_lst); + + strcpy(pvar->addr, qtfs_server_ip); + pvar->port = qtfs_server_port; + pvar->state = QTCONN_INIT; + pvar->seq_num = 0; + +#ifdef QTFS_CLIENT + mutex_unlock(&g_param_mutex); + pvar->cs = QTFS_CONN_SOCK_CLIENT; + ret = qtfs_sm_active(pvar); + if (ret < 0) { + qtfs_err("qtfs get param active connection failed, ret:%d, curstate:%s", ret, QTCONN_CUR_STATE(pvar)); + // put to vld list + qtfs_conn_put_param(pvar); + return NULL; + } + qtfs_thread_var[pvar->cur_threadidx] = pvar; +#else + pvar->cs = QTFS_CONN_SOCK_SERVER; + if (qtfs_server_main_sock == NULL) { + if (qtfs_sm_active(pvar)) { + qtfs_err("qtfs get param active connection failed, ret:%d, curstate:%s", ret, QTCONN_CUR_STATE(pvar)); + // put to vld list + mutex_unlock(&g_param_mutex); + qtfs_conn_put_param(pvar); + return NULL; + } + mutex_unlock(&g_param_mutex); + } else { + mutex_unlock(&g_param_mutex); + pvar->state = QTCONN_CONNECTING; + ret = qtfs_sm_active(pvar); + if (ret < 0) { + qtfs_err("qtfs get param active connection failed, ret:%d curstate:%s", ret, QTCONN_CUR_STATE(pvar)); + qtfs_conn_put_param(pvar); + return NULL; + } + } +#endif + qtinfo_cntinc(QTINF_ACTIV_CONN); + + return pvar; +} + +struct qtfs_sock_var_s *qtfs_epoll_establish_conn(void) +{ + struct qtfs_sock_var_s *pvar = NULL; + int ret; + + pvar = qtfs_epoll_var; + if (pvar) { + if (pvar->state == QTCONN_ACTIVE && qtfs_sock_connected(pvar) == false) { + qtfs_warn("qtfs epoll get param thread:%d disconnected, try to reconnect.", pvar->cur_threadidx); + ret = qtfs_sm_reconnect(pvar); + } else { + ret = qtfs_sm_active(pvar); + } + if (ret < 0) { + return NULL; + } + return pvar; + } + + pvar = kmalloc(sizeof(struct qtfs_sock_var_s), GFP_KERNEL); + if (pvar == NULL) { + qtfs_err("qtfs get param kmalloc failed.\n"); + return NULL; + } + if (QTFS_OK != qtfs_sock_var_init(pvar)) { + qtfs_err("qtfs sock var init failed.\n"); + kfree(pvar); + return NULL; + } + qtfs_epoll_var = pvar; + pvar->cur_threadidx = QTFS_EPOLL_THREADIDX; + strcpy(pvar->addr, qtfs_server_ip); + pvar->port = qtfs_server_port + 1; + pvar->state = QTCONN_INIT; + +#ifdef QTFS_CLIENT + pvar->cs = QTFS_CONN_SOCK_CLIENT; +#else + pvar->cs = QTFS_CONN_SOCK_SERVER; +#endif + ret = qtfs_sm_active(pvar); + if (ret < 0) { + qtfs_err("qtfs epoll get param active new param failed, ret:%d state:%s", ret, QTCONN_CUR_STATE(pvar)); + return NULL; + } + + qtfs_info("qtfs create new epoll param state:%s", QTCONN_CUR_STATE(pvar)); + return pvar; +} + +void qtfs_conn_put_param(struct qtfs_sock_var_s *pvar) +{ + int ret; + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret < 0) { + llist_add(&pvar->lazy_put, &g_lazy_put_llst); + qtfs_warn("qtfs conn put param add to lazy list idx:%d, ret:%d.", pvar->cur_threadidx, ret); + return; + } + qtfs_sock_msg_clear(pvar); + list_move_tail(&pvar->lst, &g_vld_lst); + mutex_unlock(&g_param_mutex); + return; +} + +void qtfs_epoll_cut_conn(struct qtfs_sock_var_s *pvar) +{ + int ret = qtfs_sm_exit(pvar); + if (ret < 0) { + qtfs_err("qtfs epoll put param exit failed, ret:%d state:%s", ret, QTCONN_CUR_STATE(pvar)); + } + return; +} + +void qtfs_conn_list_cnt(void) +{ + struct list_head *entry; + struct qtfs_sock_var_s *pvar; +#ifdef QTFS_CLIENT + int ret = 0; + ret = qtfs_mutex_lock_interruptible(&g_param_mutex); + if (ret < 0) { + qtfs_err("qtfs conn put param mutex lock interrup failed, ret:%d.", ret); + return; + } +#endif + qtfs_diag_info->pvar_busy = 0; + qtfs_diag_info->pvar_vld = 0; + memset(qtfs_diag_info->who_using, 0, sizeof(qtfs_diag_info->who_using)); + list_for_each(entry, &g_busy_lst) { + qtfs_diag_info->pvar_busy++; + pvar = (struct qtfs_sock_var_s *)entry; + if (pvar->cur_threadidx < 0 || pvar->cur_threadidx >= QTFS_MAX_THREADS) + continue; + strncpy(qtfs_diag_info->who_using[pvar->cur_threadidx], + qtfs_thread_var[pvar->cur_threadidx]->who_using, QTFS_FUNCTION_LEN); + } + list_for_each(entry, &g_vld_lst) + qtfs_diag_info->pvar_vld++; +#ifdef QTFS_CLIENT + mutex_unlock(&g_param_mutex); +#endif + return; +} + +#define KSYMS(sym, type) \ + qtfs_kern_syms.sym = (type) kallsyms_lookup_name(#sym);\ + qtfs_info("qtfs kallsyms get %s:0x%lx.", #sym, (unsigned long)qtfs_kern_syms.sym); + +struct qtfs_kallsyms qtfs_kern_syms; +extern unsigned long kallsyms_lookup_name(const char *name); +void qtfs_kallsyms_hack_init(void) +{ + typedef unsigned long (*kallsyms_lookup_name_t)(const char *name); + kallsyms_lookup_name_t kallsyms_lookup_name; + register_kprobe(&kp); + kallsyms_lookup_name = (kallsyms_lookup_name_t) kp.addr; + unregister_kprobe(&kp); + + KSYMS(sys_call_table, unsigned long **); + KSYMS(d_absolute_path, char * (*)(const struct path *, char *, int)); + KSYMS(do_unlinkat, long (*)(int, struct filename *)); + KSYMS(getname_kernel, struct filename * (*)(const char *)); + KSYMS(filename_parentat, struct filename * (*)(int, struct filename *, unsigned int, + struct path *, struct qstr *, int *)); + KSYMS(__lookup_hash, struct dentry * (*)(const struct qstr *, struct dentry *, + unsigned int)); + KSYMS(do_mount, long (*)(const char *, const char __user *, const char *, + unsigned long, void *)); + KSYMS(path_mount, int (*)(const char *, struct path *, const char *, unsigned long, void *)); + KSYMS(path_umount, int (*)(struct path *, int)); + KSYMS(find_get_task_by_vpid, struct task_struct *(*)(pid_t nr)); + KSYMS(do_readlinkat, int (*)(int, const char __user *, char __user *, int)); + KSYMS(do_renameat2, int (*)(int, const char __user *, int, const char __user *, unsigned int)); + KSYMS(do_mkdirat, long (*)(int, const char __user *, umode_t)); + KSYMS(do_rmdir, long (*)(int, struct filename *)); + KSYMS(getname, struct filename * (*)(const char __user *)); + KSYMS(ep_ptable_queue_proc, void (*)(struct file *, wait_queue_head_t *, struct poll_table_struct *)); + KSYMS(user_statfs, int (*)(const char __user *, struct kstatfs *)); + + KSYMS(__close_fd, int (*)(struct files_struct *, int)); + KSYMS(do_sys_open, int (*)(int, const char __user *, int ,umode_t)); + KSYMS(do_epoll_ctl, int (*)(int, int, int, struct epoll_event *, bool)); + KSYMS(do_epoll_wait, int (*)(int, struct epoll_event __user *, int, int)); + KSYMS(do_linkat, int (*)(int, const char __user *, int, const char __user *, int)); + KSYMS(mnt_get_count, int (*)(void *)); + KSYMS(do_mknodat, long (*)(int, const char __user *, umode_t, unsigned int)); + return; +} + + + + + + + + + + + + + + + + + + diff --git a/qtfs/conn.h b/qtfs/conn.h new file mode 100644 index 0000000..cf9c764 --- /dev/null +++ b/qtfs/conn.h @@ -0,0 +1,167 @@ +#ifndef __QTFS_CONN_H__ +#define __QTFS_CONN_H__ + +#include +#include +#include +#include +#include +#include +#include + +#include "comm.h" +#include "log.h" + +#ifdef QTFS_SERVER +extern int qtfs_server_thread_run; +extern struct qtfs_server_userp_s *qtfs_userps; +#endif +extern char qtfs_server_ip[20]; +extern int qtfs_server_port; +extern int qtfs_sock_max_conn; +extern struct socket *qtfs_server_main_sock; +extern struct qtfs_sock_var_s *qtfs_thread_var[QTFS_MAX_THREADS]; +extern struct qtfs_sock_var_s *qtfs_epoll_var; +extern char qtfs_log_level[QTFS_LOGLEVEL_STRLEN]; +extern int log_level; +extern struct qtinfo *qtfs_diag_info; + +#define qtfs_conn_get_param(void) _qtfs_conn_get_param(__func__) + +static inline bool err_ptr(void *ptr) +{ + if (!ptr) + return true; + if (IS_ERR(ptr)) + return true; + + return false; +} + +#define QTFS_SOCK_RCVTIMEO 1 +#define QTFS_SOCK_SNDTIMEO 1 + +typedef enum { + QTFS_CONN_SOCKET, + QTFS_CONN_PCIE, + QTFS_CONN_INVALID, +} qtfs_conn_mode_e; + +typedef enum { + QTFS_CONN_SOCK_SERVER, + QTFS_CONN_SOCK_CLIENT, +} qtfs_conn_cs_e; + +struct qtfs_pcie_var_s { + int srcid; + int dstid; +}; + +struct qtfs_sock_var_s { + struct list_head lst; + struct llist_node lazy_put; + int cs; + int cur_threadidx; + int miss_proc; + unsigned long seq_num; + qtfs_conn_type_e state; + char who_using[QTFS_FUNCTION_LEN]; + struct socket *sock; + struct socket *client_sock; + char addr[20]; + unsigned short port; + + // use to memset buf + unsigned long recv_valid; + unsigned long send_valid; + struct kvec vec_recv; + struct kvec vec_send; + struct msghdr msg_recv; + struct msghdr msg_send; +}; + +struct qtfs_conn_var_s { + union { + struct qtfs_pcie_var_s pcie; + struct qtfs_sock_var_s sock; + } mode; + char *buf_recv; + char *buf_send; + int len_recv; + int len_send; +}; + +struct qtfs_kallsyms { + /* global vars define */ + unsigned long **sys_call_table; + + /* functions define */ + char * (*d_absolute_path)(const struct path *, char *, int); + long (*do_unlinkat)(int, struct filename *); + struct filename * (*getname_kernel)(const char *); + struct filename * (*filename_parentat)(int, struct filename *, unsigned int, + struct path *, struct qstr *, int *); + struct dentry * (*__lookup_hash)(const struct qstr *, struct dentry *, unsigned int); + long (*do_mount)(const char *, const char __user *, const char *, unsigned long, void *); + int (*path_mount)(const char *, struct path *, const char *, unsigned long, void *); + int (*path_umount)(struct path *, int); + struct task_struct *(*find_get_task_by_vpid)(pid_t nr); + int (*do_readlinkat)(int, const char __user *, char __user *, int); + int (*do_renameat2)(int, const char __user *, int, const char __user *, unsigned int); + long (*do_mkdirat)(int, const char __user *, umode_t); + long (*do_rmdir)(int, struct filename *); + struct filename * (*getname)(const char __user *); + void (*ep_ptable_queue_proc)(struct file *, wait_queue_head_t *, struct poll_table_struct *); + int (*user_statfs)(const char __user *, struct kstatfs *); + int (*__close_fd)(struct files_struct *, int); + int (*do_sys_open)(int, const char __user *, int ,umode_t); + int (*do_epoll_ctl)(int, int, int, struct epoll_event *, bool); + int (*do_epoll_wait)(int, struct epoll_event __user *, int, int); + int (*do_linkat)(int, const char __user *, int, const char __user *, int); + int (*mnt_get_count)(void *); + long (*do_mknodat)(int, const char __user *, umode_t, unsigned int); +}; + +extern struct qtfs_kallsyms qtfs_kern_syms; + +static inline bool qtfs_sock_connected(struct qtfs_sock_var_s *pvar) +{ + struct socket *sock = pvar->client_sock; + __u8 tcpi_state; + if (sock == NULL) + return false; + tcpi_state = inet_sk_state_load(sock->sk); + if (tcpi_state == TCP_ESTABLISHED) + return true; + qtfs_warn("qtfs threadidx:%d tcpi state:%u(define:TCP_ESTABLISHED=1 is connected) disconnect!", pvar->cur_threadidx, tcpi_state); + + return false; +} + +int qtfs_conn_init(int msg_mode, struct qtfs_sock_var_s *pvar); +void qtfs_conn_fini(int msg_mode, struct qtfs_sock_var_s *pvar); +int qtfs_conn_send(int msg_mode, struct qtfs_sock_var_s *pvar); +int qtfs_conn_recv(int msg_mode, struct qtfs_sock_var_s *pvar); +int qtfs_conn_recv_block(int msg_mode, struct qtfs_sock_var_s *pvar); + +int qtfs_sock_var_init(struct qtfs_sock_var_s *pvar); +void qtfs_sock_var_fini(struct qtfs_sock_var_s *pvar); +void qtfs_sock_msg_clear(struct qtfs_sock_var_s *pvar); +void *qtfs_sock_msg_buf(struct qtfs_sock_var_s *pvar, int dir); + +void qtfs_conn_param_init(void); +void qtfs_conn_param_fini(void); + +struct qtfs_sock_var_s *_qtfs_conn_get_param(const char *); +void qtfs_conn_put_param(struct qtfs_sock_var_s *pvar); +struct qtfs_sock_var_s *qtfs_epoll_establish_conn(void); +void qtfs_epoll_cut_conn(struct qtfs_sock_var_s *pvar); + +int qtfs_sm_active(struct qtfs_sock_var_s *pvar); +int qtfs_sm_reconnect(struct qtfs_sock_var_s *pvar); +int qtfs_sm_exit(struct qtfs_sock_var_s *pvar); + +void qtfs_kallsyms_hack_init(void); +void qtfs_conn_list_cnt(void); + +#endif diff --git a/qtfs/demo/Makefile b/qtfs/demo/Makefile new file mode 100644 index 0000000..33a9e4d --- /dev/null +++ b/qtfs/demo/Makefile @@ -0,0 +1,12 @@ +CFLAGS=-g -O2 + +all: cfifo_r cfifo_w + +cfifo_r: cfifo_r.c + gcc $(CFLAGS) -o $@ $^ + +cfifo_w: cfifo_w.c + gcc $(CFLAGS) -o $@ $^ + +clean: + @rm -f *.o cfifo_r cfifo_w diff --git a/qtfs/demo/cfifo_r.c b/qtfs/demo/cfifo_r.c new file mode 100644 index 0000000..563e0f7 --- /dev/null +++ b/qtfs/demo/cfifo_r.c @@ -0,0 +1,94 @@ +#include +#include +#include +#include +#include +#define BUF_MAX 256 +int single_read(int argc, char *argv[]) +{ + char buf[BUF_MAX]; + char *fifo = argv[1]; + int rfd = open(fifo, O_RDONLY); + if (rfd < 0) { + printf("open file %s failed.\n", fifo); + return 0; + } + + do { + memset(buf, 0, BUF_MAX); + int ret = read(rfd, buf, BUF_MAX); + if (ret == -1) { + printf("read failed.\n"); + break; + } + printf("%s", buf); + } while (strcmp(buf, "exit") != 0); + close(rfd); + return 0; +} + +int my_epoll_read(int argc, char *argv[]) +{ + char buf[BUF_MAX]; + int *fd = (int *)malloc((argc - 1) * sizeof(int)); + + for (int i = 1; i < argc; i++) { + fd[i-1] = open(argv[i], O_RDONLY|O_NONBLOCK); + if (fd[i-1] < 0){ + printf("open file %s failed.\n", argv[i]); + return 0; + } + printf("my epoll read open file:%s success, fd:%d.\n", argv[i], fd[i-1]); + } + + struct epoll_event evt; + struct epoll_event *evts; + int epfd = epoll_create1(0); + if (epfd == -1) { + printf("epoll create failed.\n"); + abort(); + } + for (int i = 0; i < argc - 1; i++) { + evt.data.fd = fd[i]; + evt.events = EPOLLIN; + int s = epoll_ctl(epfd, EPOLL_CTL_ADD, fd[i], &evt); + if (s == -1) { + printf("epoll ctl failed, fd:%d\n", fd[i]); + abort(); + } + printf("my epoll read epoll ctl fd:%d events:%x success.\n", fd[i], evt.events); + } + + evts = calloc(64, sizeof(evt)); + + while (1) { + int n = epoll_wait(epfd, evts, 64, -1); + printf("epoll wait get new %d events.\n", n); + for (int i = 0; i < n; i++) { + int ret; + printf(" > epoll wait new events, cur:%d key:%x data:%lx n:%d.\n", i, evts[i].events, evts[i].data, n); + memset(buf, 0, sizeof(buf)); + if (evts[i].events & EPOLLHUP) { + epoll_ctl(epfd, EPOLL_CTL_DEL, evts[i].data.fd, NULL); + continue; + } + ret = read(evts[i].data.fd, buf, sizeof(buf)); + if (ret <= 0) { + printf(" >read fd:%d ret:%d data error.\n", evts[i].data.fd, ret); + } else { + printf(" >read fd:%d ret:%d data:%s.\n", evts[i].data.fd, ret, buf); + } + } + } + close(epfd); + for (int i = 0; i < argc-1; i++) { + close(fd[i]); + } + return 1; +} + +int main(int argc, char *argv[]) +{ + return my_epoll_read(argc, argv); +} + diff --git a/qtfs/demo/cfifo_w.c b/qtfs/demo/cfifo_w.c new file mode 100644 index 0000000..85aaa38 --- /dev/null +++ b/qtfs/demo/cfifo_w.c @@ -0,0 +1,30 @@ +#include +#include +#include + +#define BUF_MAX 256 +int main(int argc, char *argv[]) +{ + char buf[BUF_MAX]; + char *fifo = argv[1]; + int wfd = open(fifo, O_WRONLY); + if (wfd < 0) { + printf("open file %s failed.\n", fifo); + return 0; + } + + do { + int ret; + int len; + memset(buf, 0, BUF_MAX); + fgets(buf, BUF_MAX, stdin); + len = strlen(buf); + ret = write(wfd, buf, BUF_MAX); + if (ret == -1) { + break; + } + } while (strcmp(buf, "exit") != 0); + + close(wfd); + return 0; +} diff --git a/qtfs/doc/ Overall_architecture_diagram.png b/qtfs/doc/ Overall_architecture_diagram.png new file mode 100644 index 0000000000000000000000000000000000000000..40fd7e28707642801ec0b984690a25c08e092ac4 GIT binary patch literal 49089 zcmeFZcT|&E-!>X-%&5qW4e5@6NSAJ?jwn??=^d2brI!%E$|y}jZ$ZSsP(ll#*FicY zLg+0hCDb4U2q7dncbsRQ_c`nR)>-HK=dAO6=j^pyxk>JOXaD-|y7rE59~o$|o#i?U z0)g0cwC@{%K&QJwppz3nodMoqDbHvIfv$se?*DEQnzK1)?`BJAQ9SgusoT%YY0SMJ z9Ow~uk;E1J^REXpKeJ+b|F|C}Epf*p!jJh1I{V=E1?KDB{GYm6m>0J$cidsVf?nVF|0nt1M<=sm-TfyyS2y|& zwp2Nfo}I$fT2=61CJ0!jC&#oTV)iZVLs61I+OtK zO`5Mwq1F;69~l+r1t^M=_m?IgIcJh@r#*+yft&Wnn*G0OPj{agK{css(3%27v`_)_ ziK7_z!PW6Af99rEPm)w}FSU3NNq*;(O?%$T;UBz9y#R{7QT>z$RF6ubj-aTI-c-w0 zz+2y+$Q1x)W8(abc^at2$^6+8^s84_<#GWbZTt$MOy|&KsXJ`?s!t_Q=3sg zhMb?-Adi;FBJ~Qfg2vxO60Z(`+pt#__;BbbD#l5Dht?BfnDN|oOKG? z>6|k9L$EUEYre}wrHTDg*a;A*P8SHfS#We@osesp1f522!f)nWIwj9Bcg@;`GFUJa!lc;Q=BRiP&{We3XwYJKr=IOhcS-$qrP zYHg8Pjzi4eX>(l(MUx7l{3>|U;eUm4d+pU zx5gf9tu=MsoEc~ z7IWW<#YR2-*3}n)&j_Zc7FXFtxUTPs`94%ZdCz>Vb)C-$EmHfQ8e&&wzuomVWM%aG zx6iS@&C-6(-}*muV@K5$!K1Qh?85|Oe#ngMC7cwvVBFQko*Mtw6`t9MSG?LbEFxTn?Z)~$Pg9IgXaR!OI z&rY*)!1J7E@yb#@*$U~{ht1L(b6I7KF2$SkVT)aFIf}-DP+)ja!MX9KQS!6m?LA#Q zc4TF@|Fg2M3aT9iKTvI4pw3=vqqmU0jpnf$Ps}I~rX)P)S8S-cZ#nY9{J!OiDbid{ zffgiPx&YtMT2t{?VhxcG{-ez4PsXEQMKe~Z5}01t4XtoxW6!bKGz{UwxY-9o&moy$ zqL-+Su5w<+>?{gNYTzl$&G_m+61H?tGN(LBA-#I}?R*VF;zz)|ows6;s-8E0$>ULh zI!)hc30rn3HX~Oxh#3z+V`NM+8>-x_U40EU?ehLmt?rp2D%F15(0PKs zXF$xEPQwUPj1ua05SYDIqda{7vQ4Hh5G|YGK_v-7@l7wt6Jbk#)ijhX?FzKW&LjJd zVlXS6!U;>gHI3mL*Z>g{Izy->#_~wH3;Q)$~T0FxZ_#f zUqKdT&C_$WF7Q3>$Wa^lvw_}XoR^ocVa%v@C?+XXR6XC@hh`8AcywH|3%5ZavJ5k( zV%+t%lP-a2>*Wt!;Hmk0{L|3nS^>mn#8b2PV+YzTTm2@h;|JPXV%GeY9;zhYcNUQ? z>Zv1`!3L)S-oedZDreI*G<4P$0s>w(V9Kr8QH0Pt)@hfn8lpBKR=r=UERi~ee~lGO z?H3*(@_DjxB_9FFZP&6?AM0_3{8|?oh42as&3{Y?Q|Ft9Sx20TwFnKsEVtq8k0a9* z({qcx*&g*xO*fDR!Je@ z;57C0T-zpCe!d9?K47t3v(TWDqlYjuDjrlZ*0BP3mF_-+F}S;4QzcqF7Fq!Dv867;oK z;u}aL!QO@RxK#<0JTH?te z!qlhRd;YV3*#rkWo4=7U0cQs|wyb&U!K%*l=pD3Gt9LfB58Y8X_=-pgR!grHTpmzt zczg8jz`AWr4-6@T=?aBCS$LbX+QNYc+BFxP?rBhcKpdNsL#xkE-nr-l(PTD+Cp6i2 z=LYBM6nrj5(s_f9Yk2leJ6zdU8fo`I{lI4b`i@WK62wXynNC_W4?&nj*`(&vd8X%* z!Mw&s_FLKRw9+2Tve!!Oss*wB+|ggas;xSh_K;5})fjUsr=zT!My3>~7F#g_C~E=R zthrYaz;ahoFQMrhsdSDsvKkBM*~CSl!wgA5#R|@Qw-{$j%zJ2?5odDgx<*Qe{WWZ7 z(o2VXAjCJ%WAp5NrTnz4h^i9hNRqaJ+>1>?9U+?3Ad1u3F}YVa%1&xR0X7)X@+n=J z$Rh@c3W24D6{RnZuT)nJyDV=u#0iPCMJg!gmC^>HyVo9T@r3G3giNtfKcm z)nrbiu62~AU`}^Lib5; z@)V*Xhn%h9lT(xT_bsZU!lQOED&~-%W#r^a<8>El8D!o+X<{#RgNLXmQ=Rb>NjS78 z8}Bwwjo2~J z)JKwA5k4;M%3+a0v?j>Ng!aXre$G{8vkF;Gr5aUL)syGS<9Nl`UH|kmG+z<%+MIP# zCxN$It||wMk!zxL8N|K4RAKXriK(pqUbjSeZB|L~?Q&D-UPF~M1>$C{JkWQPs&6dM zNRg|yN%jr@xGIR?o`mTDEP+K!!zO4O>M9gcrSxa2%yZ3O?C1Ou3c&g zu#%5UW!*ihqMz4KYY<~4tEbPO4p+shwE;=V4!o;vhmX9afv^SFqr?pNf-&AdODnvG z1Izbi$BuO1z{4{N+h9GVgVWpYjsEub{!0x`3;R49{3W&(CjI3!#kD9^;C>F6`AX)Z zbE5K=ohL~-U@#kxx1hz`M!mbm3oK&aRlccxq$4L1$WN`VH5+3#cVxs)x2}-}Z^#eA z3)c1}$+`5WWYlp8!fazIm(&7@>e{K(AhB(Zw!;VIHVq@2GN4=0G5nEt)Q^S} zvupv|w^#qv{gp3pxj!{x8-n)?<^)}CuZY|)bx8;8Ug^d3D`5NwDI2qp}W^zLJzoYtT78Z=f9lQ6V}sQb1a4>V^BX2UUtQM)Dzl7T(~ z*ReQC2RX0bF~E??Po32TG)m;-<{zl-@obyh+d2rlx5Fu1M4QR3UYpsv$7G`%&M;LA zjQ^sZ_O7vh$S7|~Rm9&(4ym{~wHFGf6EP~Os4!Wq%K=~R#C(*PYm}|J;y$U3;RVm~ z8upi%nCBmwvS+Am+tFMizfWV9h7@-PLMkLj&hY|syIlpoq$^+xdAzxdC(NiHy4yj# z9ATbe4B)ooCUepuJ3KdN9lN`m>AS+J4wbGqYX+DlQ-eFbmLnR7rw^95f zDL#vr@PiQFcozZ+kHtlWf?Z(|IX^g~g$o~3&w^f)=u@%{M~5M#(WUYnZqg+4wrrQ& z&@E)M-_5Y2BCLVWLc>l~(Jrk>QK%s+B#LHME)=#oHL*Kwm}juH7}cweR!@ze$|#E5 zwX&o38E2wFKZIanlde5t1!)|;W!GoDw6+%#8&DsQ3EAiiFVV2M+0N7OjG z0df4GMp_KO`?d4x16DXo$d4*P9;%UcvAfisw(MfD<|F%s&2@jFXuFb%bVkA&UTTCQ+tOv3K3Iu)4EWQpO9$o zNP5V;pVb|4qFr@kQQgjVK^w^+8%BK(RT!hbvxkx&MpAaME@*Qd|NTVDVenOE>Mc_O zd_*M5NNOPASJqtB_xZ@<_HA#H$FjrjnlO(lB&b!h278ouZ zEM;YcyeKt1b)F1lzFC)vUAna^X#?MuU5Y!HoUL&;Zl%m^6Q&-(*2wq>*|1>O-SMg8 zqroQhT?npFrL~<<-oh}$(2pL@d)v6;7Q>F3lu4tG+{rcr*6I+jtIIa2VU4+V!v|`Z z7NU9Ek?GOV5lql?ysYmeyULfPcYqdr3~pZ_Uk7Yj`h#AcjR)eGsqz0W6Vheg!7T0l zki)Mrq*8lY$&@qb5hz>H*C&`600Y7J&!0cvN5z7C^0*a!mZYL6dZ+==2hez#Wil&Y z=m+3-pu0?Ab}+#BuR=4ddz$FSpyV+!8Hb4sw;zQKN}fwcdwM?sB7TW=?uVFtwr3no2=Ns>1V_qsE+XCWO}^b#9?pWP z?2;~_a%)PJQoItSw#>2_asya$<-3yNxEw>uH*{QLq9L>aEx(l*?4nhik{0LmX*wxE zv)z`rZ>ONsJ*UO5*qjRtPwrgY$D+$e*?<|IvHt=zf2AL-z^TbE3pfour^&=VxRKsX zj%Vhj#X0Jk1)VQ-VB_URN?cmjO4ZxX^bA_$GKRynd~|wRJJ&L3V+qiyiN}^p0l;LJ zQ<<#wR*67QKwzkf`&R#RC~qci_!B}1L4<;v3u=`j%G12u^kvkrKo1VGxS8#H0#s+d zY$9>l5ORRwcxN~Os(ihjNhI%Yv8SaBnj_slS+qpJ%AMh1I`))acy2;RTn_?{4LI z!B8LtE)YKeeZjMAlK@(LV zi!TdwCY~1uSnGnN+jMV1L}=#%Db`%x|?h&>woO_hH!4!+7-D1H`i0BxoH{v z6CRd;IXKUr;1Y2dA<8?~NBJ68ByWpKtE&qPETnyN&{Xt502+HH#k9BnI4l7ggOe7x z?yEoX9J*FN2M%1bcE`GD21tx~$Ty@3a^qwOjflW)<(%ajHC7^T8)e9MHyeLk$V^Id zb0Mvy+z5g)-aVL8>>hJl<0j6`KhfS$zG1C@d!zu#$;Xd9{<^nSdlqRQ-89m&$8i?~ z@?cg3pX9Wfhr~DFEOya>L${4L*{@dXaA*_xi-4x;i?m*C9m6WQ~y z5ePH2dU)E1$JL*Ja=CS#`ABJ6g6h{oi4|vEBt(MvG0)kG{QX1aI<@TkyAH|&iL2V~)k&f9W4b~F zDalHzGg_uWdIh~+=B>h^oCf&1ufArFY{Xt@b%>#ou7plWagI{f*BaOD=fvX5CWHW_ zQF3MVd#H?BPDeo}uG<5!22o1x5+L9#=f7EnXLS%dg+Tg|?Bbg4%BLogC=1`vHr)K! zy=%>Av^=+>T9)t7PlL%OTK>nY-vu=xG2bf>!qJt*mwzLBJCpmc&l%C><( zrC>r@aZJghAt??#S?%@*%*ORY7&Cn`FaGs!6&QtzJ^6#~1n$%6 zx32xv++J0spN(rHY6eFh>rBIzP^I9V;UW>_Qz)p62WbM_2ikRbGa+nC;%ibTwAM=Y z#b?^nm;4PP{HbCm9hNPBkTsX>!Uk5(ON0j6N>=E>+LnLa`piQ7`N!#o4_~?!QvKba zs8W!FJOD_531>Tr*vXgd%dgosj4jQnx@)+Nu;oSWFepib5;bw`8c9w)dR~Uj_nz@+ zqwN*g9ERG4*ZOEX`yd-F?ZKr{NZ1&3)U{&+gpHe~#mrA$VQc`_p!GD_pT!T;m54wM-!0nkeO_xSRgfZA(Q2-~y(^4nQ@}%f>5Hg)J8-1^K-xz7 zdXE}Lz#gBomF@e4^GK$fYsRFXIApJHPMbG~0_+vEfonsjw6Ydq#C)f+=CT!?+-xN} z_~!TGRU#+_Jk3dhd4x7I$>lLaZdZJ7hu-$kPYcRYZRxlP-@)@;>Cd*hDza@X>6@Uk zYs?l3Xevrxf1_=v4V9)2ta+6^rEezhMFQV*rP+v*&sH1}uG*6K&`vNHcVb~iW}KG2 z``4SL1w=8#le*+#*AOYt*LdE0GNtOMI5!;+NE{EL>((A8mbwqPt|3QH&@o5y!4gIj zsTJh|Zp@BWQ1%v$K|eqZ%T3xnja9(W}LeikpmKxOyW38RJHn$Z*vP8~Uf(>Z9ZVu)lS8 zqdI@2P+ue4REu}#2?6B5a*$4%A2m1XNY1iNEwutF;7IYRHXWQSWNQed6&&sRV$m#S z*-U~7RNph&IWyavX6@!$ZR2gO-rrH-t}=}0_^pIBS-5Gy&DzUMgPuQKfc1$ zcq1EnRA6Xa!n%{X&+YH#>29W%a)r-x&fGlo3h546umvgD&!Z-u0f zOF;S#s#EK{bi$^i>Y)B$;d$zv7#ttRG~xMTPN&Dix_nq$_>*}v%~}y$*uUfkyiE?TZU^n&|?KVZU^)~ zekYtxS}#(B0jY%Z>lTd@@_nO0ZgHvmW^Vla64}*ZVANx^inR@%FPDJL%Q08*(#NK; z7C~K%6Oe|w6YB6w0A~a1KH3CAz|Qwxs}J;o_gvc6%z9g`4+f!*Z6;1Cwp!s2NiJWb zJhGpS$IR~+g<>fRJ(9A4`^Kpff-#_~)z%g@w#>tW+e=kiv@$db+s#&NjapRAvo z6eA?8)f-bTnWxXZf~+~BZIMfRS`(;^?i#82LpfI097xm|-I;2L#v&hJ^Nm_BDTyy< z5NiB}#}rNf#c;T_!!miO0xAZq@$Zvsa!o)p77ghmVb${{===8Bwr46ou;S{x1K#)x zuu7SZc%Ibx-~eZQWyeiSqzhW2T=e-u7oOiVuQ~d=;{3O2bt{cJF{X&O0V1|iK=AIb z$RDVi`U5{o&t!t0QPF7ha*sOQ52&}%LO_m7(=dj(7gj&H>JO;)PJ0rW3qk#Io!sewh8MH zo3>kzJJpXh{*yS?7Pw}Qa*1cgb-kF|kD~N{>+oIwkJDEArbwry2r!3M56W-*YhZd> zsMpT#KQKU#HLf)TFQXPBTwEyQueSVkI8Io{3OA1yp5v9!FK3<%Y`xQQr`@B(%zxbp zT(G#fFI7{BrVTUuRuKWPsmRZK=EgySJ&6~JcH|T!_a{{=u5Q}T)+<6TgcKpvI0`zo z0wh?gvC4pi?Y02HQ)wky5s{uP? z+)_k?n!&o`)*5D0l54DATa@2s@VlFb{3lovY5Dww`sDO{s_GoCd#0?5=XBG-?6hV9 zO7XV0x4t<69ypl!Wx4^kQJgyu$52+HgvKf$et|07WIjwyvF*2?LsChh zRq;{;<*8K4SZPOqjxK2+1|s=DN0LA4{zRxPTfByVF`W0d0xSs zgGS)}k(#kKr2F>zU_ye?%Fr>B_uv8CVdV|X_~Z(r!)g0@_%%x=Wx+vOo!4Bo&`0n^ z)?$Bh729VMnn97j&$97)=2N@-{7O@ka(H+!p;pW=Sh|yHEQL_X3V%PHRbH+YpenHs zVQO~Rine7t)$}!7&21`)7;YQyxP`j6DD?1Y0sxj5=TD-m3WsnxRb0ROdYT^)*Z#y0 z6!Y-%=)25PpYG4zL!wTstmT#FD8YAE50`S#A*xw6zI_SA=^T0ZgvnvcA*#fdxPtXu zX_|l^l(S6@Kh|A796n!EI!=!CfN`{~zz9KMKXeuX2Usd)C={v#llL57dd|w{=KtWC z>k%%Gp3=skZ3Em^^i57aK1>^JrfnI>j@W6mCke|q#;&cedjwm)|5VT5uSh-HE(+jK zZF7jDaL-QM1@@KI9b?X769RnJNVf-GUUReWaDOmBQTzD;a75)J!)&P% zgyVGcpvn5Yq9bO+NH>H$nv)u=VwHz_ZA~1k{7f)iSIg;SUB?PcROgQ-rPX|L#DC?1 z|N2hqD@-qUVp*laxf!7ApD7;AZAYeK)fp-}yA(nmA*79Jn=Ez$KrI&X5^JmT=_7Rp z%S#Jx-ecbn-QKWj$<9uz)wSsAX(`Rb$lg)P+U-q9aoJ1$bLr*qK}0EJTu5_(bLw%{ zc=*Sw_wuZszD^~fQ`;df5{7wRr)49hQ$2cAkP=p?Cmq2YB(HCi)xV*81J*wv zb)(6XUkiEys#;_rTw+OjbD6n$*ov%mwBTYV_-N$&2|^s<=;Dc)RPze&`A2mrL}(0Q z-mzFYhTRZ~wn8o)_xWq4CwJqAv-X_T=Std;)s9ezd?K(2lfBw+LMNNA=t zfGaGymE3aEzFX z4qlGI1g37Z%xs72N7yw?HA@o0>Yx#qmUmXt7o}I})Dr%ulEy8Aqy4V|f$) z%%r%H5V;Mi=Qz*Cw!A)8%3KOckLB#sr%!Kmc6OfgX+36oFS2glYPlr(41n#-`nuRf z*KQ594SgN;*j}36Tp2;|u6ghRs(FSnSE?=umktZWz7X@Y)}5O&UoS_l^70IYcFRWi z3nH@1yS?|i?5v_Tu8c9sMCGf>`F1m*l4d1q85L83&VmAK+FHXF=o{C#xcULDah&f?p60u37OxQioQtTm z(p2w_i0$^Y8mX>2Xo?RADS4>n>Ns{-oWnn#feCxj4U~J??uvaGeD;ALXIn&Jsk1?ZMA?B{Ns`BRq1C=UJXr zxV}1+NLRwoinA~`C-h;>>~HF=&CN6EBi_xUU&Q;a-vEmmeu`T1nWjwAqy5FP(ZVqO+!p}82?6jSp(EJ%MmZTLJNf97Lr zw7=-GxSSOE1;rXRJ5NC8)XK+>*1jSH2?KFD%NUTeE^35T}xi1d7YeWc>Ya15Evz(wCOI^)s-)2~b07b6?od;rC&l zwpK+*qn^8t5i;(RN>)wqR)zx6Bs+FvOi*GhE~n(v=v-17lgVt%DCk`L5$Oq-Yi;O_ zyNyLr`9?$N>C}K0tD!xB!)~#|KO|YH{b^#HRr^=HRfC1H0yAxCt!>hny{(Q24Q2zWg<*-&k2QzoNa36|ucrKCG% zB9@vXWk0wYs;cwQ(`traAIJb<3ZpsT?w#a5`04Q1<_5XCjUj>p1YZ{^7O&^+zx+=|VrR*#ZdUxG6 zAF52=aQh3fa zvWCyU4tRzMfWJJcLWpI>Zx+J5Z4YENH!boivX?KmL?XV)dBp< zWH7!Rpk%|yfz%%S#F#LuCaDgXQ9=S&U))c$5?!=7O!)%^X%|3DQ-3oqGg!bmGm<$eVYG_aid z`{z{L-77=c`?CR9gw2Y7!yXkth7Wvz8!Tdb|2}I5%zOI^prn5d~QkvXrit4klY!=z7L?#v`YEEr*EIW zCkh;F`Z1!5fB8Y$GXKapTWk`|My%@LGY$i%D}g?I@9*#bXTr0?G7c}r#KZ*tS=r~` zm?-zhXgA5w{MYqTUx8L);QqR43kjpFmzJ8Iz5xMFA;5YvF-u_y}wA-w1|y7?BrjMQ9phH+>q?ZW9Yy$`F=iJ_D?)**Eg+^69l?+*by>+ zD+t(7(vF1)fVAWNREmGF>`Z0e)c8kfAP3P;%$-k8$oDC4B)T9ZuDnK-Ev&GIop z5nn_+ogna}$3q+7sdLqaAj{tNGhWsGSO#*|*)o_0)i(-iGRAZ}kK3Bvtuw+{NTqC*)F79_ zo>0n#u_@`Jt0q{7^l&Sg{Z3tSgV_~o4h!Yq29=Hvme+bw2j_n7Y<6Kat;tSH=FCgf zB6n7b>zK938`dm#UD$*15~Dgq%{Uf(GVR8{Tr>L4Re2TTC&z2>zMPwm;hLnagOp-3 z|6jS#LaMMxHdAopdkdoSV%`Gu462V4t!H#flj#x;HYZxi2tZrgFi zJqhxVkvz1@Um+p=%^n;;nf5mssF{TS93sA~gGSnE87Pfaj#$p?bjeH2;U4#5Zu>PBlYsT zGQ|ipof+)!skW*wgGIcp(So6N4S`~uoX6L>9b4${~3y zY{qS>darxdXrCQWTpHmdzu-|dC}<%__}ga4wch=MeO}AL+G?zB03mzQh(PI0KQrJR zG8K?EQ+4=8a=pYlNv-s~Pk4~dRo-=mbi`Ti%x7p!;n+oSdn_ z%kw%gX+8)=)0fdQH*SV>;7Z4R%G?O-->0(rRyrpS5p)H1-XMdgNkUdYWF zYAEFOr_*7QkSp)3=M)tXQ#ZA2hK*yn8gpYQU*iu(`@>|y`8pfZpVqZRew~mcNEbZv^2dXw{>jPFh>}gE5vSvZ znx9-PS~t?_KW5LEHh3-6-02cUHpMBJ5=D<&jRp#((stwZ-N()91jk~7qHF~#RR!=i zBgwOT4Hx-hPsegdz_r@S6JnAT94;HqrK`*+WpAxeEd+J0lsr@(QtR3%8o5XrIo-?q zPZoY})QQ`N&G??uO&ss`*D!?0KQ=CEh1E*NYZe`y$&F>O4V@0$dLf)f2-iO=7N}+P zOqV8HI8uiCKnc@H3i$q6((z#b^OV@G0ywZ)S!SSdxH#dHs{)Ve2M=aV_{=U+(*R0*Trn9g<-}lito2??qx+$;DoXiG%0`maPMPYF0xyzYSUkthy`x0vHOood=jNb>&G9EU@=$vg$~90 zrdLf2<=#u^Sbh2Y`f}H;g45apT_zl{MdiVO?_B+m0Ze)OXUFdL%uD8h#qY25lH7sn z^W_R#PJR7f3Ng#vupR0#pur#iFF=UZ=0;xu?yxyg_xzs?gVnOXb`SKCK=((6|}YOMy5U}2psW}--O5n$+Brzj&U zZ3vpnNO0NWO9M-R8B*NwZK!%#cwV$0(k@7BSE8*Geb=}6 z-`iAbPb%3~etg?4zW8~MFNw3aYWv(i^1}Or|RORq*Z)k%ZjJb8M5je*J)3O0plKV@KhN{=FlM6h$ww8b44dl_sfs&F7&> zewL}WJu|LxZ~1<2dxJf?q+mTaUn}@tRavIwc~ey%C*aNHjfnM{VxOnbwNNEmhJ4eO z%N^GSzJb8Ka|sA2S5{(6q>p~g*!%C(S<-2zU6-;^MRXaEwj(KzHkC)SEaqmp-qzjI zv)|X9Iyh>xsCH0KA0o159aCmtpA7ljN80-K^2t>_XpyDItFlAuhy0|SOO4Uxw4zG5 z*PQLAsSo_?t%Tb7(Cenq?j(xErC0lg_?BWtzT-D)VTZ=kw5r}t)297Qvlum$t0&7r z!__O)2=7yrbyUdSw3x4lflft-^Q&$~DV^@~(jaQ9u0|rVqojg{toDH`!Wp6HtP5s0 zebgJ-2TtnGqgQ*M_Xq(lzL^r4A09*>a~-V=J#XziM-t(ETSHi6gDjR8Bd4L2_s%ig zjnt=&>ITuZe#cLbym?hZAHht_%nI>`@%USQv7qZMJ3D^pnwpeG&RZYZAigFSti*c?8Dk&SYQNwt~=2kDvem$N)Om1yTkP4HY zGTWk$-lRmzmD(TA3fQj}D2?)Ff2s7w7S)JdeovNfo2>xOhf-42D9y&KWJakSWLZCT zYJGn*01~vy9;w$B^ufSdrEb656XoA_@ylNcqN=2A^#Z8%34dy2aR{E4TI-eE9JQ@K zymH>L9$ayzRiT`pe!MlQq27kjr*-ibLCH5&b8^RKU-rJN4VRCiEWoJK z5Jmo8di=|hJVw-BKZ7n{8`&j5-fUY>w)YulqhHyTINdlz`gf zs#nWOWc$FcF{p6GRzuZnC-uXMqc(%%JT6+;=02)^)0lLGwdbB-hql?+9CRnDWSuv? z-%YZ(uo&tRchbW5YSe*Hn9r%v;S6`P!p9=CUWdrte#)Dh%a2suH1Dmpq+)CMk|fl z<8+d~lH5H0~4lt@~;M&y!eVHVI3z|EJYL*)6Ax zoZ6;AX^AND%&4(x>B-cChN(TF_dj1!Z@h#o(7PH@Aka{ zi1{w#=ii=m--!Jaq7SA%qFuQiiAi3`GbNZ}-Tf1>8 zzsD-EZW-N^{ANYLG{{w1gkmecUP_vE146yHuioM5>nyC6>mQo_LafH(u{F+r@rwOo z;axhBbof+IEXhPgHXk|M-0x2P4Z_SIws4qmxEJ6vd4ZC-oXUhTphw^L`gf0xo7{#S z=)Q$!*l0^vKg_JA;z)D-+Ue$$85ab9AlX1b}VJ zw1zzhAP?@6SkytZ#H!R?4j{M>HVja=d8ia+ON9q6g@L1WH99?v=U>Jy~8cgpjW zM*PzkKt+y|dN_%|88+?hMyrdodmi(YJ()((pV!|I7_R+%g>>%0IHy|aCvSq=je zbXGopweu-UiFFk`B0*H(m*d^`l$)40@^~EAxf*Pk{Ig)q1GvJzyp`bVWp4kqILPk_ ze6D5x>iAEzioCmF^L^E)cKnKt= z9ti~z&u)wZJ#+Zh`X7ph|Yxq9j|Q;Fb4f)upbL3PiL`1*YDi z_Ho?EBwa&db>Dq)XOe=g#@={+ORT^<=iD`dkAn$`v(+ina9y?_*LXc$=xTG~AErhj~~pub!|-&iy642?^TJFQ?YUqiyLgDTpjWK>v;?zIvSslUEG zRGBjs31))NCChL3Idw|R3D;kLV|i;L3K}ndQ^0B>q0=E^n{C%P+nZ^|&2dLXT&rX& zFr;`p!pZtkmWWPH%i)c7-?QYQX@dU)5APTkaroYVCa z4@(9#FT+d~Mx-#;dozi3=GrnPomx1}!|p=4nWaCnb-W&H)XDni?0A=?lAQKz-ULZ? z`9Kwq{r9d>2g94fl>QTcTs>KkW-&7-dv*&zljXJ;N6oDG!$=+Brp4+%^ShqZ2;E_# zi0Xo>3BOxUGOM6y<^3KpRYrgv+xbJpLG7fp*_dRZf=*Q2xh!+q2;QtLQeURfJ^*nL9b8k4Xs5^YM5>u^Jb$KV=2VeoR*r9>=(W)O(&*f$ga}FO1jK{T{nsWyHJmB7KrjO{M@oa3sjA zPJC|v6O$8w2#x=IzhXbo{t%E8^kDc0M{;zgX%y#;eRAUG%AF+L)9sC8-ve8g4!Bvm z3eZ)}UvPH%qoc(YpOAV5!^N!~7A;#}Z0=4-hCT(CPn1>O3%UHWr~$&#^>-2pR`L<$ z%pvD@@UB_^0`GRh;DUExVcD&DCIX%_&f?wapSYOa@e3EZ7q?^)VNgCZ)Khg?L$c=z zbWTb~NJh0F%=-0Z8*Li}Re?mLL6LcccoJQgv%cI^%lAlh$1?L=Q4TVlL1_JeJ!8;wOI%}z+|^&{PG4p31J-zh;%8cd}+8mf54xVU8ko{y5xmrN);dQeK#qSFp#(BAG`nTcN9@)zbQU%ix=!+ z$a~#pkgz#{bT8t%@!&V;1Y#u_Er$2uOF&%nS1RvT0xzWt=T|yfln;y~C103RQqHUt z4^%8$cQp&h8Rnex)V8aD#&R7@tc?$ek8k*x;)f6KeLS~&zP=_guYeh}?Ts1$nYNMP zEZwuS`G+E`;?9HhczGf5iH6v9{+~*BEwYcobNv;3q=E{n(}S#j5-_B$!NX0pj6wz* zi{v5>bd4I{XS^v&_V?F%ckAYvHvxF!`u{=Qdq*|3wf&+fDq9iQDotq?5a}Wvq-+%t zsnUDsy-4T;2}h2c&K-eo64YEl@4)7qGf;FkMm2%Zz@PCsSco z1ST7fUt86m&V_f;no$%5Ap7@E%5v_4a1{OWBxLP-qslgRifKUbEarRR?Uy<}kf0Bp zgzIVPvJF29>xPT696uQ*)aJaUb-Ez&I5O^W1iwB1$M>VY+unj5aad1fTJ6WQa)+Jv zwu;XErLiHO??U8rw0lKZz)aW*#(;s`g3&YaHC5cp{=N~OSV+pk-B=8?RC?_QrQRV; z3u-i<05w3w{J%x$(USdDmcmHPW=dB=}J3L@-qxeFHO zl02QtM{9bG_=Y)s$IN$ix_jhiguu+^8l~qH%8N~=+=lH=p_;C+9J!a82BGV!c0j_m zBGaI)nwrFE=qnXkFYI|%E06)YmACEDhu)tLK&`&_r zAR-Gf0H9n+7Dsc2UX6*3nDcUC$sgr8m6qfMzsKIC*|0I_U=nL-Chf|IUB_9KXycZ* zA7Aue6mw*8h$b#N6|0F7r6#Zio3D_R;>njg^2eXm$RIrFn)7-yv8LTz)jw~cTIuhU zEwr0*bWLGdW2NGIt(sxdnwgz|ub^6awjCpRde6oG4yts^@!3fKs%fzyse>Ye7z{me z3RwEaWX}KN%=_=PhUFgwWWB7<7IE4|mC=mEC{EnQG^&-as<-EyK?nmRrqOY+vt5IR zljQpwe@L(x30{|mN(px4i=IbabTQBozy4mfk@Md<^_?}jcrGjD16Q)Sh26x~?o~pp;FDiyE6W6hE{JJ79*T)_{;_Grfs-6m1t`rD z_aa%Frbr`@Y*x7j{IT!MfOi}Ce)8%{3Y!{HOq>gXMW)f0+?e%|un|KjtJoV!~#PKLX)B^N@!Xeah zqvb4`e#G;#qU^(p)7QKg3eD;8Ya;_DKDCc7Oo3R(KSfWrUaYiu7Yrm&awRuiqQb9_ z73m(LM2_u?(w~`)0uiKk@2QtdE!U*wn6BoqMll5U_zZR*WPb8hfF}Fp-ZiZq@chdF6Q{OTuXZ9tu+7x!cY6v(BG-U@ z+bk{yCQSo?oYd_)0OW|ZCnD8w!XPDZKXe!b#w+zLx&--H=o{M5}2pfo_Qg6Y#LLwKX9R$cHKV zcyog7FRD9Px-d;Z@wMRh5z~HD&?5CEg4G5;X4AW&-1(v}TH?y1O*v;BqzJ(D@VQ@X zJ(9QbCS8!&lU{Tck8zo7jhj`F%28vvz4#V$T&qS+9WpRY|LFD`R_so6vpD;f|@Lp12wF`;Sa3V(sQteEA|m+|-xqbVuxXNGP&(^LrSkk&G!J(tC8*(eA(*h;ydqwSf|0T z+e=2{uQ8unIWD(oi{1UwZr%Ybp_P~RU+a*ntNvJO2@J6PE5_z(h#hO$SMDA!8~D-u zDHM2O{Ym+YRbW5-2vyPbDkweRE*dTxEYPEkGEu3yRnD|%Nb~dspcYz2-wjEVdNzMO z$5987Y~X`XFFHmB(#3>Fr}G1N%^~?BWlIFuxrEyUix2<8alO*C_f1MI5+IM;)Jrsn z92IOdVBXH-msDR|Zep|W1<`3uz=DuX?V95h=8_~lrDl(?=ZU!7xo zAQ7Zl)U}YrtdG{%TA*xFx@iM-KjgJ<_5*s@yC|O&Y_+-`E3@>rBc=hwRFrfkeRV5&Qwv3 zY~E&o@%vogSj$ORi=wqR1#pocIkZ^+14p1?g_a2SxnaBbng;#gSK+su9<7bbOxo4* zDBMxNj(nO@U41`l%N{oAf_pg3vO2)@zobT-zYK!fEQ!shD$bXp50c7dWr7V z-$s_dO*VR4@AEO11%sW6lfI)Wi@lx-B?!uvK?DWKI@R}sY@Q*cR9Wb`L*umzqe~I)l(KxBH~c9lZdz|(&RA^{^}O&{|bNKX+EjD#}a2)qMCcN z1D^ovcHxxBy#1yngLiH_dU;pRrS{LFqb6BS14m@;fiGl9oL>en++)Srer)t&?4UU{ z!gDE3k>k##|7tdjwlzQft!%<9=93lf2dut|W`_3-UA=@X3m;Y}eZh*>f|AMl8zrQ9U z-C;qx8@mQibd4Y0G^y@Ns&RL*;~Jlu|-|L zmwj8H@C$shJO`AeKNO|aQ8D|$yQVW&Syc??VQe1amm~dKsmfWgS9vUPm(fT){c_qa zAifq4#$Ef?)GmCvv|jLhzp!u(&)LZ3(~)7~DFw%G^3_FOj#I7MT&5B;edQ$%h#Rl@c99hUp+W4{KVgn)z+BoY zpfx)zTmet!P03k>y<6XKj?Cyn}S|HykNQ_L=1|rdh-c+k(cq>dzKKhr{ z;B)@$j|aA@Ku}3vxSc0IAV{F;WcjlPq)z3cHwBy#wQqHzMF3W`{Quc0qwV2Q!R<6! zpix{{_=~##_Xr8eW~C8%21|)ssy>=6x~Kuc?4e4n>kkeW>Un>iqF-%B)xn}VJ_p}0 zSLdw>IqU6a``0*D^W~h8)bFmR;<0pd(svkmcQ{nG{KMa?v@{^hJN&b)C5FVqg!iknb{ERdV`m8ligH`4M2CGZpNI%hF%yNZsab z6OCPptt*1bO`QIyw3?LCbb{W1?PAK94ZB|chXb&KTziW8!!qps|Ee49zvo^1_q+K- ze5k1wFq<&f!xLkY7~n$sdEV5gC>`LZO^|G-l$@g|UL&@ViT2&OISd1r!WfWIW%>m{ zX+yh)`zC$6QFc7K22_rgRKS`_#6?Bovxe0=Q6Ru`9u59L?1 z4Gav{ZUb&61YpjCOPvm7?3az~X_hv?jC(Z(}m6fVx2hE+&T*C_ah*jhN%aB%OzY)?~ zPyxkueB6j^i{-O#r>2S%P^huN6H+y@I~lFnA02moZx8E`&iW)>d4n{EmT%qRI<`4^ zB#gWg`xNzeX)D2AyPdd*5%dv5L~#WzM}NfrzdQ@zdjGi+|Gix)lP_+ebq{TNM9k$P z6;iB>lh+n*25thDL&Bs@Y*M4d(k1L(b@jdAom?&fHpa z)Ff}$wDVGS(d#|HVLW{u?E})%MnfN$DJIEVcP$volWKqJgN+Y2X_j+ z{LliPzvw3_$=kq|tF1HR?TO>mIN~pC|G8fWF^z7k?J71bh3qt0O!%FT$yEw^$Re>Z z7(zM$@$zoG+vuFj9b5iy?p2M6Szuw=^3d^tk{pFB^+Z{ufI@J!ziR8PAmRm~RBP=7(LT zk`|b!Jb_wI{!Cs+7T)%zKzz39STzdRV*Fl0AS7TaUO6SV>)VVyqw_uTiuKLIYGT!Y2S0PE^vd&5;CR{ZLR%JvE@V6u;)@ zlfBESAjhLx>|UWU9b=dx{5MY+3(c#fFu4|ql138e)%!H_QY|^gdwGYrv67mJqSvQ6 zc&Vt?IsPfT%SR8D+vCu^%EP8a3k;jmFy=}EU{S5!^PTsZO=+TSts07l z`-pr7%-(u4CqqrgvonAPFVo2Yz+)yp>?N^~>p5xwxGuKHY~Ah75w0#vHXsnjV3Tk1 z0C^-Zb8W?^J3c(Q!lPq#?|Y-#6+jg^5GK(17I{=^WMY{*zW`X{;fo-4VAQg){!_M& zbyJT4t4z+Q^_hYvm?nAfz~a0S;^WOWUhY3urn|r3OKvy-AS6f0qO4&xDayq)30lHU zda8Aay2}J%#M1m%dRDwjLfV?nP^FW1aSh*SXMIV6G5wUWKG37h?xOY&c<9<#CcK8a z4MXjzsVrrqk}kZe(w;5FOSk}gpVkA}YZUvIbG_=w%cW)7?@I*k3fy@eN8~LDyDLg6 z<-k@6B;V@S7{Yp9zog4G;E2zD_*l#mtai&1*Zm%6S>%eyuutRO4Bm+a&NYul zn;eK&|}pxt@}VaK$=XY|S#Z9*Q++TJ6?HbzQv3(XNcF|XP>h?RSMY0ypLa3m|X zL}4dzVJkqM(5OM=>wRmTw!JpuFTGU-@E(t#Yb-l1`x{K=*BI`+tJPeJSnDqQPEue& zw7erPX8`M%+&{g=SnUBbZ>lTN;IPjU%EFN|eOHUw!3IL+e;-k7bH-vX=P<}aB~k-P zmUV!Zj1KlW`*1qFOTN{_X&g&)!v_UIzNJ5Wb;!W5-FW3X5MtWsK274R{Lrj1Fgzz~ z66av6Ls~WiYlQ2TMk5(NV3xn8B`x?j`6juYrTWzWyEFALuNT?;C_b8lX4O+5-5$T= z*C#&H!YK<_hd#wH{Hdv`MhAoB$I=a~TjHztIGtnjW)6@T-x6W>=8E!;FB!NZfgPpR z10>C#OcdN$#KU0N05h4o=>Wt+Srjj3yFCvdAR>i=uG9zmWJQU9|Bi}40<5bcHcc{3{`S%VBnfN#&i5QY8u#dKS3K*kMY}w6&?+-jBz583tPsjRF5?(>;jB6*py;R zFi!($_b}CfmkA3KKeaOto22}?SZblvP2R?`)~_d=-U9L$Rduz7N64cD&f>N|R@Hqm zZwoZ0if3lroFf_cgM3V9_vly!ohLyl%JCe)95n@4lIL?n)HG&JCFQpgAM8SuQqCPQ zJ~~ZxUF3IrSYiyui*p;)GKbBk{l45*FG^@9(?vtXkE2j|6{B;Mkk)vL{}^GtiIew1 z?K+C@D?x6(*q@mxlWldV5WdW=QkBPlT5LJVMRvTY4(!{o<_+|14o=Lk3Pk#Omgd%p+4keRo1|5;>IZPSXou;suT*SQ zp)YNIwJfhk#T0*}sLizf@Ra};)M1L+@oV=%N6D3mp8)%C<=EqsM=9ekD#Gg}D06|D zGKYI1foq1Hpb8J6qjy-t^>uNp&VPlwAIFWVH@JPKOpN#jZ6^69(>W7+n4Y#kgIn*WRy|D0;twKe zU8in;9X>J;_y^!AO<1cN>Q~eidV?g&0~UD^jL(XH#GDFX*ErY7`F!*=l@eOifd69W zgRn%{#Kun4{bF9a?#nH_rTQmGv6O9(v<3~T(DcBRk3!bRxpvlnKoV*-Jg`A#6}+YF zs;$)$34x2PZcRoW!^=m0wQ|*+rgB^p0trz&f-`x0mvd1KWssfxu9t_j$HCa-9mL=@ z(txj%hoc1zZBXIe$jTt8n~LEV2fL*lQtJ5oU`@O}OPZ~+TGQj&Ojq{@|4oue{d||@ zd(iiNFS5Ilv-o@tT$;prVt+iVnz-f%Lucf|T$s8}0mjR&e;QWwQ1beO%)F>dtZw)< zq3Wj>GyrSHVkH17-G8L^^Z08i{RKd+PPmHe&A9JVLTEk8#4A+8D6k}F@2 zZE9^?f~;M5EnWqLNUrp)WdA47OLP$)CBrGZM)Qayiz@7}fTWYhicj#~AS8ja5F;is zuD7PeE@^`Ta&;d3^vP@<^7bsv0!$gNuekro3<%60Z|j|y;O$v|OaX9nl-0mdHphj< z>>9oBqh!6prd1`|%9Nv0ihEPyQk49?6YSiF^SH%l&>mi39?$k!{e2S9albcSMx2DH zfQts8KF2v&>T#-8>vwNOJ?w%=xnD?{Yv;{5*%V`ZF zrf=D(Pi_lU2X^jH{H-Rq@8m}7G+@lsVxCnPs!jg`3GyCN^1||8g~Ym3w4#%%`x=nt z_0Ny|H&apm#bE-;F%&}f*Y%%&2xSh19uXI=XswKKk)EC4BOhk9qV%{rO2W@8T+6dH z`_ik~mHI4hpq*fKQ6dLbsM%I3M`Fd0>!EnTpYzubrCyVT1Ut8M{zY!&Jm*vhD)hqy zH;fHXA<8o4{EACUOUDMfdnu=k0yqhh@7|gA697i5^IXUN&vpqb!?(h|&VJl*DVwQv zO62F~e~P+D^@O6g1p*Qg@Sk^oWKW469QC*P;VdlbYc4Gi(X$in>f%8}EZc>_=f?x&{W1j~_G68Md_WetYk8A|w5vd}WO4Qs~HRphz?zWibi4 z0mqye$tcv&zETj%0Px*mL#V%2s7yPc<@~zdGyfYi@S7v0P0F>n2wcizF$&}D^2PB^Bs&)WzHyfVsekSzj zuS*Sg;Y5&#H_FLg0>mh$FeN>x9}7Z=L4vw6*8sp789`xmtU z$DPJOKYvP}M(rf&NKqT5puxxHz$}v}_bENQzM6I_B5cp$x?kAg{n@}Xp<}FFiNDT) z1qPAjJ&GoEos#Uxd@L-ooO{_wb0>{YKwuQ$lGKpbs6GXDuwJB!=-Zh0S8xJy?Jm=Y zvmB%0Ua((0ge5l8LJXjpbd;5q-`L+9N-MX|=?}_u*h`W0P--Y(% z1j%!3Uo+fEsO;Wtk-pO1JPo<5{iqEMp=urwh9}s3a1}9$Su2}(8A$|PDi@+!|8u_b z@{jG)Mm(Ka3H~CdBBFo==E4`>9n-a-t)WH#dxuZ7IU9J1bV#lgsX(#aRnEG$eIfVG zds9;%b47MnigX&N*`6fr2J@`S##lrSLqQ}jj-UsD#5&yixX1B}&Jf9z#r1 z6)EqX63|%8xa(^d+E23Iq)X52w@?|pt%7l*)k1Fqxdu6*#kDv;1vVniCBNP6gOjwEbD$w=wyFtnX?eyUw3 z-_IR}LBE>lx;5T*i{bu(u}l_D(_C+j+!%sqi1*1pDrIl?(j$r&WEaX+eepxAU)BF2 zU(w+%vxJg**DNi&fo2Cp5M%%xTJHK+AKTFe%la>VVfi z_3cTk_geMqeDQogJh;(iz3#)FY(8^P`stiqXO)90MlEW7mUM>gEU@$_&wX$AncgW7 zC~Ppdxj^=9aOFb)1?{_lBR!NCSj`kSy{FKv_~8|Zl$;fPCWh>}L&OIWvk96rQprf$ z=wRaVG-`Kw4XMlHr1`n3ef+1(U-ILK)-Qy%iiYf?kMtn~wrZ@ndt-SNS+63*6qrvN zjW2DWIfkIbEN7w@2THeYA1P1~tXSWwwI|42)4SSea5J)=i{xDAPWl=u>Zm4KI7Obp zXghSoy;jM81(G-yGgU7qy$cJJFzHsy*8brjMev9ve_-Du>v3LPPCqp!>vLRyj<}eW zeOP7|Srf36*v|GnU2CV1(BbZ8xFNB#)&MPrO}EwU)Eytjbv}mjtL+4~;zTVPa-qe3 zI%VyTfVq>ef;&Pf7VRG7R%6YSmCWX*Pgl|}c(B`-rYyeC|9k!}UqjVoAV;_zL}9=5 z(`)OcDu=fiHEy(4WfeflD-C}Ge+Guy!iBknJejlZ&}cDX&Hppd zwqqhZ>hl*EWXIJzT`m_Z&WQ`!GL8Ie%PiJ~r`nmQ8NMseRg^~-@zX^!(i~&5*NW<& z$ay(7U{@N(;70kRY{EXTX|23TFj+u|&*OU)Vqg^*x{)+8?UR<)T!ihbljO)*k72wPTdl3mG->niNIR=E#323y7%8(=M*Mr*0Peet6<1fVIemnnw*~yHzv;M)kKSY|KD;`U{t2!?64|7&DaL;GPlq>^|D`6H+_jz98S)c9Cm^&Bq??#XxRg1J_=# zgYaZQHs46o@n$r}Zx(txJiJ0Z1Eth+hQx$K87RGhX>E z_hlbX=0N>yJuukS$O=?;BOPz_Q@y1}3p(buu$%$ck%={s>@=@0nw zA2sVW+*<)(%pCZcEFbHGJiVB?g06SPD#R_*n=7t}9I1dUk0XrUR86o^ z86l0dq2Z=n?Q$S@5qDt(`I1QA5C=(KhqNh>_)2S+Sp-r3)aTg6ee6t-J_32z@c}ns zrO7#+WSmptlKhKQ~KyNxlGc)cd4KV^l-)SFIb5(nhfu!fur+TkL;i|` z$>nlCPz$*oS@7IUSM=G2>bXo9uh;(fPQ8ki=a~ZYp4I-&0YPu{h+bxcsEY^G zBWwQ%wJLpRJH4sFj8Zs6)_u)9vYOau%OHvwXIwP7Ux<^C;u9(Gf*QtfX|$2OQDe;0 z?63Bi@2+d`0sT!1;y2_>W=xD67{BkjWD{PU2%-OJd^+C>=*f7(Z2|nTziQBe$jFIm0WmWLwoxD-r-LZz4?8-0#`yI(D2S1{Sw8gu-ol zfjRqaSvM75nYVA|TE1lUcKs|AiSWKSG4BKvvXG_=Z_Zksfj;(;?x;_SS$pvj0->E!PC{41!i_ zc9%N$@DSu>zIrssIwZ66oSyv}$q?4|D4I;p7Q_izYNmPvvu=00h#jbgSKed~P(j3JrYRPV~GI`LOvl_ib~cFP-s1HKiV{ zt$QMSuUP)%b8W6?S9ovxuEX&0XdNyV3?{&8b`c^=UmNzwrjYf{XYS{P$&C?)W*a0= zLN+Xh)Qm^Yvn*BLXrLN=gzUd6LLz-~gMDR@Wm7x;BY8G*nBLF%MIL#=EO7ZLyWLE= z$8j)ov?0>CN|cb2=SSZW>^ARebO)$Rd8#WXf{3(M$nH%IC)ArhIjIlu@)VImBc@cy zq2LQ0T~?aj++&f6)!r>e*{4n`*mekRCaL^0bXWnA%)9GgWPNYC%%`M!Bf)m%CV!Z_ zVa~(m5pdYEy%_d*fi=JG zawvY+~?-)NASx*IH8=3+o|C^9U%%yl=;xx zVUKm#JTnnt%iKjj`gGHnF`MV^?MRQ)qbpCE+DNwGyQ@vs<8Ov#1Pno`^#!~%B;}Ae zn=)$JOz)s(p_K9!Q}NZ7b+7U9AzR$MdCxHKC>WRcFspulZ79a*>e2qZpDl&~DsNbZ zb;~7w{~EAsS-rnj_b}n*+lF=0ON3{ofSZXb|u7${m{i%`^IL;ZWf609(d&W=`UZl=7k87kLg&$>RnvtfOIa ztmK|!yr&-&2DMeexdrA56?V9Kafi=nMaQlxBH?N0|1OfopYcNb% zFh`uOX*XDKfPt(m3ytFzDX8)B$U+^z613Q+4c@Nfk?{x#sfGT9CJmJX?-;5IufZuE z`(`@{~3cV9W+%Se#zjkk924V0P}BCP#=WU^qtmc^mH=24?ZI*+0KIE~aIbqXDa z(D&dK8o1-vf}@?SVUCuoTJUN?_JD21_bh5)PI@nfit;BHLQ*z42U85{YJhnzdZLN7JDGgoZr@ zoqo^g=Nq#d&3i9`+Hp`2_9)UjbBo+U{Nl&VLF{G1v5)K+dq8;1G0oh^3n>N*rO$lL zjgY`v+9u>Avp@G~i^~a>?$BOym!<4lD+(-gFo(%+YYX{tOnE7r1G*)^F3Qn$lxTsI zcYu-=$v0IFht?UgdKygjmZnHBa&QSDwI8D5BkYemjxVVou^4;K=G?yEZ+=^?#&v*L zv;XbG(O1y%(2+T)S!EfN)yJlg%#_PL)mj1PT3AsNSsHJfWCnq^E3rh77YX;7#VgOp z149N4z90f=TKk^&ewwh85A?o?(nPxfM-~0@0_Yt=Tz*3Afx-pM%%CNyAz@QxNcg1p zl9lSLj2awYnIwQttr0F2;%7B)zMtT6V6 z0g7tpxRy7__dndl@7r2v0T=ug)zkI*8F#e-riNs>s0)7FG`rZok9BM+?RA?*ru0GB zO>abqqzi2;p+dbWyG;x<#A_d%kw0=U7VPBe%D#ULoZ94pVD4<4Sgs#kY?auT7lV?I zG9a69yO=+EhnBZpp9d~j!}@-eB|!0kU2w9tbU=$t=cxkai8eVf9%U1BNiAX9z;1C( zz-tg*BQGkG@DZX>%EW zbJm7lkw8Ls==8EI76SFE@%x-~%CLa!>vivoUVz7`{v4UKNVox2{*Rh77hrekfoGKU zrp+j(+iBtYYOp}uG7k%>^Y2M2GH#U=m>3-VmbZqat&ujD$wHsCD^Ifnv09bm(Wl@4 zf5ewx zXI(KVTLG>Tz?1a9_L$zQi}X}?TG#CQMuGlbq^DlWqYbHVLf&BPhh$D!3b#5~7VHPT zp3|5hd?Q}6^!~R|RWMXi(Uz713V|c7K4tiz1D5T8F}z5hd%dCs|1{)io-0M}BLC#k z!M4{Xd^uU};(MPEz*VzYH(?>*$9fm0u0p@t$>y=}?d=sFw<|xqwTl8;{yF{Q-|DLi zcy@qG`&rcg`NsJVwei0mvls;iOzUVIK;f>gtnvG?avNKiXJQ&jyL@%CL4&0_88xZ+ z0}P4e&9QmMk5#V`=oT;HF@X)}7kj3O%_+O$3EyR}rKEdYi6W<>G&w8o(!=s&%FBal z?Tz7IulKZOhQpArw`FRcdJft@otTXoGxKzTf6e*gqu>LAKMXbQs)P8ZuGOIqG6M4Z zyB5@_aud7K|9O+uh1}ocP!u>nSX|C??%Ss!oPU>G_4C;;_Z~mIaW1g($-IZ{2o7UZV5oZ;2WdOmDT(Sep(uv?2Xrj;afEp${je|ZZSzoQ$` zQfStFI1C1hiq|guynOM!(J{oH93*w?bj|3`pnk4a2gL){Pdg<$_uKBpUZZq-Xhi<+ zLj7->9D{d&Lxiua55||5i!06~le$ut>lrFE&EXGUUKj~pnPn&MiaE)>FFjm=)!r`C|q!BB;fd((p~%gJmW-E?6aF;{_@cJMx=QlJ%9*VI4U zPk(r(P!L?Tsxuv=F5HRa?Yfp3@?sU}+J3M;+-iUhK0D&EFbbI|AkZ4GUm+$+B#E9b zb|0u%ik!xl<;v}m9TcS6>hlP{URJ+R6noE?BGM9 z!EZTZzS!pip{FKcy8iywL9^JV-(G@IT2e!Zr~oWjQCF-^A(gG`kLSJpi`H+0T<;yc zH;WqW=wHh0G3)8OjS=yh-_ObD7p0HmUhM@9sqace>gBsn$>b`w;a>;2kI!-U%+@aj zeQ?=fHp%wClkL3TJ$q7{60fg0BG#(awTJQt`_r$H^+5|>DZ(W^)!aQY0ofwfRG~qx zEAwX;Y`~&a@9&3shH{r_-OnN|(px7RH{nN0H;DnqHJI*rt9C{3j}Pc!4dbktbC0E> z3;2Cp$~hfrsVv3uRq>sDT?^J!cO@>&D|M`sbe?VAvCmaLnCb{tF&vm1rp*L8 zz)kC&?M^kw8N#f_=UV)$o5%h-O$9r3Flp4;d-8q#C03fw2~u>>?7G_&^nTCWB7Ctr z34VhHA67cprBX^9ZdlPW?fP_-NEsweB?d6}(q^&dW_(zMVhX8?XmH-7MY_Rk_XSe> zJ2nFq;0W^C{2y7r`^81{@R+p|MY6g+tbE=F0FiKPCXU#(I7<8pI*L@0@CHEVS zUwE@)A(lN!6dJ@h09{m1kMvWSj-d}m<9$H=X-ut(epsRrz#Wk0S_HOZLk>IGWhRrn z&bf(ws{+PC9nGj`EiMGlGj3dWc*PiQz2gbK(q;7K!QsM_CWg8tIB6(vekH7^J=m=p z7Ym>u0b{V2dkf)EkI(`AQA9H;v%ng;0c+dU`Ci4u%5t}PU^>Y4$V@OkRwE^~yW{c! z>uA}}7O~Vj73q%@2p?#{{*Npq)i1#vobZEao2%_0dj(u6KX86~X1_cDlqZ|o-t4$>8v4;8gXuJQ0v%IYemouz zLN@I#j|y?XE89t%<@a|8U0zT%uQsXa-CKH#uj>q#A;czpTZutIx%2mwN@|8hUAN_h zdN!y8SA{H79|@wh03k^2)R`x-A<%JbBS|7b2U{CMc;xPEuhB-)UEb~17b1T}fqmEqsFPIM-%F5YB;@1jA)x*qpX^PwQUI=$WeS`P_l}w1>^r6{|fNHJddDKO^#6d&mHEG9BfRk?BoI{bDVsk?l># zZj)g4{;O8Q(nwu)cQF?_zMh*mQI7*6P23E=4nxb6B359#dwa|swD;>@&hh85h2m4k z4<;w!MmMk9SA2 z{S27=gYcS5H-VtR#yPlLC_Duu>sYI;zlI39-XO?nH^a+0L@w=JM(8p0do%(|0erht z-#FP8+V?QP@n>c_3eycUl}*s6JMf1R^~Vpw($JkM>7#*vZ+a}yYI(z~uxlScK>kI% zqN|!ad~2+`Fn=7^!)qv{32v#ZS8ZxKw{&1;peyf)?yb78#4~r4q%qOfJnFgAJBA4i zE+!_-7c)&bVAlk~-TgOagGgJp<0?}0Q=7#lC5hhoHpgFO2)-cTF!uJWXNaR~&+*~r z#v$X$NPnXA45ii5vee7Tu85u3?=l|x$>X~r0$fc));8>>AA)s&ZaVm>;-QJ-os2Zz zDPPx3$~O*JXW5=Lt|YGm z)DG-uXhMHGvcf{{o|E|&`o>$Y30;PT{e(S-smhrLC1?T}{b+whOeVFp{>8f6M`GDwz?~=4I`&25)wq4~3u6j{#J`{PIEB7X2str>LkulBhc7CtCT7uv0#D000 z6i)p%gwj6eDC4im@)_<;9C1(K9V`WurnMWZP;8MI%P_s!Jv+#*SsggM(i4cPUQ|_4 z85vGONQ@V$`L#iBUL_%h)`Eht74%NfMw}m3YqTa3-&2GWBUu~>c3jdDFI^u?Ll;K~Sl&eLC^oNncM&!o@b8ZIt0{n(Y3qU4%C%#5v^iCXB29ZHZ#MMA(Mc*cr$PQEWAoZ5DN_ z{!XLTQ-`?_0Fm)XOImKek3N{(2Zl5pawd%KWNj~ zNHJ-1-%?Ts*vD$U7iqw14qlszA@{@OIEP@e33h#kcBp*Z)oTCIJ*^l?TI6*gt~3Vo zf947zIt*(rtUAOl|J+(&V9;YQMmjy2i8NgFa#7*>{Ou0sfPy@`)Ut~kFPryoa2 zDo{KqQ+b)W5~7xD!?TR@v6_{teAevf$Df0B z9$eYgH9a3g_g>8&q#*fBShCP_qWO7E*j(bL3F|hyX2iB3Cuy5u z7j@3~d%ZGq(Wn%!O#=H-x;M`%c~DH%&HU6YEzJ%oF#R`$aL&OO4|QWbYlAhBy!jF# zib->mp>RjvMVt9W>sK?r^4Y?pDUm#bT%#qE>@HyV(8BiM2T{!oNHV{afYzprr(%!1 zOe%qjligK5_3CQRa+wls&1zf)g^K4 z*XF;$9g4s;;Ki&f1WE^`1056-BhJAZvRX1oOy+&a>|AV-YRVj27kq53-B;uOo&6o` zG2DBhJQ(w2JqO0II-Omh2XlY9vW5`f*=(5K3bN9!zxRE2t3JpXL%!h#YZd_cOWFkW zi18P&Qj;)#^j)lK!!Y2(k8*S_okSNEP;;UJR9%_CB(+!05B>R>oz+YH3GR>N7jA&0 z)w+j$I`IPws8Mf8SM~BZ>$CkpsoblMO>%C_?NVV`zzchqdTNn-KjVUZxd0hSthTJ^ zoAk%%(}#Iuo$D|icDB1I{bm+tyRaa3rI22h?5!=i&n%ZI(K8*Y9J<*ghKr%mdMZ*XzppB$fl+_l}Bs9M~#{r!sh zL6`)LX#EgCW~dujht_P}^ZmaLHt>wPk#^?Lo649SK#1cmZe9 zYVqd2dK7V$5fq~kmki-z-TAf#VAW31Sl9|H?+kAp6w^R&f-5T+&mFLmL5tE$S|~CF zxS}w8*M%(&(&-M2p1~*I)GylDq(4=V)cB{XQfXA!zVGddn%RW#)VHkn4H#AW)kp)NqN2holgqJ^c+w;RODnjo=@f%(N zwZyuXWjwlS8zwV+m*lh=_`%~$ z%BM8h!7>sl`VJilOmOS1mn|c~+~y=La|&1u(I=0;nH=hk$fq4SEW0K1{lLrwy0eQk z`Y2*m#;aA()H&p}epaFEw`IP~Wf%V=P&TpfVg57kPx2gxvR?2t+t*HEsCCbqd`d*2Lh^ywz3!k4VvH0TdS2hV;IfQhvIX03yT+ZLA0&_An6S$S3! zTt-BhXLtz#GYrS=N2t#%#H{hmTV;S&;K#i5QNM#@{%?}rSQSUl3u1kq-N(j)m4!&C z)X%I`DGv+uZhb>iq22rDrvX@DsDdQiY~v*(L_QlHR66NG^j@%5Eds*&o{og6#Mpss zMMa0~;N83#exJ5SPK=3u!acUzO`iCo%gb57u1mX~q7T2W0^aB%HF&B+(Z-X2oQPI+ z>X}arkO!_s`qOvHX55>FxXO$THt#+Gdo0h2OG?roK>&l5V_Ge7Vv4OS`ae;W_=AL4o`be7VpP(}G5zy|hlbiP9#ryL731pUA+(%bYF zi!@sHy7iB52OKNPq?wYPRpDRnnD7lwsTWz)-_O)S8E)P!e9m&@>O-Lm9m*}5!xLi( zX(h{Elg&^$cwS_~p1Xc-pTgN}!fN@&Xmpp7d_;BYaXdz$6L$_62;e zK*a*(EsOwd`I0h1DBT198fv@@ylCW^1q53M4uAG;RE8^ktu9zNV`!9*MQ%Py3_K{a z{4TFd3hjT3S2N>|swovL@#(<4M*iu3SwGaCkp;1b4Zhyi)yr`q(R*T#46z7aCm;!} z#}=E~RH3SIu>)=t;^#fE6_*?#Pz;NuVi1P_+4X)gys4j}-qM>p2#?mEN`q=YG2R zl04^}wX@3J>$mn^d!4<@=kx)C!Il*kCTfE})~{gR!2m*j!!Ok?tt*fG9{Wb;M6t56 zAm{~jYK|nE2TLkfn*VrajC0CNxiC~ffxZAgxHtH4Y-e#ja8k%*IgSV{4{OX2gsU@A zyg_|&dv;66;MZdU3d+R3{(7)*eZ?vJ$*nhl8C)T|rOq2V2P(qnZ4Cxhuf91fc3%it ziIw5G^#Gy~FM%LP)!(5AUMqjNGC*Vniw52LWGSWU% zHcBWAZu>gcC+q5$$*Gc#v#JA@f6rcTcr)(3|4$RWY6{Y1^2P*4AI`{;Uj)kOv~C>A z)McrPdNSK)rj{q<5+3wsFMU!^T4~P1v>M%i4g#efQNqxGWibbXFkmV4g6Lbphn(2a ze}XqSYfyuuPevdY0(7A2S2HViw&}dxy9Lzw(;6qA?VV^Z*I;}rK$)K!^e~?4UEc|} z(-I!t+oS{>I2YUd6@8n-nr*r`eQ%nH4M!Yw)tHi_`!r? z=Q9WS$h_xPAdR4&ZkAf3EXXEr@5Fh^er=;t>|a(mYt3V6t8%5=LAXcA;E>d(TiU8c9mx+5`Lzj3eC2GQeI z5x}m)2|!$1KwKw4T^bWsnw#uuS2_P=SKHo157FBRdZ`2CS^S+({rJ0cR+<+|v0`a? zKfAfr+sC0^=IOV8aBEj;agwj?_$$c&88Vog@^s7jDS+F7Od5B+%C+H^sU_`P*k`$w zG3<(!CCxXu@X)JalGt{JKba|X%3foGcHQFsjrf{^*cfP_A?tbH9(SWE{d*fohI2Ux zyHjercvtN0QA=P?)siaqr{w{Ow~1&RyJDSzP#P$M+mU|s zm>+aguQa6E8~krPaD*I72Z7=NowP4H0BzZ5>;6zBsJN#Se zUZ@C&wV(8}65~IfVf^px*Z(z9={3ZNE>rOgh^{GNBMYZ`3H&Q0or~g^ENEx~sm>FV zD{mn^+2~az-e=~r`bFcpw{7#DH7u;K`y6_CCwV_`dJU+38Gz1{*ZRrKzO^w}zf%P> zcbJC_+z4T`g}5oAS#HOl0cFiVF#@39{gd<1)^usqGS4t#eezUY*z9uu3KPZWJrO5Z z+Yzbq3AYGsj#^F9#9+$6s7_;e|b1YV3J-;%TrqTNQmMsFomB|FV<;O1@?6)ec+aH z;pY2CZ4N~(fj~+&mVcT$?*h_Y?GMB%5iQM`XndLPqJ<1VMDN$02F_s7-!=jWoT9x+ zqBW8zfDzqaBF+VStpc-TjaMo{mV;3nBDlu~(i|R&N8RQWP-&~Z2N$#zKo>gVj_I1X_~c}rcAo;M{ZGKT zUurb-+AoUP=sRU18d@J!npIf%=-w={;8HgrMZ;=hiT?x~hnFKTaxGo4(!d1`S6~sn zyykm?#H7^St!K~qm=eM4T&KW^_0GBQ)-7IAw_0|9>@36&IigocFTmS(eShjQ?B8*( z)cwLAXEW!J+u9F_uVjF?gW^&E!}zDG3k4Ru4ru~#o{VeYXF16Q;J1qBN000%W-c6n z1glv3PEe*Ti@&(ImIf>Wun&RXz}-4&{mFAq6Txc)w|-mp7ql;5aKD;HN3k->p|#=1elr{2@Z- zHB5(1N(DHg_;8fEy%Hpi=2G4-utur!w5rCBYzccu*okf&KO#2t0pjC%xdots#*KqZ zlLIVtb~9KoS&^4VSZAG?lnNU1K=DQv3sexsZR(PW1jDwd@Ae6YH}_(&wm+N?(7R^3 zKBDibj<|Z9H;sY(_4C$#r&62L`snByKE|^Ty=vz~%t3brxa?8lj_9LPlpAMOVprB* z6_F#Kq!VX7+SI_boscV-LED&`S0^m3eWvj`(y>-=pVfwxDGizTbKM)+&snwg*uZ?e zw-X*-W+Xi3 z9vJf@cLh0ml9eboV>Omf)wxy&#NbDu>#bJi0v8TTheOdQm)=EZe$P8thz$xNzY*VV zOGe{_yr}Yx7_&Uh8njql1`2YT9yj3p#ji|-_0*gX>t|N(&!ITA>W`%nt4W8edgF~N z0_Y!-smgV`!9!G|lJ{GP5gl1-uIj_&cLggh=Ni$jdSn?`$NSYIY-GV`nm6GqoYN^l?oHpd#7;~ciztj zhZr4j3UbQUtGJYn@b7QB;|kex)l;dKduh&_D;s{uwlU=55W&xVTHaIkTUo-Wi`ilZ0fU$(lS~~_my$f>%rDm$Ngdxm;APC;*SiLg2|J@@nNAjzT`}i93sk;2+3td;n5_zHHGNbhUA?7W4G`pj-6Qbs6^@s&7K;G@Z0Ck8+nk7`iRaX@6JoB~v7_IQA~VE0@x82TFAU z+wE$iK5Jv*_2fSIcF=(f-pG{bp;V^x9`%nmB^I6xtz11G6E z3NI+a>{8;dgG1QEzHjl>lI+~(G3X3$%s(b5=o%)X;8lyE+o z;BjjZQQ;2iBBW@k2@2qkDx{1X6(cNmvBp@8Z@jt!jQy5CCox45;{yXwx z+uk7W*6&DaYJuQO0;Cd;Zb}xM6J1DIg#}Q@!6HNqj~MK#Uvu6>JFtcTf52bPZwHfy z8d{f@gY=;jwJzL8oVmZB9kSZ>jXhuMJ4QS}8E)GXXdA01AdWb`SCTkAq5wf#XXMnC zjC%t(8GjtA+9$r!_!boBIsxnv(r^KskvR5j3oAJ#o*5OWRJE?8-hQMDapf!`%WK?4s) z_}XiR<=5&IgFcYQ8q`KcZT%9&0Yi;fptM$?qFEe8I9fF^KxwY=;ms(W z!2r83de6Sdl6-A5l}E+Q73dW|(wYz^?DQ0cpuS&2G``&e@=6xVfR16d4FdPm!TUj( zy8F9SG;$^8+30oYEMPwPqM#y6K-4CxS%RxtDPSt9Ql>{`xE|yev&M@$a_Xhmb#S<5 zU-oM7$1B*~2f^f;pdh?X{gPUI)cm)HD?>9j-LDu}bDN}^8&07TtEKXA^u;CKRr#9y zJ3ESB8``c9t-N4;Qr|E&S*uv%y|d=5G;^O3grk^ADM`@FqpCS=heZv6QKLC-7%)os z$Hh>PthqPx(58}Ozm=IkCdH9k&wGT~R+iDT1OnDMYKqwXtDk)x^Z! zPmS+_0v+36_Sf{e#ML`svarWzYFvYOH1+-n+&X9m-`fe%uvq;w3}OFzKKxL&7ngtG z0F|^adzyD!h`(^;+KWBHwjzNBaI!}z&@CVrxs z%=_73%Zb%Hl%eZlMAM3lNMotqacyh%b9i2_t=e4g4~>E8B{eloN=CIBTSgb}J2f2{ zuV4YS_8A?6S%$^TA*aot8dEgVt7oCR-dj3iVPN?bz(r9%ro4JY&*aZ2_CB$Fm=LXg z3(cnu>6Bm*_2E)$q#bEYsN^7H@WlThfTy4X2lf~_MU6zd4ryD2? z%uG#9okHxH90mLJVJBxLB%!skK$40`?1vMKeAdnsW<6^Wji;gi#pXxl-wjGKjr+E~ z2>AGXwBx`ekrDOJTtEn<&cPQ3ldq(uB_51dyGnti_gz$N#?9pA287Hp2}dw^_(8nX z*PGReH6p+sSL=`aRPMX>FC6c_FVM}}P|n??uoyxHjtpo%^LeMOQr8xa_g$)tS?H17 z$f3{RLayfDmpR56r)O4xF)>&-TJj*R-p{-!Oe~HU^)k|Xy{`&xOY41BWAox~4G#j2 ztv>KNOpTc80>jS7ZbnMQkDg$yCVR{9P+q5!RZ8ncy{trHHZvv!zkdDx-<#V-HI5&E z+(5p`%5&^|JpW(0x@d2Ym>GVs`}~=e>#oHP;;0{FC$noJKeKde<3GzE_rDWgiktdd z!{phNTJo8Mq=Lhnn^kS}AI@*Jak+cldwsve=9W~iqLT?SuZATS`u`aDI)@6i`1L3z z>V*m3aA%XWeWSv^4-Kj#Kw-V&IL%l;Ru`ilaw#RJ#mhVESavhOD-F>$AGUj{ajNXe z7cnpG34h6Aw@WPaf`f!|Sudh?ZC*W3IhkO8cWcz=Rmm?R7CWTRO@AZn-ll*KdH|Zg zJa2;kEHN))r%1m5E1$^6X{#pA0H~#@I{B>dW^Mq5ZeZcFsaT_v2``ejuAJKSr8t#1 z%V<(_%chZmCBYNEOzX!+B@h^Btyh7EP#fIZ1Zw+9 z&7_;fAun@}&}_nII~U2{2X`vO&{w-OZ-`|AJ)D=*C7k!D`=sFa?B>GjKM1P4-zDb% z-kn#q=AcO5xKK&Kt5n!xoM?=k82N*^wTD49F#DkF2%UnKC};%#@?AIidgU?Q!n!)$ z7`%I`Oa#o5_bCHWMQX}@blcr22E7^2Bg}v%6u` z=kWqouq(9MRxE;fv4+Qv4T%Ny7DQO1Syruj+noNwsIBMaJ?hj?D@`t6sk`~5mMPgm zx7}ewc|o{Wtwo6br)qYv6f=cf%WDfa!%`-7w{eEywC7GvPz0;mf|?p{OK~7CttDcd zFa_QgLD%X>lJ+@4Bj9u(A!+hvZaBE7D4trdu|m-BrTujSC49J7b<_j-AsKcXh`RP2 ziqYKAakh@&Tz_s$Tdgl8bq&@5hvb#?ZT7l+gZ%L=&NWXNtp(#pY^V$MqVPtu5)GNj zFH~Q4M$@6dk2$sn^$S{yOlbJsv{?rWBeKh3#mExv(yHD}Bc=Xh9KhL`)t52>%e z$nrU=F6B+>kva|-S2ou{>{hglNjcCDZO|Ck?_O@INMcbrn3^b~r(d=K;8|CPkk2#~ z34$FqV_`4lhoPB3RO3y{tJTx>voM2cAr(2RFv_q4#%!tF+TGoKlw%_wq2o91>bxUB zt(S1DB`iVdS~ZXz!WXu({{Fqz^%ruHm^lXnps9UExaMm3k%8=NMt~jA!So&teb5gi zge|R(hs_ISDP+ivXnXttt;9K&&ul=DZ*R1wjHfxyGlqhj!!ul0l zO~6u}!2$5HnA=QlXbu}F7JDX##{FI1U3ZEm-hc1*=;3iJrHrEkw=YYQQ~*SndRKkr^G3Jaj5TTs`wg;-j%BfCuqxm)WiD9Id5P--odonhMD#(jPrX*(#(*_62V z-*tzQLqG_Qa*q(mJS8V9RE)7cG^xs6HL8gS%f_p?0aKsSL6h`B8Bus;W6`l@Ja^XZ zea#Ut03=u4YH)>`ZuL6YXyV~I4CX`*ycy?*h@g|lB2uW-_+VAt+Z1iSj(ZtW=%JQKmiRi(}OcJcI&nrn;PTm55GB9 z@sOx0_G;|`*4~Hg_5lGs@1t%@=cgk{2rhBdy&q>`Dl3*JCDTg6PFgk5b!ols*pn}o zj;WKB@QNQevgjoHhy2W;{wM1z{`1@*chDo)=s{@2 zxvN|at7&4cGz(8c7K*15z-PaWH*yDZ8X?r-WuTRpkdx{|i)^$L_m zVUw3EZsXEAx6v(@{ap;po+0<O*0qLT*KG=$;DqnUU#dp|)N$&D9m{Te9y15-O9SNDiKbFm0WRiN^8New zgN+9Q9>VWx%27mqV!@2_3Vxby3S=Z52u z1mm6pD)CBLM&InULPu+OG4@d|><_sRsaY%Wydci)_XOs>^GHKQ-}5GNzH5$Mq_=TUX*3`$@LCF_f``J(M}tB7$`T zn<~r3Ha3rq`eeG=*9&}NW#-?9sbC3`l=ORLH#b z*Gtl?87V7tOd!L;wfg=sw(OF@hV|Zv^RR7c(aiO^Vn5A4=Z?;}Qjv0F+%{j^ss)|w z0b8AINvAJfOINp_I{!C!VoIgAG zRWZ}Yc@A2yucU2=SsR%{uZ8xn>j5SLjmv<$JUgW3n?vWoJwYwA_?glZSqVF)bXbqz z9y(k*vyY5`+a9Hbbt87VslO5NIreTj1;&!GsRcuQ<}KMa&lVQ`(SGW!Pv59p{NuvB zZw{v;HF(i~>~!6^AeYpH8bn#Dp3K@ENbl;@q4c_Q6(>^5u_ITbYEuGJ-cB6tHHgVq zP9kOZ7mPN4yjIjBd}`^)%-lHaV99X>k0|qn9sF0z6{CP;+`aoJpgvW~P*p;=>6bh%<9b5F0 z7l+iG%8BnBNw<)J6U~i#Z^B2jeIe>6dQ-dE^|@owV8eztG0Tx3Z6$&)z)xKFF@F=^?6TGMBRmmI9;UcXs8`@n;+_(UFFZ34WNZSK=ya7AD3h;>*P zer8RfX$iiGOtjwxovpmGoG(4sn0?#8oh@KvS-^E7UB;obaD>*|!3|l3 zZi!ME{Csxy?6a`Q!HT3SYOph{&-3#wwW9Mrr$+@EZ`{9s==w6PQTFe)ed)Ck5#&l2 z5=yyY+@d*U)ZBQa_b?5IJ}L0@P3eEN1lFC=WPBX&UmZ24!Ud;7rcBy)i`50U))4M; z^F(H(PAEzCFsnPyuYXtO6l&312w<3`R#Y{FzSlh}N2ReTz5B9x z)lAu6?!7oh2zA7AM`D>&Bw=p!P&yB?-^P6EdO2>iALtTq*@+b5Q~`rISm*Bi2O%Oh z#Fxrk`)&!@32qrao#cV;cN|};7X#}j81fcVf*_4=T|O&jXHaR~3H5!wSga!640BzO zn8)ua z$oTs0G`ldA0aTLi18h>r%_dx+vg*e@s*B7FWp`%9vu~ z%cm=X>&h%o$c4-E)()!I)I{9*Zloxs&R;c&eKBEe?B&6_nez1(ymj09kl=lWDvV>qlKZJ=_EIjM}F!DN=YEK z8SF}f=P7(L;CLDvKd6z7z!}Q=Y2}{eRCo|!m(=0|^mb$oXGGaiPQapK%@M?X->uBwZ_l5(Hitq!(wanM*XRHNhXd#>PW0_7Z2WB zGTSGtZ(-sH2_4I3yWd5{biToC)G-Rrx1NlrjH7ZC?rMm&Bb|KZwDrr*z7fS6{rfX+ zj@wS$+<4+XZFxKiSFN@=8TsV>dxHA&IHc8K52?<1Z(ulc8w{AdW?5(hw*OJq=3#-` zj>w{S_Q@wpzUwa>qV|S}ETypeterRYFjK9_v@6!9#UV=j+rsW;1p|($a6}r?GbZ{f z-4<@#g^pUDTVXPVSj8VJ4Kbz~s7J!pW>F?{jc&g{Ct{h1Q`Wqej<&B5A+K#gnz`1$ zrY0fg{ge6j3Gp{5?TO;QKYpAG66d)0XKwNIz0Fk-kk!SXxv|n;e$?`wJkYT@X9N`< n{aMlK{NJtpZzKN^oh*og=!6f+SeF#Bkt{FT{aJaz>)!tXz%XaA literal 0 HcmV?d00001 diff --git a/qtfs/log.h b/qtfs/log.h new file mode 100644 index 0000000..74e81f9 --- /dev/null +++ b/qtfs/log.h @@ -0,0 +1,106 @@ +#ifndef __QTFS_LOG_H__ +#define __QTFS_LOG_H__ + +#include + +enum level { + LOG_NONE, + LOG_ERROR, + LOG_WARN, + LOG_INFO, + LOG_DEBUG +}; + +extern int log_level; + +#define qtfs_err(fmt, ...) \ +( \ + { \ + if (likely(log_level >= LOG_ERROR)) { \ + pr_err("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +static inline int qtfs_log_init(char *level) { + if (!strcmp(level, "WARN")) { + log_level = LOG_WARN; + } else if (!strcmp(level, "INFO")) { + log_level = LOG_INFO; + } else if (!strcmp(level, "DEBUG")) { + log_level = LOG_DEBUG; + } else if (!strcmp(level, "NONE")) { + log_level = LOG_NONE; + } else if(!strcmp(level, "ERROR")){ + log_level = LOG_ERROR; + } else { + qtfs_err("qtfs log set failed, unknown type:%s.", level); + return QTERROR; + } + return QTOK; +} + + +#define qtfs_warn(fmt, ...) \ +( \ + { \ + if (unlikely(log_level >= LOG_WARN)) { \ + pr_warn("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_info(fmt, ...) \ +( \ +{ \ + if (unlikely(log_level >= LOG_INFO)) { \ + pr_info("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_debug(fmt, ...) \ +( \ +{ \ + if (unlikely(log_level >= LOG_DEBUG)) { \ + pr_debug("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_err_ratelimited(fmt, ...) \ +( \ + { \ + if (likely(log_level >= LOG_ERROR)) { \ + pr_err_ratelimited("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_info_ratelimited(fmt, ...) \ +( \ +{ \ + if (unlikely(log_level >= LOG_INFO)) { \ + pr_info_ratelimited("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + +#define qtfs_warn_ratelimited(fmt, ...) \ +( \ +{ \ + if (unlikely(log_level >= LOG_WARN)) { \ + pr_warn_ratelimited("[%s::%s:%4d] " fmt, \ + KBUILD_MODNAME, kbasename(__FILE__), __LINE__, ##__VA_ARGS__); \ + } \ +} \ +) + + +#endif diff --git a/qtfs/misc.c b/qtfs/misc.c new file mode 100644 index 0000000..2c694ed --- /dev/null +++ b/qtfs/misc.c @@ -0,0 +1,154 @@ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "comm.h" +#include "log.h" +#include "req.h" +#include "conn.h" + +extern struct file_operations qtfs_misc_fops; + +static struct miscdevice qtfs_misc_dev = { + .minor = MISC_DYNAMIC_MINOR, +#ifndef QTFS_CLIENT + .name = "qtfs_server", +#else + .name = "qtfs_client", +#endif + .fops = &qtfs_misc_fops, +}; + +int qtfs_misc_register(void) +{ + int ret = misc_register(&qtfs_misc_dev); + if (ret) { + qtfs_err("qtfs misc register failed, ret:%d.", ret); + return -EFAULT; + } + return 0; +} + +void qtfs_misc_destroy(void) +{ + misc_deregister(&qtfs_misc_dev); + return; +} + +void qtfs_misc_flush_threadstate(void) +{ + int i; + for (i = 0; i < QTFS_MAX_THREADS; i++) { + if (qtfs_thread_var[i] == NULL) { + qtfs_diag_info->thread_state[i] = -1; + continue; + } + qtfs_diag_info->thread_state[i] = qtfs_thread_var[i]->state; + } + qtfs_diag_info->epoll_state = (qtfs_epoll_var == NULL) ? -1 : qtfs_epoll_var->state; +} + +void qtfs_req_size(void) +{ + qtfs_diag_info->req_size[QTFS_REQ_NULL] = sizeof(struct qtreq); + qtfs_diag_info->req_size[QTFS_REQ_IOCTL] = sizeof(struct qtreq_ioctl); + qtfs_diag_info->req_size[QTFS_REQ_STATFS] = sizeof(struct qtreq_statfs); + qtfs_diag_info->req_size[QTFS_REQ_MOUNT] = sizeof(struct qtreq_mount); + qtfs_diag_info->req_size[QTFS_REQ_OPEN] = sizeof(struct qtreq_open); + qtfs_diag_info->req_size[QTFS_REQ_CLOSE] = sizeof(struct qtreq_close); + qtfs_diag_info->req_size[QTFS_REQ_READ] = sizeof(struct qtreq_read); + qtfs_diag_info->req_size[QTFS_REQ_READITER] = sizeof(struct qtreq_readiter); + qtfs_diag_info->req_size[QTFS_REQ_WRITE] = sizeof(struct qtreq_write); + qtfs_diag_info->req_size[QTFS_REQ_LOOKUP] = sizeof(struct qtreq_lookup); + qtfs_diag_info->req_size[QTFS_REQ_READDIR] = sizeof(struct qtreq_readdir); + qtfs_diag_info->req_size[QTFS_REQ_MKDIR] = sizeof(struct qtreq_mkdir); + qtfs_diag_info->req_size[QTFS_REQ_RMDIR] = sizeof(struct qtreq_rmdir); + qtfs_diag_info->req_size[QTFS_REQ_GETATTR] = sizeof(struct qtreq_getattr); + qtfs_diag_info->req_size[QTFS_REQ_SETATTR] = sizeof(struct qtreq_setattr); + qtfs_diag_info->req_size[QTFS_REQ_ICREATE] = sizeof(struct qtreq_icreate); + qtfs_diag_info->req_size[QTFS_REQ_MKNOD] = sizeof(struct qtreq_mknod); + qtfs_diag_info->req_size[QTFS_REQ_UNLINK] = sizeof(struct qtreq_unlink); + qtfs_diag_info->req_size[QTFS_REQ_SYMLINK] = sizeof(struct qtreq_symlink); + qtfs_diag_info->req_size[QTFS_REQ_LINK] = sizeof(struct qtreq_link); + qtfs_diag_info->req_size[QTFS_REQ_GETLINK] = sizeof(struct qtreq_getlink); + qtfs_diag_info->req_size[QTFS_REQ_READLINK] = sizeof(struct qtreq_readlink); + qtfs_diag_info->req_size[QTFS_REQ_RENAME] = sizeof(struct qtreq_rename); + qtfs_diag_info->req_size[QTFS_REQ_XATTRLIST] = sizeof(struct qtreq_xattrlist); + qtfs_diag_info->req_size[QTFS_REQ_XATTRGET] = sizeof(struct qtreq_xattrget); + qtfs_diag_info->req_size[QTFS_REQ_SYSMOUNT] = sizeof(struct qtreq_sysmount); + qtfs_diag_info->req_size[QTFS_REQ_SYSUMOUNT] = sizeof(struct qtreq_sysumount); + qtfs_diag_info->req_size[QTFS_REQ_FIFOPOLL] = sizeof(struct qtreq_poll); + qtfs_diag_info->req_size[QTFS_REQ_EPOLL_CTL] = sizeof(struct qtreq_epollctl); + qtfs_diag_info->req_size[QTFS_REQ_EPOLL_EVENT] = sizeof(struct qtreq_epollevt); + + qtfs_diag_info->rsp_size[QTFS_REQ_NULL] = sizeof(struct qtreq); + qtfs_diag_info->rsp_size[QTFS_REQ_IOCTL] = sizeof(struct qtrsp_ioctl); + qtfs_diag_info->rsp_size[QTFS_REQ_STATFS] = sizeof(struct qtrsp_statfs); + qtfs_diag_info->rsp_size[QTFS_REQ_MOUNT] = sizeof(struct qtrsp_mount); + qtfs_diag_info->rsp_size[QTFS_REQ_OPEN] = sizeof(struct qtrsp_open); + qtfs_diag_info->rsp_size[QTFS_REQ_CLOSE] = sizeof(struct qtrsp_close); + qtfs_diag_info->rsp_size[QTFS_REQ_READ] = sizeof(struct qtrsp_read); + qtfs_diag_info->rsp_size[QTFS_REQ_READITER] = sizeof(struct qtrsp_readiter); + qtfs_diag_info->rsp_size[QTFS_REQ_WRITE] = sizeof(struct qtrsp_write); + qtfs_diag_info->rsp_size[QTFS_REQ_LOOKUP] = sizeof(struct qtrsp_lookup); + qtfs_diag_info->rsp_size[QTFS_REQ_READDIR] = sizeof(struct qtrsp_readdir); + qtfs_diag_info->rsp_size[QTFS_REQ_MKDIR] = sizeof(struct qtrsp_mkdir); + qtfs_diag_info->rsp_size[QTFS_REQ_RMDIR] = sizeof(struct qtrsp_rmdir); + qtfs_diag_info->rsp_size[QTFS_REQ_GETATTR] = sizeof(struct qtrsp_getattr); + qtfs_diag_info->rsp_size[QTFS_REQ_SETATTR] = sizeof(struct qtrsp_setattr); + qtfs_diag_info->rsp_size[QTFS_REQ_ICREATE] = sizeof(struct qtrsp_icreate); + qtfs_diag_info->rsp_size[QTFS_REQ_MKNOD] = sizeof(struct qtrsp_mknod); + qtfs_diag_info->rsp_size[QTFS_REQ_UNLINK] = sizeof(struct qtrsp_unlink); + qtfs_diag_info->rsp_size[QTFS_REQ_SYMLINK] = sizeof(struct qtrsp_symlink); + qtfs_diag_info->rsp_size[QTFS_REQ_LINK] = sizeof(struct qtrsp_link); + qtfs_diag_info->rsp_size[QTFS_REQ_GETLINK] = sizeof(struct qtrsp_getlink); + qtfs_diag_info->rsp_size[QTFS_REQ_READLINK] = sizeof(struct qtrsp_readlink); + qtfs_diag_info->rsp_size[QTFS_REQ_RENAME] = sizeof(struct qtrsp_rename); + qtfs_diag_info->rsp_size[QTFS_REQ_XATTRLIST] = sizeof(struct qtrsp_xattrlist); + qtfs_diag_info->rsp_size[QTFS_REQ_XATTRGET] = sizeof(struct qtrsp_xattrget); + qtfs_diag_info->rsp_size[QTFS_REQ_SYSMOUNT] = sizeof(struct qtrsp_sysmount); + qtfs_diag_info->rsp_size[QTFS_REQ_SYSUMOUNT] = sizeof(struct qtrsp_sysumount); + qtfs_diag_info->rsp_size[QTFS_REQ_FIFOPOLL] = sizeof(struct qtrsp_poll); + qtfs_diag_info->rsp_size[QTFS_REQ_EPOLL_CTL] = sizeof(struct qtrsp_epollctl); + qtfs_diag_info->rsp_size[QTFS_REQ_EPOLL_EVENT] = sizeof(struct qtrsp_epollevt); +} + +long qtfs_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + long ret = QTOK; + qtfs_info("qtfs client misc ioctl."); + switch (cmd) { + case QTFS_IOCTL_ALLINFO: + if (qtfs_diag_info == NULL) { + qtfs_err("ioctl allinfo failed, qtfs_diag_info is invalid."); + break; + } + qtfs_req_size(); + qtfs_diag_info->log_level = log_level; + qtfs_misc_flush_threadstate(); + qtfs_conn_list_cnt(); + if (copy_to_user((void *)arg, qtfs_diag_info, sizeof(struct qtinfo))) { + qtfs_err("ioctl allinfo copy to user failed."); + } + break; + case QTFS_IOCTL_CLEARALL: + qtinfo_clear(); + break; + case QTFS_IOCTL_LOGLEVEL: + { + char level_str[QTFS_LOGLEVEL_STRLEN] = {0}; + if (arg == 0 || copy_from_user(level_str, (void *)arg, QTFS_LOGLEVEL_STRLEN - 1)) { + qtfs_err("ioctl set log level failed, arg:%lu.", arg); + break; + } + ret = (long)qtfs_log_init(level_str); + break; + } + } + return ret; +} diff --git a/qtfs/qtfs/Makefile b/qtfs/qtfs/Makefile new file mode 100644 index 0000000..f3c6014 --- /dev/null +++ b/qtfs/qtfs/Makefile @@ -0,0 +1,14 @@ +ccflags-y += -I$(src)/../ -I$(src) -DQTFS_CLIENT +KBUILD=/lib/modules/$(shell uname -r)/build/ + +obj-m:=qtfs.o +qtfs-objs:=qtfs-mod.o sb.o syscall.o ../conn.o xattr.o proc.o ../misc.o miss.o + +all: qtfs + +qtfs: + make -C $(KBUILD) M=$(PWD) modules + +clean: + make -C $(KBUILD) M=$(PWD) clean + rm -rf ../*.o diff --git a/qtfs/qtfs/miss.c b/qtfs/qtfs/miss.c new file mode 100644 index 0000000..b79fb9b --- /dev/null +++ b/qtfs/qtfs/miss.c @@ -0,0 +1,66 @@ +#include +#include +#include +#include +#include +#include +#include "conn.h" +#include "qtfs-mod.h" +#include "req.h" + +#include "log.h" +#include "ops.h" + +static int miss_open(struct qtreq *miss) +{ + struct qtrsp_open *missrsp = (struct qtrsp_open *)miss->data; + struct qtreq_close *req; + struct qtrsp_close *rsp; + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + + if (missrsp->ret == QTFS_ERR) + return QTFS_OK; // no need to close + + if (pvar == NULL) { + qtfs_err("qtfs miss open pvar invalid."); + return QTFS_ERR; + } + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + req->fd = missrsp->fd; + qtfs_err("miss open proc fd:%d.", req->fd); + rsp = qtfs_remote_run(pvar, QTFS_REQ_CLOSE, sizeof(struct qtreq_close)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return QTFS_ERR; + } + + qtfs_conn_put_param(pvar); + return QTFS_OK; +} + +static struct qtmiss_ops qtfs_miss_handles[] = { + {QTFS_REQ_NULL, NULL, "null"}, + {QTFS_REQ_MOUNT, NULL, "mount"}, + {QTFS_REQ_OPEN, miss_open, "open"}, +}; + +int qtfs_missmsg_proc(struct qtfs_sock_var_s *pvar) +{ + struct qtreq *req = (struct qtreq *)pvar->vec_send.iov_base; + struct qtreq *rsp = (struct qtreq *)pvar->vec_recv.iov_base; + int ret; + qtfs_err("qtfs miss message proc req type:%u rsp type:%u.", req->type, rsp->type); + if (rsp->type > QTFS_REQ_OPEN) { + qtfs_err("qtfs miss message proc failed, type:%u invalid, req type:%u.", rsp->type, req->type); + return -EINVAL; + } + if (qtfs_miss_handles[rsp->type].misshandle == NULL) { + qtfs_info("qtfs miss message proc not support:%u, req type:%u.", rsp->type, req->type); + return -ESRCH; + } + ret = qtfs_miss_handles[rsp->type].misshandle(rsp); + if (ret != QTFS_OK) { + qtfs_err("qtfs miss message proc failed, req type:%u rsp type:%u.", req->type, rsp->type); + } + return ret; +} diff --git a/qtfs/qtfs/ops.h b/qtfs/qtfs/ops.h new file mode 100644 index 0000000..5cab367 --- /dev/null +++ b/qtfs/qtfs/ops.h @@ -0,0 +1,21 @@ +#ifndef __QTFS_OPS_H__ +#define __QTFS_OPS_H__ + +#include + +#include "qtfs-mod.h" + +extern struct inode_operations qtfs_proc_inode_ops; +extern struct file_operations qtfs_proc_file_ops; +extern struct inode_operations qtfs_proc_sym_ops; + +enum qtfs_type qtfs_get_type(char *str); +bool is_sb_proc(struct super_block *sb); + +struct inode *qtfs_iget(struct super_block *sb, struct inode_info *ii); +const char *qtfs_getlink(struct dentry *dentry, + struct inode *inode, struct delayed_call *done); +int qtfs_getattr(const struct path *, struct kstat *, u32, unsigned int); +struct dentry * qtfs_lookup(struct inode *, struct dentry *, unsigned int); + +#endif diff --git a/qtfs/qtfs/proc.c b/qtfs/qtfs/proc.c new file mode 100644 index 0000000..8db0148 --- /dev/null +++ b/qtfs/qtfs/proc.c @@ -0,0 +1,221 @@ +#include +#include + +#include "conn.h" +#include "qtfs-mod.h" +#include "req.h" +#include "log.h" +#include "ops.h" + +struct dentry *qtfs_proc_lookup(struct inode *parent_inode, struct dentry *child_dentry, unsigned int flags); +const char *qtfs_proc_getlink(struct dentry *dentry, struct inode *inode, struct delayed_call *done); +int qtfs_proc_getattr(const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags); + +enum qtfs_type qtfs_get_type(char *str) +{ + if (str && !strcmp(str, "proc")) + return QTFS_PROC; + return QTFS_NORMAL; +} + +bool is_sb_proc(struct super_block *sb) +{ + struct qtfs_fs_info *qfi = sb->s_fs_info; + + return qfi->type == QTFS_PROC; +} + +const char *match_str[] = { + "bash", + "isula", + "isulad", + "isulad-real", + "kubelet", + "kubelet-real", + "dockerd", + "dockerd-real", + "containerd", + "containerd-real" +}; + +int is_local_process(const char *path) +{ + int pid = -1; + char cmdline[TASK_COMM_LEN]; + char *pos = NULL; + struct task_struct *t = NULL; + int i = 0; + + sscanf(path, "/proc/%d", &pid); + if (pid <= 0) + return -1; + + t = qtfs_kern_syms.find_get_task_by_vpid((pid_t)pid); + if (!t) { + qtfs_info("[is_local_process] Failed to get task_struct from pid(%d)", pid); + return -1; + } + get_task_comm(cmdline, t); + + pos = strrchr(cmdline, '/'); + if (!pos) { + pos = cmdline; + } else { + pos++; + } + + for (i = 0; i < sizeof(match_str)/sizeof(char *); i++) { + if (!strncmp(pos, match_str[i], NAME_MAX)) { + qtfs_debug("[is_local_process] cmdline: %s is local process %d\n", cmdline, pid); + return pid; + } + } + + qtfs_debug("[is_local_process] cmdline: %s is not local process", cmdline); + return -1; +} + +struct inode_operations qtfs_proc_inode_ops = { + .lookup = qtfs_proc_lookup, + .getattr = qtfs_proc_getattr, +}; + +struct inode_operations qtfs_proc_sym_ops = { + .get_link = qtfs_proc_getlink, + .getattr = qtfs_proc_getattr, +}; + +struct dentry *qtfs_proc_lookup(struct inode *parent_inode, struct dentry *child_dentry, unsigned int flags) +{ + char cpath[NAME_MAX] = {0}; + char local_path[NAME_MAX] = {0}; + char tmp[NAME_MAX] = {0}; + struct path spath; + struct dentry *d = NULL; + struct inode_info ii; + struct inode *inode = NULL; + int ret = 0; + int pid = -1; + + if (qtfs_fullname(cpath, child_dentry) < 0) { + qtfs_err("%s: failed to get fullname", __func__); + goto remote; + } + + pid = is_local_process(cpath); + if (pid > 0) { + sscanf(cpath, "/proc/%s", tmp); + sprintf(local_path, "/test_proc/%s", tmp); + qtfs_debug("[%s]: get path:%s from local: %s\n", __func__, cpath, local_path); + ret = kern_path(local_path, 0, &spath); + if(ret) { + qtfs_err("[%s]: kern_path(%s) failed: %d\n", __func__, local_path, ret); + goto remote; + } + + ii.mode = spath.dentry->d_inode->i_mode; + ii.mode = (ii.mode & ~(S_IFMT)) | S_IFLNK; + ii.i_size = spath.dentry->d_inode->i_size; + ii.i_ino = spath.dentry->d_inode->i_ino; + ii.atime = spath.dentry->d_inode->i_atime; + ii.mtime = spath.dentry->d_inode->i_mtime; + ii.ctime = spath.dentry->d_inode->i_ctime; + path_put(&spath); + + inode = qtfs_iget(parent_inode->i_sb, &ii); + if (inode == NULL) { + qtfs_err("%s: failed to get inode for %s:%s", __func__, cpath, local_path); + return ERR_PTR(-ENOMEM); + } + d = d_splice_alias(inode, child_dentry); + return d; + } + +remote: + return qtfs_lookup(parent_inode, child_dentry, flags); +} + +const char *qtfs_proc_getlink(struct dentry *dentry, + struct inode *inode, struct delayed_call *done) +{ + char path[NAME_MAX] = {0}; + char tmp[NAME_MAX] = {0}; + char *link; + int pid = -1; + + link = kmalloc(MAX_PATH_LEN, GFP_KERNEL); + if (!link) { + qtfs_err("[%s]: failed to alloc memory", __func__); + goto remote; + } + memset(link, 0, MAX_PATH_LEN); + + if (qtfs_fullname(path, dentry) < 0) { + qtfs_info("[%s]: get path failed", __func__); + goto link_remote; + } + + if (!strncmp(path, "/proc/self", 11)) { + sprintf(link, "/test_proc/%d", (int)current->pid); + qtfs_info("[%s] success: %s getlink: %s", __func__, path, link); + return link; + } + + if (!strcmp(path, "/proc/mounts")) { + sprintf(link, "/proc/1/mounts"); + qtfs_info("[%s] success: %s getlink /proc/1/mounts", __func__, path); + return link; + } + + pid = is_local_process(path); + if (pid > 0) { + sscanf(path, "/proc/%s", tmp); + sprintf(link, "/test_proc/%s", tmp); + qtfs_info("[%s] success: %s getlink: %s", __func__, path, link); + return link; + } + +link_remote: + kfree(link); +remote: + return qtfs_getlink(dentry, inode, done); +} + +int qtfs_proc_getattr(const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags) +{ + char cpath[NAME_MAX] = {0}; + char tmp[NAME_MAX] = {0}; + char local_path[NAME_MAX] = {0}; + struct path spath; + int ret = 0; + int pid = -1; + + if (qtfs_fullname(cpath, path->dentry) < 0) { + qtfs_err("%s: failed to get fullname", __func__); + goto remote; + } + + pid = is_local_process(cpath); + if (pid > 0) { + sscanf(cpath, "/proc/%s", tmp); + sprintf(local_path, "/test_proc/%s", tmp); + ret = kern_path(local_path, 0, &spath); + if (ret) { + qtfs_err("[%s]: kern_path(%s) failed: %d", __func__, local_path, ret); + goto remote; + } + + ret = vfs_getattr(&spath, stat, req_mask, flags); + path_put(&spath); + if (ret) { + qtfs_err("[%s]: vfs_getattr %s failed: %d", __func__, local_path, ret); + goto remote; + } + qtfs_debug("[%s]: %s success", __func__, local_path); + stat->mode = (stat->mode & ~(S_IFMT)) | S_IFLNK; + return 0; + } + +remote: + return qtfs_getattr(path, stat, req_mask, flags); +} diff --git a/qtfs/qtfs/qtfs-mod.c b/qtfs/qtfs/qtfs-mod.c new file mode 100644 index 0000000..4f2a833 --- /dev/null +++ b/qtfs/qtfs/qtfs-mod.c @@ -0,0 +1,269 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include "conn.h" + +#include "qtfs-mod.h" +#include "syscall.h" + +static struct file_system_type qtfs_fs_type = { + .owner = THIS_MODULE, + .name = QTFS_FSTYPE_NAME, + .mount = qtfs_fs_mount, + .kill_sb = qtfs_kill_sb,//qtfs_kill_sb, +}; +MODULE_ALIAS_FS("qtfs"); + +/* + * 转发框架层： + * 1. 调用者先在pvar里预留框架头后，填好自己的私有发送数据。 + 2. 框架负责填充框架头，将整体消息发到对端。 + 3. 等待对端执行完回复。 + 4. 将接收buf的私有数据段首指针返回给调用者，完成文件操作层的通信。 + */ +void *qtfs_remote_run(struct qtfs_sock_var_s *pvar, unsigned int type, unsigned int len) +{ + int ret; + unsigned long retrytimes = 0; + struct qtreq *req = (struct qtreq *)pvar->vec_send.iov_base; + struct qtreq *rsp = (struct qtreq *)pvar->vec_recv.iov_base; + if (req == NULL || type >= QTFS_REQ_INV) { + qtfs_err("qtfs remote run failed, req is NULL type:%u.\n", type); + return NULL; + } + pvar->seq_num++; + req->type = type; + req->len = len; + req->seq_num = pvar->seq_num; + + // 调用qtfs_remote_run之前，调用者应该先把消息在iov_base里面封装好 + // 如果不是socket通信，则是在其他通信模式定义的buf里，消息协议统一 + // 都是struct qtreq *xx + // 给server发一个消息 + pvar->vec_send.iov_len = QTFS_MSG_LEN - (QTFS_REQ_MAX_LEN - len); + pvar->send_valid = pvar->vec_send.iov_len + 1; + ret = qtfs_conn_send(QTFS_CONN_SOCKET, pvar); + if (ret == -EPIPE) { + qtfs_err("qtfs remote run thread:%d send get EPIPE, try reconnect.", pvar->cur_threadidx); + qtfs_sm_reconnect(pvar); + } + if (ret <= 0) { + qtfs_err("qtfs remote run send failed, ret:%d pvar sendlen:%lu.", ret, pvar->vec_send.iov_len); + qtinfo_senderrinc(req->type); + } + qtinfo_sendinc(type); + + // wait for response +retry: + ret = qtfs_conn_recv_block(QTFS_CONN_SOCKET, pvar); + if (ret == -EAGAIN) + goto retry; + if (ret > 0 && req->seq_num != rsp->seq_num) { + qtinfo_cntinc(QTINF_SEQ_ERR); + qtfs_debug("qtfs remote run recv msg mismatch type:%d, ret:%d pvaridx:%d req:%lu rsp:%lu.", + req->type, ret, pvar->cur_threadidx, req->seq_num, rsp->seq_num); + qtinfo_recvinc(rsp->type); + if (pvar->miss_proc == 0) { + pvar->miss_proc = 1; + qtfs_missmsg_proc(pvar); + pvar->miss_proc = 0; + } + goto retry; + } + if (ret == -ERESTARTSYS || ret == -EINTR) { + if (retrytimes == 0) { + qtinfo_cntinc(QTINF_RESTART_SYS); + qtinfo_recverrinc(req->type); + } + retrytimes++; + msleep(1); + goto retry; + } + if (ret < 0) { + qtfs_err("qtfs remote run error, ret:%d.", ret); + qtinfo_recverrinc(req->type); + return NULL; + } + if (retrytimes > 0) + qtfs_debug("qtfs remote run retry times:%lu.", retrytimes); + pvar->recv_valid = ret + 1; + qtinfo_recvinc(rsp->type); + + if (rsp->err == QTFS_ERR) { + qtfs_err("qtfs remote run error, req errcode:%d type:%u len:%lu\n", req->err, req->type, req->len); + return NULL; + } + return qtfs_sock_msg_buf(pvar, QTFS_RECV); +} + +static int qtfs_epoll_thread(void *data) +{ + struct qtfs_sock_var_s *pvar = NULL; + struct qtreq_epollevt *req; + struct qtrsp_epollevt *rsp; + struct qtreq *head; + int ret; + struct inode *inode; + struct file *file; + int i; + +connecting: + while (qtfs_mod_exiting == false) { + pvar = qtfs_epoll_establish_conn(); + if (pvar != NULL) + break; + msleep(500); + } + if (pvar == NULL) { + do_exit(0); + } + qtfs_info("qtfs epoll thread establish a new connection."); + req = qtfs_sock_msg_buf(pvar, QTFS_RECV); + rsp = qtfs_sock_msg_buf(pvar, QTFS_SEND); + + // init ack head only once + do { + head = pvar->vec_send.iov_base; + pvar->vec_send.iov_len = QTFS_MSG_LEN - QTFS_REQ_MAX_LEN + sizeof(struct qtrsp_epollevt); + head->type = QTFS_REQ_EPOLL_EVENT; + head->len = sizeof(struct qtrsp_epollevt); + rsp->ret = QTFS_OK; + } while (0); + + while (!kthread_should_stop()) { + ret = qtfs_conn_recv(QTFS_CONN_SOCKET, pvar); + if (ret == -EPIPE || qtfs_sock_connected(pvar) == false) + goto connecting; + if (ret < 0 || req->event_nums <= 0) { + continue; + } + qtfs_debug("epoll thread recv %d events.", req->event_nums); + for (i = 0; i < req->event_nums; i++) { + // events[i].data is *file ptr + file = (struct file *)req->events[i].data; + if (IS_ERR(file) || file == NULL) { + qtfs_err("epoll thread event file invalid!"); + continue; + } + inode = file->f_inode; + // 暂时只支持fifo文件的epoll + if (inode == NULL || !S_ISFIFO(inode->i_mode)) { + qtfs_err("epoll thread event file:%lx not a fifo.", (unsigned long)file); + continue; + } + do { + struct qtfs_inode_priv *priv = inode->i_private; + __poll_t key; + if (req->events[i].events & EPOLLHUP) + key = EPOLLHUP; + else + key = EPOLLIN | EPOLLRDNORM; + if (priv == NULL) { + qtfs_err("epoll epoll wake up file:%lx error, inode priv is invalid.", (unsigned long)file); + WARN_ON(1); + } else { + wake_up_interruptible_sync_poll(&priv->readq, key); + } + } while (0); + } + ret = qtfs_conn_send(QTFS_CONN_SOCKET, pvar); + if (ret < 0) + qtfs_err("conn send failed, ret:%d\n", ret); + } + qtfs_epoll_cut_conn(pvar); + do_exit(0); +} + +struct file_operations qtfs_misc_fops = { + .owner=THIS_MODULE, + .unlocked_ioctl = qtfs_misc_ioctl, +}; + +struct kmem_cache *qtfs_inode_priv_cache; +struct task_struct *g_qtfs_epoll_thread = NULL; +static int __init qtfs_init(void) +{ + int ret; + qtfs_log_init(qtfs_log_level); + + ret = register_filesystem(&qtfs_fs_type); + if (ret != 0) { + qtfs_err("QTFS file system register failed, ret:%d.\n", ret); + return -1; + } + qtfs_inode_priv_cache = kmem_cache_create("qtfs_inode_priv", + sizeof(struct qtfs_inode_priv), + 0, + (SLAB_RECLAIM_ACCOUNT| SLAB_MEM_SPREAD), + NULL); + if (!qtfs_inode_priv_cache) { + qtfs_err("qtfs inode priv cache create failed.\n"); + return -ENOMEM; + } + g_qtfs_epoll_thread = kthread_run(qtfs_epoll_thread, NULL, "qtfs_epoll"); + if (IS_ERR(g_qtfs_epoll_thread)) { + qtfs_err("qtfs epoll thread run failed.\n"); + } + qtfs_diag_info = (struct qtinfo *)kmalloc(sizeof(struct qtinfo), GFP_KERNEL); + if (qtfs_diag_info == NULL) { + qtfs_err("kmalloc qtfs diag info failed."); + } else { + memset(qtfs_diag_info, 0, sizeof(struct qtinfo)); + } + + qtfs_misc_register(); + qtfs_kallsyms_hack_init(); + qtfs_conn_param_init(); + qtfs_syscall_init(); + + qtfs_info("QTFS file system register success!\n"); + return 0; +} + +static void __exit qtfs_exit(void) +{ + int ret; + qtfs_mod_exiting = true; + + if (g_qtfs_epoll_thread) { + kthread_stop(g_qtfs_epoll_thread); + } + + qtfs_conn_param_fini(); + qtfs_misc_destroy(); + if (qtfs_epoll_var != NULL) { + qtfs_epoll_cut_conn(qtfs_epoll_var); + if (qtfs_epoll_var->sock != NULL) { + sock_release(qtfs_epoll_var->sock); + qtfs_epoll_var->sock = NULL; + } + qtfs_sock_var_fini(qtfs_epoll_var); + kfree(qtfs_epoll_var); + qtfs_epoll_var = NULL; + } + + kfree(qtfs_diag_info); + qtfs_diag_info = NULL; + qtfs_syscall_fini(); + + ret = unregister_filesystem(&qtfs_fs_type); + if (ret != 0) { + qtfs_err("QTFS file system unregister failed, ret:%d.\n", ret); + } + + kmem_cache_destroy(qtfs_inode_priv_cache); + qtfs_info("QTFS file system unregister success!\n"); + return; +} + +module_param_string(qtfs_server_ip, qtfs_server_ip, sizeof(qtfs_server_ip), 0600); +MODULE_PARM_DESC(qtfs_server_ip, "qtfs server ip"); +module_param(qtfs_server_port, int, 0644); +module_param(qtfs_sock_max_conn, int, 0644); +module_param_string(qtfs_log_level, qtfs_log_level, sizeof(qtfs_log_level), 0600); + +module_init(qtfs_init); +module_exit(qtfs_exit); +MODULE_AUTHOR("liqiang64@huawei.com"); +MODULE_LICENSE("GPL"); + + diff --git a/qtfs/qtfs/qtfs-mod.h b/qtfs/qtfs/qtfs-mod.h new file mode 100644 index 0000000..5a30868 --- /dev/null +++ b/qtfs/qtfs/qtfs-mod.h @@ -0,0 +1,172 @@ +#ifndef __QTFS_INCLUDE_H__ +#define __QTFS_INCLUDE_H__ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "comm.h" +#include "log.h" +#include "req.h" + + +#define QTFS_MAXLEN 8 +#define QTFS_MAX_FILES 32 +#define QTFS_MAX_BLOCKSIZE 512 + +#define QTFS_FSTYPE_NAME "qtfs" + +extern struct kmem_cache *qtfs_inode_priv_cache; + +struct private_data { + int fd; + unsigned long long file; +}; + +struct qtfs_inode_priv { + unsigned int files; + wait_queue_head_t readq; + wait_queue_head_t writeq; +}; + +enum { + QTFS_ROOT_INO = 1, + QTFS_IPC_INIT_INO = 0xEFFFFFFFU, + QTFS_UTS_INIT_INO = 0xEFFFFFFEU, + QTFS_USER_INIT_INO = 0xEFFFFFFDU, + QTFS_PID_INIT_INO = 0xEFFFFFFCU, + QTFS_CGROUP_INIT_INO = 0xEFFFFFFBU, + QTFS_TIME_INIT_INO = 0xEFFFFFFAU, + QTFS_IMA_INIT_INO = 0xEFFFFFF9U, +}; + +struct qtfs_inode { + mode_t mode; + uint64_t i_no; + uint64_t d_no; + char *peer_path; + union { + uint64_t file_size; + uint64_t dir_childrens; + }; + struct list_head entry; +}; + +struct qtfs_fs_info { + char peer_path[NAME_MAX]; + char *mnt_path; + + enum qtfs_type type; +}; + +struct qtfs_dir_entry { + struct list_head node; + char filename[NAME_MAX]; + struct qtfs_inode *priv; +}; + +struct qtfs_file_blk { + uint8_t busy; + mode_t mode; + uint8_t idx; + + union { + uint8_t file_size; + uint8_t dir_children; + }; + char data[0]; +}; + +struct qtmiss_ops { + int type; + // return int is output len. + int (*misshandle) (struct qtreq *); + char str[32]; +}; + +static inline int qtfs_fullname(char *fullname, struct dentry *d) +{ + struct qtfs_fs_info *fsinfo = NULL; + int len = 0; + char *name = NULL; + char *ret = NULL; + + if (!d) { + qtfs_info("%s: get dentry fullname NULL\n", __func__); + return -1; + } + name = __getname(); + ret = dentry_path_raw(d, name, MAX_PATH_LEN); + if (err_ptr(ret)) { + qtfs_err("qtfs fullname failed:%ld\n", PTR_ERR(ret)); + __putname(name); + return -1; + } + + if (d && d->d_sb && d->d_sb->s_fs_info) { + fsinfo = d->d_sb->s_fs_info; + } else { + qtfs_err("%s: failed to get private fs_info\n", __func__); + __putname(name); + return -1; + } + if (strcmp(fsinfo->peer_path, "/")) { + /* if peer_path is not root '/' */ + len = strlcpy(fullname, fsinfo->peer_path, MAX_PATH_LEN); + } + if (len + strlen(ret) >= MAX_PATH_LEN - 1) { + qtfs_err("qtfs fullname may reach max len:%d reallen:%ld path:%s", len, strlen(fullname), fullname); + __putname(name); + return -1; + } + len += strlcpy(&fullname[len], ret, MAX_PATH_LEN - len); + if (strcmp(fullname, "/")) { + if (fullname[strlen(fullname) - 1] == '/') + fullname[strlen(fullname) - 1] = '\0'; + } + //qtfs_info("fullname:%s, len:%d\n", fullname, len); + __putname(name); + return len; +} + +#define QTFS_FULLNAME(fullname, d) \ + if (qtfs_fullname(fullname, d)<0) { \ + qtfs_err("qtfs fullname failed\n"); \ + qtfs_conn_put_param(pvar); \ + return -EINVAL; \ + } + +extern const struct xattr_handler qtfs_xattr_user_handler; +extern const struct xattr_handler qtfs_xattr_trusted_handler; +extern const struct xattr_handler qtfs_xattr_security_handler; +extern const struct xattr_handler qtfs_xattr_hurd_handler; +extern struct qtinfo *qtfs_diag_info; +extern int qtfs_mod_exiting; + +void qtfs_kill_sb(struct super_block *sb); +struct dentry *qtfs_fs_mount(struct file_system_type *fs_type, + int flags, const char *dev_name, + void *data); +void *qtfs_remote_run(struct qtfs_sock_var_s *pvar, unsigned int type, unsigned int len); +int qtfs_misc_register(void); +void qtfs_misc_destroy(void); +long qtfs_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +int qtfs_missmsg_proc(struct qtfs_sock_var_s *pvar); + +#endif + diff --git a/qtfs/qtfs/sb.c b/qtfs/qtfs/sb.c new file mode 100644 index 0000000..7303d17 --- /dev/null +++ b/qtfs/qtfs/sb.c @@ -0,0 +1,1376 @@ +#include +#include +#include +#include +#include +#include + +#include "conn.h" +#include "qtfs-mod.h" +#include "req.h" +#include "log.h" +#include "ops.h" + +#define CURRENT_TIME(inode) (current_time(inode)) +static struct inode_operations qtfs_inode_ops; +static struct inode_operations qtfs_symlink_inode_ops; +struct inode *qtfs_iget(struct super_block *sb, struct inode_info *ii); + +int qtfs_statfs(struct dentry *dentry, struct kstatfs *buf) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_statfs *req; + struct qtrsp_statfs *rsp; + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + rsp = qtfs_sock_msg_buf(pvar, QTFS_RECV); + + QTFS_FULLNAME(req->path, dentry); + rsp = qtfs_remote_run(pvar, QTFS_REQ_STATFS, QTFS_SEND_SIZE(struct qtreq_statfs, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return -EINVAL; + } + if (rsp->ret == QTFS_ERR) { + int ret = rsp->errno; + qtfs_err("qtfs statfs failed. %d", rsp->errno); + qtfs_conn_put_param(pvar); + return ret; + } + qtfs_info("%s: get path %s\n", __func__, req->path); + memcpy(buf, &(rsp->kstat), sizeof(struct kstatfs)); + qtfs_conn_put_param(pvar); + return 0; +} + +static const struct super_operations qtfs_ops = { + .statfs = qtfs_statfs, +}; + +static inline struct qtfs_fs_info *qtfs_priv_byinode(struct inode *inode) +{ + struct super_block *sb = inode->i_sb; + return sb->s_fs_info; +} + +static inline char *qtfs_mountpoint_path_init(struct dentry *dentry, struct path *path, char *mnt_file) +{ + char *name = NULL; + char *ret; + char *mnt_point; + int len; + struct qtfs_fs_info *fsinfo = qtfs_priv_byinode(d_inode(dentry)); + if (fsinfo && fsinfo->mnt_path) { + return fsinfo->mnt_path; + } + name = __getname(); + path_get(path); + ret = qtfs_kern_syms.d_absolute_path(path, name, MAX_PATH_LEN); + qtfs_debug("mntfile:%s absolute:%s", mnt_file, ret); + if (err_ptr(ret)) { + qtfs_err("d_absolute_path failed:%ld", PTR_ERR(ret)); + } else { + if (strcmp(mnt_file, "/")) { + mnt_point = strstr(ret, mnt_file); + qtfs_info("mnt point:%s", mnt_point); + if (mnt_point) { + *mnt_point = '\0'; + } else { + qtfs_err("Failed to get mount root path"); + } + } + len = strlen(ret); + fsinfo->mnt_path = (char *)kmalloc(len + 1, GFP_KERNEL); + if (fsinfo->mnt_path) { + strlcpy(fsinfo->mnt_path, ret, len + 1); + } + qtfs_debug("d_absolute_path get mnt path:%s", fsinfo->mnt_path); + } + path_put(path); + __putname(name); + return fsinfo->mnt_path; +} + +int qtfs_readdir(struct file *filp, struct dir_context *ctx) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_readdir *req; + struct qtrsp_readdir *rsp; + struct qtfs_dirent64 *dirent = NULL; + int idx; + int ret; + int namelen; + int dircnt; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + + if (ctx->pos == -1) { + qtfs_conn_put_param(pvar); + return -ENOENT; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + rsp = qtfs_sock_msg_buf(pvar, QTFS_RECV); + QTFS_FULLNAME(req->path, filp->f_path.dentry); + req->count = sizeof(rsp->dirent); + req->pos = ctx->pos; + + rsp = qtfs_remote_run(pvar, QTFS_REQ_READDIR, QTFS_SEND_SIZE(struct qtreq_readdir, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->d.ret == QTFS_ERR) { + qtfs_err("qtfs readdir failed."); + qtfs_conn_put_param(pvar); + return -EFAULT; + } + + idx = 0; + dircnt = rsp->d.vldcnt; + while (dircnt--) { + dirent = (struct qtfs_dirent64 *)&rsp->dirent[idx]; + namelen = strlen(dirent->d_name); + ret = dir_emit(ctx, dirent->d_name, namelen, + dirent->d_ino, dirent->d_type); + idx += dirent->d_reclen; + qtfs_debug("qtfs readdir direntoff:0x%lx name:<%s>, ret:%d, reclen:%u namelen:%d, ino:%llu type:%d", + (void *)dirent - (void *)rsp->dirent, dirent->d_name, ret, dirent->d_reclen, namelen, dirent->d_ino, dirent->d_type); + } + + ctx->pos = (rsp->d.over) ? -1 : rsp->d.pos; + qtfs_info("qtfs readdir<%s> success ret:%d vldcnt:%d over:%d pos:%lld.", + req->path, rsp->d.ret, rsp->d.vldcnt, rsp->d.over, ctx->pos); + qtfs_conn_put_param(pvar); + return 0; +} + +int qtfs_open(struct inode *inode, struct file *file) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_open *req; + struct qtrsp_open *rsp; + struct private_data *data = NULL; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + data = (struct private_data *)kmalloc(sizeof(struct private_data), GFP_KERNEL); + if (err_ptr(data)) { + qtfs_err("qtfs_open alloc private_data failed: %ld", PTR_ERR(data)); + qtfs_conn_put_param(pvar); + return -ENOMEM; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, file->f_path.dentry); + + req->flags = file->f_flags; + req->mode = file->f_mode; + rsp = qtfs_remote_run(pvar, QTFS_REQ_OPEN, QTFS_SEND_SIZE(struct qtreq_open, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + qtfs_err("qtfs open:%s failed, f_mode:%o flag:%x", req->path, file->f_mode, file->f_flags); + return -EINVAL; + } + + if (rsp->ret == QTFS_ERR) { + int err = rsp->fd; + if (rsp->fd != -ENOENT) { + qtfs_err("qtfs_open failed with %d ret:%d", rsp->fd, rsp->ret); + } else { + qtfs_info("qtfs_open file %s failed, not exist.", req->path); + } + qtfs_conn_put_param(pvar); + return err; + } + qtfs_info("qtfs open:%s success, f_mode:%o flag:%x, fd:%d", req->path, file->f_mode, file->f_flags, rsp->fd); + data->file = rsp->file; + data->fd = rsp->fd; + WARN_ON(file->private_data); + file->private_data = data; + qtfs_conn_put_param(pvar); + + return 0; +} + +int qtfs_dir_open(struct inode *inode, struct file *file) +{ + qtfs_info("qtfs dir open enter: %s.", file->f_path.dentry->d_iname); + return 0; +} + +int qtfs_dir_release(struct inode *inode, struct file *file) +{ + qtfs_info("qtfs dir release enter: %s.", file->f_path.dentry->d_iname); + return 0; +} + +int qtfs_release(struct inode *inode, struct file *file) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_close *req; + struct qtrsp_close *rsp; + struct private_data *private = NULL; + int ret; + + if (pvar == NULL) { + qtfs_err("qtfs release pvar invalid."); + return -EFAULT; + } + + if (err_ptr(file)) { + qtfs_err("qtfs release: invalid file: 0x%llx", (__u64)file); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + private = (struct private_data *)file->private_data; + + if (err_ptr(private)) { + qtfs_err("qtfs_close(%s): invalid private_data pointer:%ld", file->f_path.dentry->d_iname, PTR_ERR(private)); + WARN_ON(1); + qtfs_conn_put_param(pvar); + return -EFAULT; + } + req->fd = private->fd; + rsp = qtfs_remote_run(pvar, QTFS_REQ_CLOSE, sizeof(struct qtreq_close)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_err("qtfs release fd:%d failed, rsp is invalid.", req->fd); + ret = PTR_ERR(rsp); + goto end; + } + qtfs_info("qtfs release success fd:%d ret:%d %s", req->fd, rsp->ret, (rsp->ret == QTFS_ERR) ? "failed" : "success"); + ret = rsp->ret; +end: + qtfs_conn_put_param(pvar); + kfree(file->private_data); + file->private_data = NULL; + return ret; +} + +ssize_t qtfs_readiter(struct kiocb *kio, struct iov_iter *iov) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_readiter *req; + struct qtrsp_readiter *rsp; + int reqlen; + size_t leftlen = iov_iter_count(iov); + size_t allcnt = leftlen; + size_t tocnt = 0; + ssize_t ret; + struct private_data *private = NULL; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + + private = (struct private_data *)kio->ki_filp->private_data; + if (err_ptr(private)) { + qtfs_err("qtfs_readiter(%s): invalid private_data pointer:%ld", kio->ki_filp->f_path.dentry->d_iname, PTR_ERR(private)); + qtfs_conn_put_param(pvar); + return -ENOMEM; + } + + req->file = private->file; + if (req->file <= 0) { + qtfs_err("qtfs_readiter: invalid file(0x%llx)", req->file); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + reqlen = sizeof(struct qtreq_readiter); + + do { + req->len = leftlen; + req->pos = kio->ki_pos; + rsp = qtfs_remote_run(pvar, QTFS_REQ_READITER, reqlen); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->d.ret == QTFS_ERR || rsp->d.len <= 0) { + qtfs_info("qtfs readiter error."); + ret = (ssize_t)rsp->d.len; + qtfs_conn_put_param(pvar); + return ret; + } + tocnt = copy_to_iter(rsp->readbuf, rsp->d.len, iov); + if (rsp->d.len != tocnt) { + qtfs_err("copy to iter failed, errno:%ld", tocnt); + qtfs_conn_put_param(pvar); + return allcnt - leftlen + tocnt; + } + + leftlen -= rsp->d.len; + kio->ki_pos += rsp->d.len; + } while (leftlen > 0 && rsp->d.len >= sizeof(rsp->readbuf) - 1); + qtfs_info("qtfs readiter over, leftlen:%lu, reqlen:%lu, fullname:<%s>, ino:%lu, pos:%lld, iovcnt:%lu(%lu)\n", leftlen, + req->len, kio->ki_filp->f_path.dentry->d_iname, kio->ki_filp->f_inode->i_ino, kio->ki_pos, iov_iter_count(iov), iov->iov->iov_len); + + qtfs_conn_put_param(pvar); + return allcnt - leftlen; +} + +ssize_t qtfs_writeiter(struct kiocb *kio, struct iov_iter *iov) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_write *req; + struct qtrsp_write *rsp; + char *wrbuf = NULL; + int wrbuflen; + int maxbuflen; + size_t len = iov_iter_count(iov); + size_t leftlen = len; + struct private_data *private = NULL; + ssize_t ret; + struct file *filp; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var."); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + filp = kio->ki_filp; + private = (struct private_data *)filp->private_data; + if (err_ptr(private)) { + qtfs_err("qtfs_write(%s): invalid private_data pointer:%ld", filp->f_path.dentry->d_iname, PTR_ERR(private)); + qtfs_conn_put_param(pvar); + return -ENOMEM; + } + + req->d.file = private->file; + if (req->d.file < 0) { + qtfs_err("qtfs_write: invalid file(0x%llx)", req->d.file); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + req->d.mode = filp->f_mode; + req->d.flags = filp->f_flags; + + wrbuf = req->path_buf; + maxbuflen = sizeof(req->path_buf); + do { + wrbuflen = (leftlen >= maxbuflen) ? (maxbuflen - 1) : leftlen; + req->d.buflen = wrbuflen; + req->d.pos = kio->ki_pos; + if (copy_from_iter(wrbuf, wrbuflen, iov) == 0) { + qtfs_err("qtfs write copy from iter failed, len:%d.", wrbuflen); + qtfs_conn_put_param(pvar); + return -EFAULT; + } + rsp = qtfs_remote_run(pvar, QTFS_REQ_WRITE, sizeof(struct qtreq_write) - sizeof(req->path_buf) + wrbuflen); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR || rsp->len <= 0) { + qtfs_err("qtfs write remote error, errno:%ld, leftlen:%lu.", rsp->len, leftlen); + ret = rsp->len; + qtfs_conn_put_param(pvar); + return ret; + } + if (rsp->len != wrbuflen) { + WARN_ON(1); + } + kio->ki_pos += rsp->len; + leftlen -= wrbuflen; + } while (leftlen); + + if (S_ISFIFO(kio->ki_filp->f_inode->i_mode)) { + struct inode *inode = kio->ki_filp->f_inode; + struct qtfs_inode_priv *priv = inode->i_private; + wake_up_interruptible_sync_poll(&priv->readq, EPOLLIN | EPOLLRDNORM); + qtfs_info("qtfs write iter fifo %s sync poll.", filp->f_path.dentry->d_iname); + } + qtfs_info("qtfs write %s over, leftlen:%lu.", filp->f_path.dentry->d_iname, leftlen); + qtfs_conn_put_param(pvar); + return len - leftlen; +} + +loff_t qtfs_llseek(struct file *file, loff_t off, int whence) +{ + qtfs_info("qtfs llseek off:%lld, whence:%d.", off, whence); + return 0; +} + +static void qtfs_vma_close(struct vm_area_struct *vma) +{ + qtfs_info("qtfs vma close enter."); + filemap_write_and_wait(vma->vm_file->f_mapping); +} + +static vm_fault_t qtfs_vm_fault(struct vm_fault *vmf) +{ + vm_fault_t ret = filemap_fault(vmf); + + qtfs_info("qtfs vm ops fault enter, filemap fault:0x%x, pgoff:%lu.", ret, vmf->pgoff); + return ret; +} + +static void qtfs_map_pages(struct vm_fault *vmf, + pgoff_t start_pgoff, pgoff_t end_pgoff) +{ + qtfs_info("qtfs map pages enter, pgoff:%lu start:%lu end:%lu.", vmf->pgoff, start_pgoff, end_pgoff); + + filemap_map_pages(vmf, start_pgoff, end_pgoff); + return; +} + +static vm_fault_t qtfs_page_mkwrite(struct vm_fault *vmf) +{ + qtfs_info("qtfs page mkwrite enter."); + return filemap_page_mkwrite(vmf); +} + +static const struct vm_operations_struct qtfs_file_vm_ops = { + .fault = qtfs_vm_fault, + .map_pages = qtfs_map_pages, + .close = qtfs_vma_close, + .page_mkwrite = qtfs_page_mkwrite, +}; + +int qtfs_mmap(struct file *file, struct vm_area_struct *vma) +{ + qtfs_info("qtfs mmap enter."); + + if (IS_DAX(file_inode(file))) { + qtfs_info("qtfs mmap is dax mmap."); + } + file_accessed(file); + vma->vm_ops = &qtfs_file_vm_ops; + return 0; +} + +int qtfs_fsync(struct file *file, loff_t start, loff_t end, int datasync) +{ + qtfs_info("qtfs fsync enter."); + return 0; +} + +long qtfs_do_ioctl(struct file *filp, unsigned int cmd, unsigned long arg, unsigned int size) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_ioctl *req; + struct qtrsp_ioctl *rsp; + unsigned int len = 0; + int ret = -EINVAL; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return -EINVAL; + } + + WARN_ON(size >= MAX_PATH_LEN); + if (size >= MAX_PATH_LEN) { + qtfs_conn_put_param(pvar); + return -ENOMEM; + } + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + rsp = qtfs_sock_msg_buf(pvar, QTFS_RECV); + + QTFS_FULLNAME(req->path, filp->f_path.dentry); + + req->d.cmd = cmd; + if (size > 0) { + req->d.offset = ALIGN(strlen(req->path) + 1, 64); + ret = copy_from_user(req->path + req->d.offset, (char __user *)arg, size); + if (ret) { + qtfs_err("%s: copy_from_user %p, 0x%lx, %u failed.", __func__, req->path + req->d.offset, arg, size); + goto out; + } + len = sizeof(struct qtreq_ioctl) - sizeof(req->path) + req->d.offset + size; + } else { + len = sizeof(struct qtreq_ioctl) - sizeof(req->path) + strlen(req->path) + 1; + } + + rsp = qtfs_remote_run(pvar, QTFS_REQ_IOCTL, len); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs ioctl failed. %d", rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + + qtfs_info("qtfs do ioctl success, path: %s", req->path); + ret = rsp->errno; + if (rsp->size > 0) + ret = copy_to_user((char __user *)arg, rsp->buf, rsp->size); +out: + qtfs_conn_put_param(pvar); + return (long)ret; +} + +long qtfs_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + switch(cmd) { + case FS_IOC_FSGETXATTR: + return qtfs_do_ioctl(filp, cmd, arg, 0); + case FS_IOC_FSSETXATTR: + return qtfs_do_ioctl(filp, cmd, arg, sizeof(struct fsxattr)); + default: + return -EOPNOTSUPP; + } +} + +static struct file_operations qtfs_file_ops = { + .read_iter = qtfs_readiter, + .write_iter = qtfs_writeiter, + .open = qtfs_open, + .release = qtfs_release, + .mmap = qtfs_mmap, + .llseek = qtfs_llseek, + .fsync = qtfs_fsync, + .unlocked_ioctl = qtfs_ioctl, +}; + +loff_t qtfs_dir_file_llseek(struct file *file, loff_t offset, int whence) +{ + qtfs_info("qtfs generic file llseek: %s.", file->f_path.dentry->d_iname); + return generic_file_llseek(file, offset, whence); +} + +ssize_t qtfs_dir_read_dir(struct file *filp, char __user *buf, size_t siz, loff_t *ppos) +{ + qtfs_err("qtfs generic read dir: %s.", filp->f_path.dentry->d_iname); + return generic_read_dir(filp, buf, siz, ppos); +} + +static struct file_operations qtfs_dir_ops = { + .owner = THIS_MODULE, + .iterate_shared = qtfs_readdir, + .unlocked_ioctl = qtfs_ioctl, + .open = qtfs_dir_open, + .release = qtfs_dir_release, + .llseek = qtfs_dir_file_llseek, + .read = qtfs_dir_read_dir, +}; + +static __poll_t +qtfsfifo_poll(struct file *filp, poll_table *wait) +{ + struct qtfs_inode_priv *priv = filp->f_inode->i_private; + __poll_t mask = 0; + struct list_head *p; + struct qtfs_sock_var_s *pvar; + struct qtreq_poll *req; + struct qtrsp_poll *rsp; + struct private_data *fpriv = (struct private_data *)filp->private_data; + + poll_wait(filp, &priv->readq, wait); + + p = &priv->readq.head; + qtfs_debug("fifo poll readq,qproc:%lx key:%x head:%lx next:%lx pre:%lx", + (unsigned long)wait->_qproc, (unsigned)wait->_key, (unsigned long)p, (unsigned long)p->next, (unsigned long)p->prev); + + if (IS_ERR((void *)fpriv->file) || (void *)fpriv->file == NULL) { + qtfs_err("fifo poll priv file invalid."); + return 0; + } + pvar = qtfs_conn_get_param(); + if (pvar == NULL) { + qtfs_err("qtfs fifo poll get param failed."); + return 0; + } + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + req->file = fpriv->file; + rsp = qtfs_remote_run(pvar, QTFS_REQ_FIFOPOLL, sizeof(struct qtreq_poll)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return 0; + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs fifo poll remote run error."); + qtfs_conn_put_param(pvar); + return 0; + } + mask = rsp->mask; + qtfs_info("fifo poll success, mask:%x.", mask); + qtfs_conn_put_param(pvar); + return mask; +} + +struct file_operations qtfsfifo_ops = { + .read_iter = qtfs_readiter, + .write_iter = qtfs_writeiter, + .open = qtfs_open, + .release = qtfs_release, + .llseek = no_llseek, + .poll = qtfsfifo_poll, +}; + +static int qtfs_readpage(struct file *file, struct page *page) +{ + void *kaddr = NULL; + loff_t offset = page->index << PAGE_SHIFT; + qtfs_info("qtfs readpage enter, page pos:%lld.", offset); + + kaddr = kmap_atomic(page); + kernel_read(file, kaddr, PAGE_SIZE, &offset); + flush_dcache_page(page); + kunmap_atomic(kaddr); + SetPageUptodate(page); + unlock_page(page); + + return 0; +} + +static struct page **qtfs_alloc_pages(unsigned int nr) +{ + struct page **pages = kzalloc(nr * (sizeof(struct page *)), GFP_KERNEL); + if (pages == NULL) { + qtfs_err("qtfs alloc pages failed."); + return NULL; + } + return pages; +} + +static void qtfs_free_pages(struct page **pages) +{ + kfree(pages); +} + +static void qtfs_readahead(struct readahead_control *rac) +{ + int i; + unsigned int nr_pages = readahead_count(rac); + struct page **pages = qtfs_alloc_pages(nr_pages); + qtfs_info("qtfs readahead."); + + nr_pages = __readahead_batch(rac, pages, nr_pages); + + for (i = 0; i < nr_pages; i++) { + qtfs_readpage(rac->file, pages[i]); + } + qtfs_free_pages(pages); + return; +} + +static int qtfs_writepage(struct page *page, struct writeback_control *wbc) +{ + qtfs_info("qtfs write page."); + return 0; +} + +static int qtfs_writepages(struct address_space *mapping, + struct writeback_control *wbc) +{ + qtfs_info("qtfs write pages."); + return 0; +} + +static int qtfs_setpagedirty(struct page *page) +{ + qtfs_info("qtfs set page dirty."); + __set_page_dirty_nobuffers(page); + return 0; +} + +static const struct address_space_operations qtfs_aops = { + .readpage = qtfs_readpage, + .readahead = qtfs_readahead, + .writepage = qtfs_writepage, + .writepages = qtfs_writepages, + .set_page_dirty = qtfs_setpagedirty, +}; + +int qtfs_new_entry(struct inode *inode, struct dentry *dentry) +{ + struct dentry *d = NULL; + + if (!inode) + return -ENOMEM; + + d_drop(dentry); + d = d_splice_alias(inode, dentry); + if (IS_ERR(d)) { + return PTR_ERR(d); + } + if (d) + dput(d); + return 0; +} + +int qtfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_mkdir *req = NULL; + struct qtrsp_mkdir *rsp = NULL; + int ret; + struct inode *inode; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var."); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry); + + req->mode = mode; + rsp = qtfs_remote_run(pvar, QTFS_REQ_MKDIR, QTFS_SEND_SIZE(struct qtreq_mkdir, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs mkdir failed %d.", rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + inode = qtfs_iget(dentry->d_sb, &(rsp->inode_info)); + ret = qtfs_new_entry(inode, dentry); + qtfs_info("mkdir path:%s success.", req->path); + qtfs_conn_put_param(pvar); + return ret; +} + +int qtfs_create(struct inode *dir, struct dentry *dentry, umode_t mode, bool excl) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_icreate *req; + struct qtrsp_icreate *rsp; + struct inode *inode; + int ret = 0; + int ret2 = 0; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var."); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry); + + req->mode = mode; + req->excl = excl; + rsp = qtfs_remote_run(pvar, QTFS_REQ_ICREATE, QTFS_SEND_SIZE(struct qtreq_icreate, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + + if (rsp->ret == QTFS_ERR) { + ret = rsp->errno; + qtfs_err("qtfs icreate failed %d.", rsp->errno); + qtfs_conn_put_param(pvar); + return ret; + } + ret = rsp->errno; + inode = qtfs_iget(dentry->d_sb, &(rsp->inode_info)); + ret2 = qtfs_new_entry(inode, dentry); + + qtfs_info("qtfs icreate get ret:%d, mode:%ho.", rsp->errno, rsp->inode_info.mode); + qtfs_conn_put_param(pvar); + return ret ? ret : ret2; +} + +int qtfs_mknod(struct inode *dir, struct dentry *dentry, umode_t mode, dev_t dev) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_mknod *req; + struct qtrsp_mknod *rsp; + struct inode *inode; + int ret = 0; + int ret2 = 0; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry); + + if (S_ISSOCK(mode)) { + qtfs_conn_put_param(pvar); + return -ENOTSUPP; + } + + req->mode = mode; + req->dev = dev; + rsp = qtfs_remote_run(pvar, QTFS_REQ_MKNOD, sizeof(struct qtreq_mknod) - sizeof(req->path) + strlen(req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs mknod failed %d.", rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + ret = rsp->errno; + qtfs_info("qtfs mknod success, path:<%s>.\n", req->path); + inode = qtfs_iget(dentry->d_sb, &(rsp->inode_info)); + ret2 = qtfs_new_entry(inode, dentry); + qtfs_conn_put_param(pvar); + return ret ? ret : ret2; +} + +static void qtfs_inode_priv_alloc(struct inode *inode) +{ + struct qtfs_inode_priv *priv = kmem_cache_alloc(qtfs_inode_priv_cache, GFP_KERNEL); + if (priv == NULL) { + qtfs_err("qtfs inode priv alloc kmem cache alloc failed."); + return; + } + inode->i_private = priv; + priv->files = 0; + init_waitqueue_head(&priv->readq); + init_waitqueue_head(&priv->writeq); + return; +} + +static void qtfs_init_inode(struct super_block *sb, struct inode *inode, struct inode_info *ii) +{ + inode->i_sb = sb; + inode->i_mode = ii->mode; + inode->i_ino = ii->i_ino; + inode->i_size = ii->i_size; + inode->i_atime = ii->atime; + inode->i_mtime = ii->mtime; + inode->i_ctime = ii->ctime; + + if (S_ISLNK(inode->i_mode)) { + if (is_sb_proc(sb)) { + qtfs_info("inode link ops set to qtfs_proc_sym_ops."); + inode->i_op = &qtfs_proc_sym_ops; + } else { + inode->i_op = &qtfs_symlink_inode_ops; + } + } else { + if (is_sb_proc(sb)) { + inode->i_op = &qtfs_proc_inode_ops; + } else { + inode->i_op = &qtfs_inode_ops; + } + } + inode->i_mapping->a_ops = &qtfs_aops; + + if (S_ISDIR(ii->mode)) { + inode->i_fop = &qtfs_dir_ops; + } else if (S_ISREG(ii->mode)) { + inode->i_fop = &qtfs_file_ops; + } else if (S_ISFIFO(ii->mode)) { + inode->i_fop = &qtfsfifo_ops; + } + qtfs_inode_priv_alloc(inode); + return; +} + +struct inode *qtfs_iget(struct super_block *sb, struct inode_info *ii) +{ + struct inode *inode; + + inode = new_inode(sb); + if (!inode) + return NULL; + qtfs_init_inode(sb, inode, ii); + return inode; +} + +struct dentry *qtfs_lookup(struct inode *parent_inode, struct dentry *child_dentry, unsigned int flags) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_lookup *req; + struct qtrsp_lookup *rsp; + struct inode *inode; + struct dentry *d = NULL; + int ret; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return NULL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + ret = qtfs_fullname(req->fullname, child_dentry); + if (ret < 0) { + qtfs_err("qtfs lookup get fullname failed, too many path layers, <%s>!", req->fullname); + goto err_end; + } + rsp = qtfs_remote_run(pvar, QTFS_REQ_LOOKUP, strlen(req->fullname)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return (void *)rsp; + } + if (rsp->ret != QTFS_OK) { + qtfs_info("qtfs fs lookup failed, path:<%s> not exist at peer.\n", req->fullname); + goto err_end; + } + inode = qtfs_iget(parent_inode->i_sb, &(rsp->inode_info)); + if (inode == NULL) + goto err_end; + d = d_splice_alias(inode, child_dentry); + qtfs_debug("qtfs lookup fullname:%s mode:%o(rsp:%o), ino:%lu(rsp:%lu).", + req->fullname, inode->i_mode, rsp->inode_info.mode, inode->i_ino, rsp->inode_info.i_ino); + + qtfs_conn_put_param(pvar); + return d; + +err_end: + qtfs_conn_put_param(pvar); + return NULL; +} +int qtfs_rmdir(struct inode *dir, struct dentry *dentry) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_rmdir *req; + struct qtrsp_rmdir *rsp; + int ret; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry); + + rsp = qtfs_remote_run(pvar, QTFS_REQ_RMDIR, QTFS_SEND_SIZE(struct qtreq_rmdir, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs rmdir <%s> failed, errno:%d.\n", req->path, rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + qtfs_info("qtfs rmdir success:<%s>.\n", req->path); + qtfs_conn_put_param(pvar); + return 0; +} + +int qtfs_unlink(struct inode *dir, struct dentry *dentry) +{ + struct qtreq_unlink *req; + struct qtrsp_unlink *rsp; + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + int ret; + struct inode *inode = d_inode(dentry); + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry); + qtfs_info("qtfs unlink %s.\n", req->path); + + rsp = qtfs_remote_run(pvar, QTFS_REQ_UNLINK, QTFS_SEND_SIZE(struct qtreq_unlink, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->errno < 0) { + qtfs_err("qtfs unlink %s failed, errno:%d\n", req->path, rsp->errno); + } else { + qtfs_info("qtfs unlink %s success\n", req->path); + inode->i_ctime = dir->i_ctime; + inode_dec_link_count(inode); + } + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; +} + +int qtfs_link(struct dentry *old_dentry, struct inode *dir, struct dentry *new_dentry) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_link *req; + struct qtrsp_link *rsp; + int error; + struct inode *inode = d_inode(old_dentry); + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, old_dentry); + req->d.oldlen = strlen(req->path) + 1; + QTFS_FULLNAME(req->path + req->d.oldlen, new_dentry); + req->d.newlen = strlen(req->path + req->d.oldlen) + 1; + rsp = qtfs_remote_run(pvar, QTFS_REQ_LINK, sizeof(struct qtreq_link) - sizeof(req->path) + req->d.newlen + req->d.oldlen); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs link failed %d\n", rsp->errno); + error = rsp->errno; + goto err_end; + } + inode->i_ctime = current_time(inode); + inode_inc_link_count(inode); + ihold(inode); + d_instantiate(new_dentry, inode); + qtfs_info("qtfs link success, old:%s new:%s", req->path, req->path + req->d.oldlen); + qtfs_conn_put_param(pvar); + return 0; + +err_end: + qtfs_conn_put_param(pvar); + return error; +} + +int qtfs_symlink(struct inode *dir, struct dentry *dentry, const char *symname) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_symlink *req; + struct qtrsp_symlink *rsp; + struct inode *inode; + int error; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry); + req->d.newlen = strlen(req->path) + 1; + if (req->d.newlen + strlen(symname) + 1 > sizeof(req->path)) { + qtfs_conn_put_param(pvar); + qtfs_err("qtfs symlink path name too long\n"); + return -EINVAL; + } + strncpy(&req->path[req->d.newlen], symname, sizeof(req->path) - req->d.newlen - 1); + + req->d.oldlen = strlen(&req->path[req->d.newlen]) + 1; + rsp = qtfs_remote_run(pvar, QTFS_REQ_SYMLINK, sizeof(struct qtreq_symlink) - sizeof(req->path) + req->d.newlen + req->d.oldlen); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs symlink failed %d\n", rsp->errno); + error = rsp->errno; + goto err_end; + } + inode = qtfs_iget(dentry->d_sb, &(rsp->inode_info)); + error = qtfs_new_entry(inode, dentry); + qtfs_info("qtfs symlink success, path:%s symname:%s", req->path, symname); + qtfs_conn_put_param(pvar); + return error; + +err_end: + qtfs_conn_put_param(pvar); + return error; +} + +int qtfs_getattr(const struct path *path, struct kstat *stat, u32 req_mask, unsigned int flags) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_getattr *req; + struct qtrsp_getattr *rsp; + int ret; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, path->dentry); + req->request_mask = req_mask; + req->query_flags = flags; + + (void)qtfs_mountpoint_path_init(path->dentry, (struct path*)path, req->path); + rsp = qtfs_remote_run(pvar, QTFS_REQ_GETATTR, QTFS_SEND_SIZE(struct qtreq_getattr, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->ret) { + qtfs_err("qtfs getattr <%s> failed.errno: %d %s\n", req->path, rsp->errno, + (rsp->errno != -ENOENT) ? "." : "file not exist"); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + *stat = rsp->stat; + qtfs_debug("qtfs getattr success:<%s> blksiz:%u size:%lld mode:%o ino:%llu pathino:%lu.\n", req->path, rsp->stat.blksize, + rsp->stat.size, rsp->stat.mode, rsp->stat.ino, path->dentry->d_inode->i_ino); + qtfs_conn_put_param(pvar); + return 0; +} + +int qtfs_setattr(struct dentry *dentry, struct iattr *attr) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_setattr *req; + struct qtrsp_setattr *rsp; + int ret; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + QTFS_FULLNAME(req->path, dentry); + req->attr = *attr; + req->attr.ia_file = NULL; + qtfs_info("iattr iavalid:%u mode:0x%o size:%lld file:0x%lx\n", + req->attr.ia_valid, req->attr.ia_mode, req->attr.ia_size, (unsigned long)req->attr.ia_file); + rsp = qtfs_remote_run(pvar, QTFS_REQ_SETATTR, QTFS_SEND_SIZE(struct qtreq_setattr, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs setattr <%s> failed. %d\n", req->path, rsp->errno); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; + } + qtfs_info("qtfs setattr <%s> success.\n", req->path); + qtfs_conn_put_param(pvar); + return 0; +} +const char *qtfs_getlink(struct dentry *dentry, + struct inode *inode, struct delayed_call *done) +{ + struct qtfs_sock_var_s *pvar = NULL; + struct qtreq_getlink *req; + struct qtrsp_getlink *rsp; + int len = 0; + struct qtfs_fs_info *fsinfo = qtfs_priv_byinode(inode); + char *link = NULL; + + link = READ_ONCE(inode->i_link); + if (link) { + qtfs_info("qtfs get link cache.\n"); + return link; + } + + if (dentry == NULL) { + return ERR_PTR(-ECHILD); + } + pvar = qtfs_conn_get_param(); + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return ERR_PTR(-EINVAL); + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + if (qtfs_fullname(req->path, dentry) < 0) { + qtfs_err("qtfs fullname failed\n"); + qtfs_conn_put_param(pvar); + return ERR_PTR(-EINVAL); + } + rsp = qtfs_remote_run(pvar, QTFS_REQ_GETLINK, QTFS_SEND_SIZE(struct qtreq_getlink, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return (void *)rsp; + } + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs getlink <%s> failed. %d\n", req->path, rsp->errno); + qtfs_conn_put_param(pvar); + return ERR_PTR(-ENOENT); + } + if (fsinfo->mnt_path) + len = strlen(fsinfo->mnt_path) + strlen(rsp->path) + 1; + else + len = strlen(rsp->path) + 1; + link = kmalloc(len, GFP_KERNEL); + if (!link) { + qtfs_conn_put_param(pvar); + return ERR_PTR(-ENOMEM); + } + memset(link, 0, len); + if (rsp->path[0] == '/' && fsinfo->mnt_path) + strcat(link, fsinfo->mnt_path); + strcat(link, rsp->path); + qtfs_info("get link success <%s>\n", link); + + set_delayed_call(done, kfree_link, link); + qtfs_conn_put_param(pvar); + return link; +} + +int qtfs_rename(struct inode *old_dir, struct dentry *old_dentry, + struct inode *new_dir, struct dentry *new_dentry, + unsigned int flags) +{ + struct qtreq_rename *req; + struct qtrsp_rename *rsp; + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + int ret; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + req->d.oldlen = qtfs_fullname(req->path, old_dentry); + if (req->d.oldlen < 0) { + qtfs_err("qtfs fullname failed\n"); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + req->d.oldlen += 1; + req->d.newlen = qtfs_fullname(&req->path[req->d.oldlen], new_dentry); + if (req->d.newlen < 0) { + qtfs_err("qtfs fullname failed\n"); + qtfs_conn_put_param(pvar); + return -EINVAL; + } + req->d.newlen += 1; + req->d.flags = flags; + + rsp = qtfs_remote_run(pvar, QTFS_REQ_RENAME, sizeof(struct qtreq_rename) - sizeof(req->path) + req->d.oldlen + req->d.newlen); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + + if (rsp->ret == QTFS_ERR) { + qtfs_err("qtfs rename failed,errno:%d\n", rsp->errno); + } else { + qtfs_info("qtfs rename success, oldname:%s newname:%s flags:%x\n", req->path, &req->path[req->d.oldlen], flags); + } + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; +} + +static struct inode_operations qtfs_inode_ops = { + .create = qtfs_create, + .lookup = qtfs_lookup, + .mkdir = qtfs_mkdir, + .rmdir = qtfs_rmdir, + .unlink = qtfs_unlink, + .symlink = qtfs_symlink, + .link = qtfs_link, + .mknod = qtfs_mknod, + .getattr = qtfs_getattr, + .setattr = qtfs_setattr, + .rename = qtfs_rename, +}; + +static struct inode_operations qtfs_symlink_inode_ops = { + .get_link = qtfs_getlink, + .getattr = qtfs_getattr, + .setattr = qtfs_setattr, +}; + +const struct xattr_handler *qtfs_xattr_handlers[] = { + &qtfs_xattr_user_handler, + &qtfs_xattr_trusted_handler, + &qtfs_xattr_security_handler, + &qtfs_xattr_hurd_handler, + NULL +}; + +static int qtfs_fill_super(struct super_block *sb, void *priv_data, int silent) +{ + struct inode *root_inode; + int mode = S_IFDIR; + int err; + struct qtfs_fs_info *priv = (struct qtfs_fs_info *)priv_data; + + root_inode = new_inode(sb); + root_inode->i_ino = 1; + + inode_init_owner(root_inode, NULL, mode); + root_inode->i_sb = sb; + if (priv->type == QTFS_PROC) { + qtfs_info("qtfs type: proc\n"); + root_inode->i_op = &qtfs_proc_inode_ops; + } else { + qtfs_info("qtfs type: normal\n"); + root_inode->i_op = &qtfs_inode_ops; + } + root_inode->i_fop = &qtfs_dir_ops; + root_inode->i_atime = root_inode->i_mtime = root_inode->i_ctime = CURRENT_TIME(root_inode); + + sb->s_xattr = qtfs_xattr_handlers; + err = super_setup_bdi(sb); + if (err) { + qtfs_err("qtfs fill super bdi setup err:%d.\n", err); + } + sb->s_fs_info = priv; + sb->s_op = &qtfs_ops; + sb->s_time_gran = 1; + + sb->s_root = d_make_root(root_inode); + return 0; +} + +struct dentry *qtfs_fs_mount(struct file_system_type *fs_type, + int flags, const char *dev_name, void *data) +{ + struct qtreq_mount *req = NULL; + struct qtrsp_mount *rsp = NULL; + struct dentry *ret; + struct qtfs_fs_info *priv = NULL; + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return ERR_PTR(-ENXIO); + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + strlcpy(req->path, dev_name, PATH_MAX); + rsp = qtfs_remote_run(pvar, QTFS_REQ_MOUNT, strlen(dev_name)); + if (IS_ERR(rsp) || rsp == NULL || rsp->ret != QTFS_OK) { + qtfs_err("qtfs fs mount failed, path:<%s> not exist at peer.\n", dev_name); + qtfs_conn_put_param(pvar); + return ERR_PTR(-ENOENT); + } + + priv = (struct qtfs_fs_info *)kmalloc(sizeof(struct qtfs_fs_info), GFP_KERNEL); + if (err_ptr(priv)) { + qtfs_err("qtfs priv kmalloc failed:%ld\n", PTR_ERR(priv)); + qtfs_conn_put_param(pvar); + return ERR_PTR(-ENOMEM); + } + + memset(priv, 0, sizeof(struct qtfs_fs_info)); + priv->type = qtfs_get_type((char *)data); + strlcpy(priv->peer_path, dev_name, NAME_MAX); + priv->mnt_path = NULL; + + ret = mount_nodev(fs_type, flags, (void *)priv, qtfs_fill_super); + if (err_ptr(ret)) { + qtfs_err("mount qtfs error.\n"); + } else { + qtfs_info("mount qtfs success dev name:%s.\n", dev_name); + } + + qtfs_conn_put_param(pvar); + return ret; +} + +void qtfs_kill_sb(struct super_block *sb) +{ + qtfs_info("qtfs superblock deleted.\n"); + kill_anon_super(sb); +} + diff --git a/qtfs/qtfs/syscall.c b/qtfs/qtfs/syscall.c new file mode 100644 index 0000000..c83270f --- /dev/null +++ b/qtfs/qtfs/syscall.c @@ -0,0 +1,526 @@ +#include +#include +#include +#include +#include +#include +#include + +#include "conn.h" +#include "qtfs-mod.h" + +static long qtfs_remote_mount(char __user *dev_name, char __user *dir_name, char __user *type, + unsigned long flags, void __user *data); +static int qtfs_remote_umount(char __user *name, int flags); + +static char *qtfs_copy_mount_string(const void __user *data) +{ + return data ? strndup_user(data, PATH_MAX) : NULL; +} + +static void *qtfs_copy_mount_options(const void __user *data) +{ + char *copy; + unsigned left, offset; + + if (!data) + return NULL; + + copy = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!copy) + return ERR_PTR(-ENOMEM); + left = copy_from_user(copy, data, PAGE_SIZE); + /* + * Not all architectures have an exact copy_from_user, Resort to + * byte at a time. + */ + offset = PAGE_SIZE - left; + while (left) { + char c; + if (get_user(c, (const char __user *)data + offset)) + break; + copy[offset] = c; + left--; + offset++; + } + + if (left == PAGE_SIZE) { + kfree(copy); + return ERR_PTR(-EFAULT); + } + return copy; +} + +static inline int qtfs_fstype_judgment(char __user *dir) +{ + struct path path; + int ret; + + ret = user_path_at(AT_FDCWD, dir, LOOKUP_FOLLOW, &path); + if (ret) + return 0; + + if (strcmp(path.mnt->mnt_sb->s_type->name, QTFS_FSTYPE_NAME) == 0) { + qtfs_info("qtfs fstype judge <%s> is qtfs.\n", path.dentry->d_iname); + path_put(&path); + return 1; + } + path_put(&path); + + return 0; +} + +/* if this dir is root node of qtfs */ +static inline int qtfs_root_judgment(char __user *dir) +{ + struct dentry *dentry; + struct path path; + int ret = 0; + + ret = user_path_at(AT_FDCWD, dir, LOOKUP_FOLLOW, &path); + if (ret) + return 0; + + dentry = path.dentry; + if (dentry->d_parent == dentry) + ret = 1; + path_put(&path); + + return ret; +} + +static void do_epoll_ctl_remote(int op, struct epoll_event __user *event, struct file *file) +{ + struct qtreq_epollctl *req; + struct qtrsp_epollctl *rsp; + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct private_data *priv = file->private_data; + struct epoll_event tmp; + + if (pvar == NULL) { + qtfs_err("qtfs do epoll ctl remote get pvar failed."); + return; + } + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + req->fd = priv->fd; + req->file = priv->file; + req->op = op; + if (ep_op_has_event(op) && copy_from_user(&tmp, event, sizeof(struct epoll_event))) { + qtfs_err("qtfs do epoll ctl remote copy from user failed."); + qtfs_conn_put_param(pvar); + return; + } + req->event.events = tmp.events; + req->event.data = (__u64)file; + rsp = qtfs_remote_run(pvar, QTFS_REQ_EPOLL_CTL, sizeof(struct qtreq_epollctl)); + if (rsp == NULL || IS_ERR(rsp) || rsp->ret == QTFS_ERR) { + qtfs_err("qtfs do epoll ctl remote failed."); + qtfs_conn_put_param(pvar); + qtinfo_cntinc(QTINF_EPOLL_FDERR); + return; + } + if (op == EPOLL_CTL_ADD) { + qtinfo_cntinc(QTINF_EPOLL_ADDFDS); + } else { + qtinfo_cntinc(QTINF_EPOLL_DELFDS); + } + qtfs_info("qtfs do epoll ctl remote success, fd:%d file:%lx.", req->fd, (unsigned long)req->file); + qtfs_conn_put_param(pvar); + return; +} + +int qtfs_epoll_ctl_remote(int op, int fd, struct epoll_event __user * event) +{ + struct fd f; + struct file *file; + struct private_data *priv; + int ret = 0; + f = fdget(fd); + if (!f.file) { + // qtfs_err("epoll ctl remote fd:%d file error.", fd); + return -1; + } + file = f.file; + if (strcmp(file->f_path.mnt->mnt_sb->s_type->name, QTFS_FSTYPE_NAME) != 0) { + ret = 0; + goto end; + } + if (!S_ISFIFO(file->f_inode->i_mode)) { + char *fullname = (char *)kmalloc(MAX_PATH_LEN, GFP_KERNEL); + memset(fullname, 0, MAX_PATH_LEN); + if (qtfs_fullname(fullname, file->f_path.dentry) < 0) { + qtfs_err("qtfs fullname failed\n"); + kfree(fullname); + ret = -1; + goto end; + } + qtfs_info("qtfs remote epoll not support file:%s mode:%o.", fullname, file->f_inode->i_mode); + kfree(fullname); + ret = -1; + goto end; + } + + priv = file->private_data; + if (priv == NULL) { + qtfs_err("epoll ctl remote failed, private data invalid."); + ret = -1; + goto end; + } + + qtfs_info("qtfs qtfs remote epoll file:%s mode:%x.", file->f_path.dentry->d_iname, file->f_inode->i_mode); + do_epoll_ctl_remote(op, event, file); + +end: + fdput(f); + return ret; +} + +__SYSCALL_DEFINEx(4, _qtfs_epoll_ctl, int, epfd, int, op, int, fd, + struct epoll_event __user *, event) +{ + struct epoll_event epds; + int ret = -1; + + ret = qtfs_epoll_ctl_remote(op, fd, event); + if (ep_op_has_event(op) && + copy_from_user(&epds, event, sizeof(struct epoll_event))) + return -EFAULT; + if (!ret) { + return qtfs_kern_syms.do_epoll_ctl(epfd, op, fd, &epds, false); + } else { + return -1; + } +} + +__SYSCALL_DEFINEx(5, _qtfs_mount, char __user *, dev_name, char __user *, dir_name, + char __user *, type, unsigned long, flags, void __user *, data) +{ + int ret; + char *kernel_type; + char *kernel_dev; + void *options = NULL; + + // if both dev_name and dir_name are qtfs, it is a remote mount operator. + kernel_type = qtfs_copy_mount_string(type); + ret = PTR_ERR(kernel_type); + if (IS_ERR(kernel_type)) + goto out_type; + + kernel_dev = qtfs_copy_mount_string(dev_name); + ret = PTR_ERR(kernel_dev); + if (IS_ERR(kernel_dev)) + goto out_dev; + + options = qtfs_copy_mount_options(data); + ret = PTR_ERR(options); + if (IS_ERR(options)) + goto out_data; + + // if both dev_name and dir_name are qtfs, it is a remote mount operator, + if (qtfs_fstype_judgment(dir_name) == 1) { + ret = qtfs_remote_mount(kernel_dev, dir_name, kernel_type, flags, options); + goto remote_mount; + } + + ret = qtfs_kern_syms.do_mount(kernel_dev, dir_name, kernel_type, flags, options); + +remote_mount: + kfree(options); +out_data: + kfree(kernel_dev); +out_dev: + kfree(kernel_type); +out_type: + return ret; +} + +__SYSCALL_DEFINEx(2, _qtfs_umount, char __user *, name, int, flags) +{ + int lookup_flags = LOOKUP_MOUNTPOINT; + struct path path; + int ret; + + // basic validate checks done first + if (flags & ~(MNT_FORCE | MNT_DETACH | MNT_EXPIRE | UMOUNT_NOFOLLOW)) + return -EINVAL; + + /* if umount path is qtfs and not qtfs root, then do remote umount */ + if (qtfs_fstype_judgment(name) && !qtfs_root_judgment(name)) { + return qtfs_remote_umount(name, flags); + } + + if (!(flags & UMOUNT_NOFOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + ret = user_path_at(AT_FDCWD, name, lookup_flags, &path); + if (ret) + return ret; + return qtfs_kern_syms.path_umount(&path, flags); +} + +// make the page writable +int make_rw(unsigned long address) +{ + unsigned int level; + pte_t *pte = lookup_address(address, &level); + pte->pte |= _PAGE_RW; + + return 0; +} +// make the page write protected +int make_ro(unsigned long address) +{ + unsigned int level; + pte_t *pte = lookup_address(address, &level); + pte->pte &= ~_PAGE_RW; + return 0; +} + +int qtfs_dir_to_qtdir(char *dir, char *qtdir) +{ + int ret; + struct path path; + ret = kern_path(dir, LOOKUP_FOLLOW, &path); + if (ret) { + qtfs_err("qtfs dir to qtdir failed, ret: %d\n", ret); + return ret; + } + + ret = qtfs_fullname(qtdir, path.dentry); + path_put(&path); + return ret; +} + +int qtfs_parse_data(void *data, void *newc) +{ + char *options = data, *key; + int ret = 0; + char *newchar = (char *)newc; + char *lowers; + int firstkey = 0; + + if (!options) + return 0; + + while ((key = strsep(&options, ",")) != NULL) { + if (*key) { + size_t v_len = 0; + char *value = strchr(key, '='); + + if (value) { + if (value == key) + continue; + *value++ = 0; + v_len = strlen(value); + } + if (firstkey != 0) + strcat(newchar, ","); + firstkey++; + + qtfs_info("qtfs parse mount data: key:%s value:%s v_len:%ld\n", key, value, v_len); + if (strcmp(key, "lowerdir") == 0) { + int firstlower = 0; + strcat(newchar, "lowerdir="); + while ((lowers = strsep(&value, ":")) != NULL) { + if (*lowers) { + if (firstlower != 0) + strcat(newchar, ":"); + firstlower++; + qtfs_dir_to_qtdir(lowers, &newchar[strlen(newchar)]); + } + } + } else { + strcat(newchar, key); + strcat(newchar, "="); + qtfs_dir_to_qtdir(value, &newchar[strlen(newchar)]); + } + } + } + qtfs_info("qtfs parse mount data result:%s\n", newchar); + + return ret; +} + +/* + * overlay -o format: lowerdir=/xxx/overlay/lower:/xx/xx,upperdir= + * /xx/overlay/upper,workdir=/xxx/overlay/work + */ +static inline void *qtfs_mount_data_overlay(void *data) +{ + void *mod; + int len; + len = strlen((char *)data); + mod = (void *)kmalloc(len + 1, GFP_KERNEL); + if (mod == NULL) { + qtfs_err("kmalloc failed.\n"); + return (void *)-ENOMEM; + } + memset(mod, 0, len+1); + qtfs_parse_data(data, mod); + memset(data, 0, len); + strcpy(data, mod); + + kfree(mod); + return data; +} + +static inline void *qtfs_mount_datapage_modify(char *type, void *data) +{ + if (type != NULL && strcmp(type, "overlay") == 0) { + return qtfs_mount_data_overlay(data); + } + return data; +} + +static long qtfs_remote_mount(char *dev_name, char __user *dir_name, char *type, + unsigned long flags, void *data) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_sysmount *req; + struct qtrsp_sysmount *rsp; + char *kernel_dir; + int ret; + int totallen; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var\n"); + return -EINVAL; + } + kernel_dir = qtfs_copy_mount_string(dir_name); + ret = PTR_ERR(kernel_dir); + if (IS_ERR(kernel_dir)) { + qtfs_err("qtfs remote mount %s, kernel dir dup failed ret:%d.\n", dev_name, ret); + qtfs_conn_put_param(pvar); + return ret; + } + + // data = qtfs_mount_datapage_modify(type, data); + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + if (dev_name != NULL) { + qtfs_dir_to_qtdir(dev_name, req->buf); + req->d.dev_len = strlen(dev_name) + 1; + } else { + req->d.dev_len = 0; + } + + strcpy(&req->buf[req->d.dev_len], kernel_dir); + qtfs_dir_to_qtdir(kernel_dir, &req->buf[req->d.dev_len]); + req->d.dir_len = strlen(&req->buf[req->d.dev_len]) + 1; + if (type != NULL) { + strcpy(&req->buf[req->d.dev_len + req->d.dir_len], type); + req->d.type_len = strlen(type) + 1; + } else { + req->d.type_len = 0; + } + + if (data != NULL) { + req->d.data_len = strlen(data) + 1; + strcpy(&req->buf[req->d.dev_len + req->d.dir_len + req->d.type_len], data); + } else { + req->d.data_len = 0; + } + req->d.flags = flags; + + totallen = req->d.dev_len + req->d.dir_len + req->d.type_len + req->d.data_len; + if (totallen >= sizeof(req->buf)) { + qtfs_err("qtfs remote mount devname:%s, dir_name:%s failed, len:%d is too big.\n", dev_name, kernel_dir, totallen); + rsp->errno = -EFAULT; + goto out_free; + } + + rsp = qtfs_remote_run(pvar, QTFS_REQ_SYSMOUNT, sizeof(struct qtreq_sysmount) - sizeof(req->buf) + totallen); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->errno < 0) { + qtfs_err("qtfs remote mount failed, devname:%s dir_name:%s type:%s, data:%s, flags(0x%lx), errno:%d\n", + dev_name, kernel_dir, type, (char *)data, flags, rsp->errno); + } else { + qtfs_info("qtfs remote mount success devname:%s dir_name:%s type:%s, data:%s, flags(0x%lx)\n", + dev_name, kernel_dir, type, (char *)data, flags); + } + +out_free: + kfree(kernel_dir); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; +} + +static int qtfs_remote_umount(char __user *name, int flags) +{ + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + struct qtreq_sysumount *req; + struct qtrsp_sysumount *rsp; + char *kernel_name; + int ret; + + if (pvar == NULL) { + qtfs_err("qtfs remote umount get pvar failed."); + return -EINVAL; + } + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + kernel_name = qtfs_copy_mount_string(name); + req->flags = flags; + qtfs_dir_to_qtdir(kernel_name, req->buf); + qtfs_info("qtfs remote umount string:%s reqbuf:%s", (kernel_name == NULL) ? "INVALID":kernel_name, req->buf); + + rsp = qtfs_remote_run(pvar, QTFS_REQ_SYSUMOUNT, sizeof(struct qtreq_sysumount) - sizeof(req->buf) + strlen(req->buf)); + if (IS_ERR(rsp) || rsp == NULL) { + kfree(kernel_name); + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->errno) + qtfs_err("qtfs remote umount failed, errno:%d\n", rsp->errno); + + kfree(kernel_name); + ret = rsp->errno; + qtfs_conn_put_param(pvar); + return ret; +} + +static unsigned long *qtfs_oldsyscall_mount = NULL; +static unsigned long *qtfs_oldsyscall_umount = NULL; +static unsigned long *qtfs_oldsyscall_epoll_ctl = NULL; + +int qtfs_syscall_init(void) +{ + qtfs_debug("qtfs use my_mount instead of mount:0x%lx umount:0x%lx\n", + (unsigned long)qtfs_kern_syms.sys_call_table[__NR_mount], (unsigned long)qtfs_kern_syms.sys_call_table[__NR_umount2]); + qtfs_debug("qtfs use my_epoll_ctl instead of epoll_ctl:0x%lx\n", + (unsigned long)qtfs_kern_syms.sys_call_table[__NR_epoll_ctl]); + qtfs_oldsyscall_mount = qtfs_kern_syms.sys_call_table[__NR_mount]; + qtfs_oldsyscall_umount = qtfs_kern_syms.sys_call_table[__NR_umount2]; + qtfs_oldsyscall_epoll_ctl = qtfs_kern_syms.sys_call_table[__NR_epoll_ctl]; + make_rw((unsigned long)qtfs_kern_syms.sys_call_table); +#ifdef __x86_64__ + qtfs_kern_syms.sys_call_table[__NR_mount] = (unsigned long *)__x64_sys_qtfs_mount; + qtfs_kern_syms.sys_call_table[__NR_umount2] = (unsigned long *)__x64_sys_qtfs_umount; + qtfs_kern_syms.sys_call_table[__NR_epoll_ctl] = (unsigned long *)__x64_sys_qtfs_epoll_ctl; +#endif +#ifdef __aarch64__ + qtfs_kern_syms.sys_call_table[__NR_mount] = (unsigned long *)__arm64_sys_qtfs_mount; + qtfs_kern_syms.sys_call_table[__NR_umount2] = (unsigned long *)__arm64_sys_qtfs_umount; + qtfs_kern_syms.sys_call_table[__NR_epoll_ctl] = (unsigned long *)__arm64_sys_qtfs_epoll_ctl; +#endif + make_ro((unsigned long)qtfs_kern_syms.sys_call_table); + qtfs_debug("qtfs use my_mount:0x%lx, my_umount:0x%lx, my_epoll_ctl:0x%lx\n", + (unsigned long)qtfs_kern_syms.sys_call_table[__NR_mount], + (unsigned long)qtfs_kern_syms.sys_call_table[__NR_umount2], + (unsigned long)qtfs_kern_syms.sys_call_table[__NR_epoll_ctl]); + return 0; +} + +int qtfs_syscall_fini(void) +{ + qtfs_info("qtfs mount resume to 0x%lx.\n", (unsigned long)qtfs_oldsyscall_mount); + + make_rw((unsigned long)qtfs_kern_syms.sys_call_table); + qtfs_kern_syms.sys_call_table[__NR_mount] = (unsigned long *)qtfs_oldsyscall_mount; + qtfs_kern_syms.sys_call_table[__NR_umount2] = (unsigned long *)qtfs_oldsyscall_umount; + qtfs_kern_syms.sys_call_table[__NR_epoll_ctl] = (unsigned long *)qtfs_oldsyscall_epoll_ctl; + /*set mkdir syscall to the original one */ + make_ro((unsigned long)qtfs_kern_syms.sys_call_table); + return 0; +} diff --git a/qtfs/qtfs/syscall.h b/qtfs/qtfs/syscall.h new file mode 100644 index 0000000..1308a73 --- /dev/null +++ b/qtfs/qtfs/syscall.h @@ -0,0 +1,7 @@ +#ifndef __QTFS_SYSCALL_H__ +#define __QTFS_SYSCALL_H__ + +extern int qtfs_syscall_init(void); +extern int qtfs_syscall_fini(void); + +#endif diff --git a/qtfs/qtfs/xattr.c b/qtfs/qtfs/xattr.c new file mode 100644 index 0000000..12398d1 --- /dev/null +++ b/qtfs/qtfs/xattr.c @@ -0,0 +1,169 @@ +#include +#include + +#include "conn.h" +#include "qtfs-mod.h" +#include "req.h" +#include "log.h" + +static bool qtfs_xattr_list(struct dentry *dentry) +{ + struct qtreq_xattrlist *req; + struct qtrsp_xattrlist *rsp; + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + bool ret; + + if (!pvar) { + qtfs_err("qtfs_xattr_list Failed to get qtfs sock var"); + return -EINVAL; + } + + if (dentry == NULL) { + qtfs_err("qtfs_xattr_list dentry is NULL."); + qtfs_conn_put_param(pvar); + return false; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + if (qtfs_fullname(req->path, dentry) < 0) { + qtfs_err("qtfs fullname failed"); + qtfs_conn_put_param(pvar); + return false; + } + + rsp = qtfs_remote_run(pvar, QTFS_REQ_XATTRLIST, strlen(req->path) + 1); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_err("qtfs_xattr_list remote run failed."); + qtfs_conn_put_param(pvar); + return false; + } + + if (rsp->d.ret == QTFS_ERR) { + qtfs_err("qtfs_xattr_list failed with ret:%d.", rsp->d.ret); + ret = rsp->d.result; + qtfs_conn_put_param(pvar); + return ret; + } + ret = rsp->d.result; + qtfs_conn_put_param(pvar); + return ret; +} + +static bool qtfs_xattr_user_list(struct dentry *dentry) +{ + return qtfs_xattr_list(dentry); +} + +static bool qtfs_xattr_trusted_list(struct dentry *dentry) +{ + return qtfs_xattr_list(dentry); +} + +static bool qtfs_xattr_security_list(struct dentry *dentry) +{ + return qtfs_xattr_list(dentry); +} + +static bool qtfs_xattr_hurd_list(struct dentry *dentry) +{ + return qtfs_xattr_list(dentry); +} + +static int qtfs_xattr_get(const struct xattr_handler *handler, + struct dentry *dentry, struct inode *inode, + const char *name, void *buffer, size_t size) +{ + struct qtreq_xattrget *req; + struct qtrsp_xattrget *rsp; + struct qtfs_sock_var_s *pvar = qtfs_conn_get_param(); + size_t leftlen = size; + char *buf = (char *)buffer; + + if (!pvar) { + qtfs_err("Failed to get qtfs sock var"); + return 0; + } + if (buf == NULL || size <= 0) { + qtfs_conn_put_param(pvar); + return 0; + } + + if (dentry == NULL) { + qtfs_conn_put_param(pvar); + return 0; + } + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + if (qtfs_fullname(req->path, dentry) < 0) { + qtfs_err("qtfs fullname failed"); + qtfs_conn_put_param(pvar); + return 0; + } + + if (strlen(handler->prefix) + strlen(name) <= (sizeof(req->d.prefix_name) - 1)) { + strcpy(req->d.prefix_name, handler->prefix); + strcat(req->d.prefix_name, name); + } else { + qtfs_err("strcpy len too long"); + qtfs_conn_put_param(pvar); + return 0; + } + + rsp = qtfs_sock_msg_buf(pvar, QTFS_RECV); + do { + req->d.pos = rsp->d.pos; + req->d.size = size; + rsp = qtfs_remote_run(pvar, QTFS_REQ_XATTRGET, QTFS_SEND_SIZE(struct qtreq_xattrget, req->path)); + if (IS_ERR(rsp) || rsp == NULL) { + qtfs_conn_put_param(pvar); + return PTR_ERR(rsp); + } + if (rsp->d.ret == QTFS_ERR || rsp->d.size > req->d.size || leftlen < rsp->d.size) { + goto err_end; + } + if (rsp->d.size > 0 && rsp->d.size <= leftlen) { + memcpy(&buf[size - leftlen], rsp->buf, rsp->d.size); + } else { + qtfs_err("qtfs xattr get error <%s>, rsp size:%ld leftlen:%lu", req->path, rsp->d.size, leftlen); + break; + } + leftlen -= rsp->d.size; + } while (leftlen > 0 && rsp->d.size > 0); + qtfs_info("qtfs getxattr success:<<%s>>", buf); + + qtfs_conn_put_param(pvar); + + return size - leftlen; + +err_end: + qtfs_conn_put_param(pvar); + return -ENODATA; +} + +const struct xattr_handler qtfs_xattr_user_handler = { + .prefix = XATTR_USER_PREFIX, + .list = qtfs_xattr_user_list, + .get = qtfs_xattr_get, + //.set = qtfs_xattr_set, +}; + +const struct xattr_handler qtfs_xattr_trusted_handler = { + .prefix = XATTR_TRUSTED_PREFIX, + .list = qtfs_xattr_trusted_list, + .get = qtfs_xattr_get, + //.set = qtfs_xattr_set, +}; + +const struct xattr_handler qtfs_xattr_security_handler = { + .prefix = XATTR_SECURITY_PREFIX, + .list = qtfs_xattr_security_list, + .get = qtfs_xattr_get, + //.set = qtfs_xattr_set, +}; + +const struct xattr_handler qtfs_xattr_hurd_handler = { + .prefix = XATTR_HURD_PREFIX, + .list = qtfs_xattr_hurd_list, + .get = qtfs_xattr_get, + //.set = qtfs_xattr_set, +}; diff --git a/qtfs/qtfs_server/Makefile b/qtfs/qtfs_server/Makefile new file mode 100644 index 0000000..c1c5ef6 --- /dev/null +++ b/qtfs/qtfs_server/Makefile @@ -0,0 +1,18 @@ +ccflags-y += -I$(src)/../ -I$(src) -DQTFS_SERVER +KBUILD=/lib/modules/$(shell uname -r)/build/ + +obj-m:=qtfs_server.o +qtfs_server-objs:=../conn.o fsops.o qtfs-server.o ../misc.o + +all: qtfs_server engine + +qtfs_server: + make -C $(KBUILD) M=$(PWD) modules + +engine: + gcc -O2 -o engine user_engine.c -lpthread -I../ -DQTFS_SERVER + +clean: + make -C $(KBUILD) M=$(PWD) clean + rm -rf engine + rm -rf ../*.o diff --git a/qtfs/qtfs_server/fsops.c b/qtfs/qtfs_server/fsops.c new file mode 100644 index 0000000..7fc5aa2 --- /dev/null +++ b/qtfs/qtfs_server/fsops.c @@ -0,0 +1,1070 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "conn.h" +#include "qtfs-server.h" +#include "req.h" +#include "log.h" +#include "fsops.h" +#include "comm.h" + +#define REQ(arg) (arg->data) +#define RSP(arg) (arg->out) +#define USERP(arg) (arg->userp) + +static inline void qtfs_inode_info_fill(struct inode_info *ii, struct inode *inode) +{ + ii->mode = inode->i_mode; + ii->i_opflags = inode->i_opflags; + ii->i_uid = inode->i_uid; + ii->i_gid = inode->i_gid; + ii->i_flags = inode->i_flags; + ii->i_ino = inode->i_ino; + ii->i_rdev = inode->i_rdev; + ii->i_size = inode->i_size; + ii->atime = inode->i_atime; + ii->mtime = inode->i_mtime; + ii->ctime = inode->i_ctime; + ii->i_bytes = inode->i_bytes; + ii->i_blkbits = inode->i_blkbits; + ii->i_write_hint = inode->i_write_hint; + ii->i_blocks = inode->i_blocks; + ii->i_state = inode->i_state; + ii->dirtied_when = inode->dirtied_when; + ii->dirtied_time_when = inode->dirtied_time_when; + ii->i_generation = inode->i_generation; + return; +} + +static int handle_ioctl(struct qtserver_arg *arg) +{ + int ret; + int iret; + struct file *file; + struct qtreq_ioctl *req = (struct qtreq_ioctl *)REQ(arg); + struct qtrsp_ioctl *rsp = (struct qtrsp_ioctl *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + file = filp_open(req->path, O_RDONLY, 0); + if (err_ptr(file)) { + qtfs_err("handle ioctl error, path:<%s> failed.\n", req->path); + rsp->ret = QTFS_ERR; + rsp->size = 0; + rsp->errno = -ENOENT; + return sizeof(struct qtrsp_ioctl) - sizeof(rsp->buf); + } + + switch (req->d.cmd) { + case FS_IOC_FSGETXATTR: + iret = file->f_op->unlocked_ioctl(file, req->d.cmd, (unsigned long)userp->userp); + if (iret) { + qtfs_err("fsgetxattr ioctl failed with %d\n", iret); + rsp->errno = iret; + goto err; + } + ret = copy_from_user(rsp->buf, userp->userp, sizeof(struct fsxattr)); + if (ret) { + qtfs_err("fsgetxattr copy_from_user failed with %d\n", ret); + rsp->errno = ret; + goto err; + } + rsp->ret = QTFS_OK; + rsp->errno = iret; + rsp->size = sizeof(struct fsxattr); + filp_close(file, NULL); + return sizeof(struct qtrsp_ioctl) - sizeof(rsp->buf) + sizeof(struct fsxattr); + case FS_IOC_FSSETXATTR: + if (req->d.size <= 0) { + rsp->errno = -EINVAL; + goto err; + } + ret = copy_to_user(userp->userp, req->path+req->d.offset, req->d.size); + if (ret) { + qtfs_err("fssetxattr copy_to_user failed with %d\n", ret); + rsp->errno = ret; + goto err; + } + iret = file->f_op->unlocked_ioctl(file, req->d.cmd, (unsigned long)userp->userp); + if (iret) { + qtfs_err("fssetxattr ioctl failed with %d\n", iret); + rsp->errno = iret; + goto err; + } + rsp->ret = QTFS_OK; + rsp->errno = iret; + rsp->size = 0; + filp_close(file, NULL); + return sizeof(struct qtrsp_ioctl) - sizeof(rsp->buf); + default: + rsp->errno = -EOPNOTSUPP; + goto err; + } +err: + rsp->ret = QTFS_ERR; + rsp->size = 0; + filp_close(file, NULL); + return sizeof(struct qtrsp_ioctl) - sizeof(rsp->buf); +} + +static int handle_statfs(struct qtserver_arg *arg) +{ + int ret; + struct qtreq_statfs *req = (struct qtreq_statfs *)REQ(arg); + struct qtrsp_statfs *rsp = (struct qtrsp_statfs *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + ret = copy_to_user(userp->userp, req->path, strlen(req->path)+1); + if (ret) { + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_statfs); + } + + ret = qtfs_kern_syms.user_statfs((char *)userp->userp, &(rsp->kstat)); + if (ret) { + qtfs_err("qtfs server handle statfs path:%s failed with ret:%d.\n", req->path, ret); + rsp->ret = QTFS_ERR; + } else { + qtfs_info("qtfs server handle statfs path:%s success.\n", req->path); + rsp->ret = QTFS_OK; + } + rsp->errno = ret; + return sizeof(struct qtrsp_statfs); +} + +static int handle_mount(struct qtserver_arg *arg) +{ + struct path path; + int ret; + struct qtreq_mount *req = (struct qtreq_mount *)REQ(arg); + struct qtrsp_mount *rsp = (struct qtrsp_mount *)RSP(arg); + + ret = kern_path(req->path, LOOKUP_DIRECTORY, &path); + if (ret) { + qtfs_err("handle mount path:%s not exist.\n", req->path); + rsp->ret = QTFS_ERR; + } else { + rsp->ret = QTFS_OK; + qtfs_info("handle mount path:%s success.\n", req->path); + path_put(&path); + } + return sizeof(rsp->ret); +} + +int handle_open(struct qtserver_arg *arg) +{ + int fd; + int ret; + struct fd f; + struct file *file = NULL; + struct qtreq_open *req = (struct qtreq_open *)REQ(arg); + struct qtrsp_open *rsp = (struct qtrsp_open *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + ret = copy_to_user(userp->userp, req->path, strlen(req->path)+1); + if (ret) { + qtfs_err("handle open copy to user failed, ret:%d userp:%lx path:%s", ret, (unsigned long)userp->userp, req->path); + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_open); + } + fd = qtfs_kern_syms.do_sys_open(AT_FDCWD, (char *)userp->userp, req->flags, req->mode); + if (fd == -EEXIST) { + qtfs_err("handle open file <<%s>> flags:%llx mode:%o, opened:failed %d, do again\n", req->path, req->flags, req->mode, fd); + req->flags &= ~(O_CREAT | O_EXCL); + fd = qtfs_kern_syms.do_sys_open(AT_FDCWD, (char *)userp->userp, req->flags, req->mode); + } + if (fd < 0) { + if (fd != -ENOENT) { + qtfs_err("handle open file <<%s>>flags:%llx mode:%o, opened:failed %d\n", req->path, req->flags, req->mode, fd); + } else { + qtfs_info("handle open file <<%s>>flags:%llx mode:%o, opened:failed - file not exist\n", req->path, req->flags, req->mode); + } + rsp->ret = QTFS_ERR; + rsp->fd = fd; + rsp->file = 0; + return sizeof(struct qtrsp_open); + } + + f = fdget(fd); + file = f.file; + if (err_ptr(file)) { + rsp->ret = QTFS_ERR; + rsp->fd = PTR_ERR(file); + // must close_fd(fd)? + WARN_ON(1); + qtfs_err("handle open get file pointer of <<%s>> error, fd:%d file err:%d.", req->path, fd, rsp->fd); + // XXX: fileclose here? + } else { + rsp->ret = QTFS_OK; + rsp->file = (__u64)file; + rsp->fd = fd; + } + qtfs_info("handle open file :%s fd:%d filep:%lx.", req->path, fd, (unsigned long)rsp->file); + fdput(f); + return sizeof(struct qtrsp_open); +} + +int handle_close(struct qtserver_arg *arg) +{ + struct qtreq_close *req = (struct qtreq_close *)REQ(arg); + struct qtrsp_close *rsp = (struct qtrsp_close *)RSP(arg); + + // fd >= 3 is valid + if (req->fd <= 2) { + qtfs_err("handle close an invalid fd:%d.", req->fd); + WARN_ON(1); + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_close); + } + + rsp->ret = qtfs_kern_syms.__close_fd(current->files, req->fd); + qtfs_info("handle close file, fd:%d ret:%d", req->fd, rsp->ret); + return sizeof(struct qtrsp_close); +} + +static int handle_readiter(struct qtserver_arg *arg) +{ + struct file *file = NULL; + struct qtreq_readiter *req = (struct qtreq_readiter *)REQ(arg); + struct qtrsp_readiter *rsp = (struct qtrsp_readiter *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + size_t maxlen = (req->len >= sizeof(rsp->readbuf)) ? (sizeof(rsp->readbuf) - 1) : req->len; + + file = (struct file *)req->file; + if (err_ptr(file)) { + qtfs_err("handle readiter error, open failed, file:%p.\n", file); + rsp->d.ret = QTFS_ERR; + rsp->d.len = 0; + rsp->d.errno = -ENOENT; + return sizeof(struct qtrsp_readiter) - sizeof(rsp->readbuf) + rsp->d.len; + } + if (file->f_op->read) { + int idx = 0; + int ret = 0; + do { + if (idx + userp->size < maxlen) { + ret = file->f_op->read(file, userp->userp, userp->size, &req->pos); + } else { + ret = file->f_op->read(file, userp->userp, maxlen - idx, &req->pos); + } + if (ret <= 0) + break; + if (copy_from_user(&rsp->readbuf[idx], userp->userp, ret)) { + qtfs_err("readiter copy from user failed."); + break; + } + rsp->d.len += ret; + idx += ret; + } while (ret > 0 && idx < maxlen); + if (ret < 0) { + qtfs_err("handle readiter ret:%d.", ret); + rsp->d.len = ret; + } + } else { + rsp->d.len = kernel_read(file, rsp->readbuf, maxlen, &req->pos); + } + if (rsp->d.len > maxlen || rsp->d.len < 0) { + rsp->d.ret = QTFS_ERR; + rsp->d.errno = (int)rsp->d.len; + } else { + rsp->d.ret = QTFS_OK; + } + + qtfs_info("handle readiter file:<%s>, len:%lu, rsplen:%ld, pos:%lld, ret:%d errno:%d.\n", + file->f_path.dentry->d_iname, req->len, rsp->d.len, req->pos, rsp->d.ret, rsp->d.errno); + return sizeof(struct qtrsp_readiter) - sizeof(rsp->readbuf) + rsp->d.len; +} + +static int handle_write(struct qtserver_arg *arg) +{ + struct file *file = NULL; + struct qtreq_write *req = (struct qtreq_write *)REQ(arg); + struct qtrsp_write *rsp = (struct qtrsp_write *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + int idx = 0, leftlen = 0, ret = 0, len = 0; + + file = (struct file *)req->d.file; + if (err_ptr(file)) { + qtfs_err("qtfs handle write error, filp:<%p> open failed.\n", file); + rsp->ret = QTFS_ERR; + rsp->len = 0; + return sizeof(struct qtrsp_write); + } + + file->f_mode = req->d.mode; + file->f_flags = req->d.flags; + if (file->f_op->write) { + leftlen = req->d.buflen; + rsp->len = 0; + while (leftlen > 0) { + len = leftlen > userp->size ? userp->size : leftlen; + if (copy_to_user(userp->userp, &req->path_buf[idx], len)) { + qtfs_err("write copy to userp failed.\n"); + rsp->len = -EFAULT; + break; + } + ret = file->f_op->write(file, userp->userp, len, &req->d.pos); + if (ret < 0) { + rsp->len = ret; + break; + } + leftlen -= ret; + idx += ret; + rsp->len += ret; + } + } else { + rsp->len = kernel_write(file, req->path_buf, req->d.buflen, &req->d.pos); + } + rsp->ret = (rsp->len <= 0) ? QTFS_ERR : QTFS_OK; + qtfs_info("handle write file<%s> %s, write len:%ld pos:%lld mode:%o flags:%x.", file->f_path.dentry->d_iname, + (rsp->ret == QTFS_ERR) ? "failed" : "succeded", rsp->len, req->d.pos, file->f_mode, file->f_flags); + return sizeof(struct qtrsp_write); +} + +static int handle_lookup(struct qtserver_arg *arg) +{ + struct path path; + struct inode *inode; + struct qtreq_lookup *req = (struct qtreq_lookup *)REQ(arg); + struct qtrsp_lookup *rsp = (struct qtrsp_lookup *)RSP(arg); + int ret; + ret = kern_path(req->fullname, 0, &path); + if (ret) { + qtfs_info("qtfs handle lookup(%s) not exist, ret%d.\n", req->fullname, ret); + rsp->ret = QTFS_ERR; + } else { + inode = path.dentry->d_inode; + rsp->ret = QTFS_OK; + qtfs_inode_info_fill(&rsp->inode_info, inode); + qtfs_debug("handle lookup name:%s, mode:%o ino:%lu", req->fullname, rsp->inode_info.mode, rsp->inode_info.i_ino); + path_put(&path); + } + return sizeof(struct qtrsp_lookup); +} + +static int qtfs_filldir(struct dir_context *ctx, const char *name, int namelen, + loff_t offset, u64 ino, unsigned int d_type) +{ + struct qtfs_dirent64 *dirent, *prev; + struct qtfs_getdents *buf = container_of(ctx, struct qtfs_getdents, ctx); + int reclen = ALIGN(offsetof(struct qtfs_dirent64, d_name) + namelen + 1, sizeof(u64)); + int prev_reclen; + + if (reclen > buf->count) + return -EINVAL; + + prev_reclen = buf->prev_reclen; + dirent = buf->dir; + prev = (void *)dirent - prev_reclen; + prev->d_off = offset; + dirent->d_ino = ino; + dirent->d_reclen = reclen; + dirent->d_type = d_type; + memcpy(dirent->d_name, name, namelen); + + buf->prev_reclen = reclen; + buf->dir = (void *)dirent + reclen; + buf->count -= reclen; + buf->vldcnt++; + return 0; +} + +static int handle_readdir(struct qtserver_arg *arg) +{ + struct file *file = NULL; + struct qtreq_readdir *req = (struct qtreq_readdir *)REQ(arg); + struct qtrsp_readdir *rsp = (struct qtrsp_readdir *)RSP(arg); + int ret; + struct qtfs_getdents buf = { + .ctx.actor = qtfs_filldir, + .ctx.pos = req->pos, + .prev_reclen = 0, + .count = req->count, + .dir = (struct qtfs_dirent64 *)rsp->dirent, + .vldcnt = 0, + }; + file = filp_open(req->path, O_RDONLY|O_NONBLOCK|O_DIRECTORY, 0); + if (err_ptr(file)) { + qtfs_err("handle readdir error, filp:<%s> open failed.\n", req->path); + rsp->d.ret = QTFS_ERR; + rsp->d.vldcnt = 0; + return sizeof(struct qtrsp_readdir) - sizeof(rsp->dirent); + } + file->f_pos = req->pos; + ret = iterate_dir(file, &buf.ctx); + rsp->d.pos = file->f_pos; + rsp->d.ret = QTFS_OK; + rsp->d.vldcnt = buf.vldcnt; + rsp->d.over = (req->pos == rsp->d.pos) ? 1 : 0; + qtfs_info("handle readdir ret:%d, pos:%lld path:%s, valid count:%d, leftcount:%d validbyte:%lu\n", + ret, req->pos, req->path, buf.vldcnt, buf.count, sizeof(rsp->dirent) - buf.count); + filp_close(file, NULL); + + return sizeof(struct qtrsp_readdir) - buf.count; +} + +static int handle_mkdir(struct qtserver_arg *arg) +{ + struct qtreq_mkdir *req = (struct qtreq_mkdir *)REQ(arg); + struct qtrsp_mkdir *rsp = (struct qtrsp_mkdir *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + struct inode *inode; + struct path path; + int ret; + + if (copy_to_user(userp->userp, req->path, strlen(req->path) + 1)) { + qtfs_err("handle mkdir copy to userp failed.\n"); + rsp->errno = -EFAULT; + goto err; + } + rsp->errno = qtfs_kern_syms.do_mkdirat(AT_FDCWD, userp->userp, req->mode); + if (rsp->errno < 0) { + qtfs_err("handle mkdir failed with ret:%d.", rsp->errno); + goto err; + } + ret = kern_path(req->path, 0, &path); + if (ret) { + qtfs_err("handle mkdir failed in kern path, ret:%d.\n", ret); + } else { + inode = d_inode(path.dentry); + qtfs_inode_info_fill(&rsp->inode_info, inode); + path_put(&path); + } + rsp->ret = QTFS_OK; + qtfs_info("handle mkdir path:%s success.", req->path); + return sizeof(struct qtrsp_mkdir); + +err: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_mkdir); +} + +static int handle_rmdir(struct qtserver_arg *arg) +{ + struct qtreq_rmdir *req = (struct qtreq_rmdir *)REQ(arg); + struct qtrsp_rmdir *rsp = (struct qtrsp_rmdir *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + if (copy_to_user(userp->userp, req->path, strlen(req->path) + 1)) { + qtfs_err("handle rmdir copy to userp failed.\n"); + rsp->errno = -EFAULT; + goto err; + } + rsp->errno = qtfs_kern_syms.do_rmdir(AT_FDCWD, qtfs_kern_syms.getname(userp->userp)); + if (rsp->errno < 0) { + qtfs_err("handle rmdir error:%d.", rsp->errno); + goto err; + } + qtfs_info("handle rmdir path:%s success.", req->path); + rsp->ret = QTFS_OK; + return sizeof(struct qtrsp_rmdir); + +err: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_rmdir); +} + +static int handle_getattr(struct qtserver_arg *arg) +{ + struct qtreq_getattr *req = (struct qtreq_getattr *)REQ(arg); + struct qtrsp_getattr *rsp = (struct qtrsp_getattr *)RSP(arg); + struct path path; + int ret; + + qtfs_debug("handle getattr path:%s\n", req->path); + ret = kern_path(req->path, 0, &path); + if (ret) { + qtfs_err("handle getattr path:%s failed, ret:%d %s\n", req->path, ret, (ret != -ENOENT) ? "." : "file not exist"); + goto failed; + } + + ret = vfs_getattr(&path, &rsp->stat, req->request_mask, req->query_flags); + if (ret) { + qtfs_err("vfs getattr path:%s ret:%d\n", req->path, ret); + rsp->errno = ret; + path_put(&path); + goto failed; + } + rsp->ret = QTFS_OK; + path_put(&path); + qtfs_debug("handle getattr:<%s> blksize:%u size:%lld mode:%o ino:%llu req_mask:%x req_flags:%u.\n", req->path, rsp->stat.blksize, + rsp->stat.size, rsp->stat.mode, rsp->stat.ino, req->request_mask, req->query_flags); + return sizeof(struct qtrsp_getattr); + +failed: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_getattr); +} + +static int handle_setattr(struct qtserver_arg *arg) +{ + struct qtreq_setattr *req = (struct qtreq_setattr *)REQ(arg); + struct qtrsp_setattr *rsp = (struct qtrsp_setattr *)RSP(arg); + struct inode *inode = NULL; + struct path path; + int ret; + + ret = kern_path(req->path, 0, &path); + if (ret) { + qtfs_err("handle setattr path:%s failed in kern_path with %d\n", req->path, ret); + rsp->ret = QTFS_ERR; + rsp->errno = -ENOENT; + return sizeof(struct qtrsp_setattr); + } + inode = path.dentry->d_inode; + inode_lock(inode); + rsp->errno = notify_change(path.dentry, &req->attr, NULL); + if (rsp->errno < 0) { + rsp->ret = QTFS_ERR; + qtfs_err("handle setattr, path:<%s> failed with %d.\n", req->path, ret); + goto end; + } + + qtfs_info("handle setattr iattr success iavalid:%u mode:%o size:%lld file:0x%lx\n", + req->attr.ia_valid, req->attr.ia_mode, req->attr.ia_size, (unsigned long)req->attr.ia_file); + rsp->ret = QTFS_OK; + +end: + inode_unlock(inode); + path_put(&path); + return sizeof(struct qtrsp_setattr); +} + +int handle_icreate(struct qtserver_arg *arg) +{ + struct file *file = NULL; + struct inode *inode; + struct qtreq_icreate *req = (struct qtreq_icreate *)REQ(arg); + struct qtrsp_icreate *rsp = (struct qtrsp_icreate *)RSP(arg); + + file = filp_open(req->path, O_CREAT, req->mode); + if (err_ptr(file)) { + qtfs_err("handle icreate filp:<%s> failed in open.\n", req->path); + rsp->ret = QTFS_ERR; + rsp->errno = PTR_ERR(file); + return sizeof(struct qtrsp_icreate); + } + inode = file->f_inode; + qtfs_inode_info_fill(&rsp->inode_info, inode); + filp_close(file, NULL); + rsp->ret = QTFS_OK; + qtfs_info("handle icreate path:%s success, inode mode:%ho\n", req->path, + rsp->inode_info.mode); + return sizeof(struct qtrsp_icreate); +} + +static int handle_mknod(struct qtserver_arg *arg) +{ + struct qtreq_mknod *req = (struct qtreq_mknod *)REQ(arg); + struct qtrsp_mknod *rsp = (struct qtrsp_mknod *)RSP(arg); + struct dentry *dent = NULL; + struct path path; + int error; + unsigned int flags = LOOKUP_DIRECTORY; + +retry: + dent = kern_path_create(AT_FDCWD, req->path, &path, flags); + if (err_ptr(dent)) { + rsp->ret = QTFS_ERR; + qtfs_info("handle mknod path:<%s>, mode:%o in kern_path_create with ret:%ld\n", req->path, req->mode, PTR_ERR(dent)); + return sizeof(struct qtrsp_mknod); + } + + if (!IS_POSIXACL(path.dentry->d_inode)) + req->mode &= ~current_umask(); + error = security_path_mknod(&path, dent, req->mode, req->dev); + if (!error) + error = vfs_mknod(path.dentry->d_inode, dent, req->mode, req->dev); + done_path_create(&path, dent); + if (error == -ESTALE && !(flags & LOOKUP_REVAL)) { + flags |= LOOKUP_REVAL; + qtfs_debug("retry mknod.\n"); + rsp->errno = error; + goto retry; + } + qtfs_inode_info_fill(&rsp->inode_info, dent->d_inode); + rsp->ret = QTFS_OK; + qtfs_info("handle mknod path:<%s>, mode:%o success\n", req->path, req->mode); + rsp->errno = 0; + return sizeof(struct qtrsp_mknod); +} + +int handle_unlink(struct qtserver_arg *arg) +{ + struct qtreq_unlink *req = (struct qtreq_unlink *)REQ(arg); + struct qtrsp_unlink *rsp = (struct qtrsp_unlink *)RSP(arg); + + rsp->errno = qtfs_kern_syms.do_unlinkat(AT_FDCWD, qtfs_kern_syms.getname_kernel(req->path)); + if (rsp->errno < 0) { + qtfs_err("handle unlink failed, errno:%d\n", rsp->errno); + } else { + qtfs_info("handle unlink path:%s success\n", req->path); + } + return sizeof(struct qtrsp_unlink); +} + +int handle_link(struct qtserver_arg *arg) +{ + char *oldname, *newname; + struct qtreq_link *req = (struct qtreq_link *)REQ(arg); + struct qtrsp_link *rsp = (struct qtrsp_link *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + oldname = req->path; + newname = req->path + req->d.oldlen; + if (copy_to_user(userp->userp, oldname, strlen(oldname) + 1) || + copy_to_user(userp->userp2, newname, strlen(newname) + 1)) { + qtfs_err("handle link failed in copy to userp.\n"); + rsp->errno = -EFAULT; + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_link); + } + + rsp->errno = qtfs_kern_syms.do_linkat(AT_FDCWD, userp->userp, AT_FDCWD, userp->userp2, 0); + qtfs_info("handle link new:%s old:%s return %d\n", newname, oldname, rsp->errno); + rsp->ret = rsp->errno == 0 ? QTFS_OK : QTFS_ERR; + return sizeof(struct qtrsp_link); +} + +int handle_symlink(struct qtserver_arg *arg) +{ + char *oldname, *newname; + struct qtreq_symlink *req = (struct qtreq_symlink *)REQ(arg); + struct qtrsp_symlink *rsp = (struct qtrsp_symlink *)RSP(arg); + int error; + struct dentry *dentry; + struct path path; + unsigned int lookup_flags = 0; + + newname = req->path; + oldname = &req->path[req->d.newlen]; +retry: + dentry = kern_path_create(AT_FDCWD, newname, &path, lookup_flags); + error = PTR_ERR(dentry); + if (err_ptr(dentry)) { + rsp->ret = QTFS_ERR; + qtfs_err("handle_symlink: newname(%s), oldname(%s) in kern_path_create %d\n", newname, oldname, error); + return sizeof(struct qtrsp_symlink); + } + + rsp->errno = vfs_symlink(path.dentry->d_inode, dentry, oldname); + done_path_create(&path, dentry); + if (rsp->errno == -ESTALE && !(lookup_flags & LOOKUP_REVAL)) { + lookup_flags |= LOOKUP_REVAL; + goto retry; + } + rsp->ret = QTFS_OK; + qtfs_info("handle_symlink: newname(%s), oldname(%s) success\n", newname, oldname); + qtfs_inode_info_fill(&rsp->inode_info, dentry->d_inode); + return sizeof(struct qtrsp_symlink); +} + +int handle_getlink(struct qtserver_arg *arg) +{ + struct qtreq_getlink *req = (struct qtreq_getlink *)REQ(arg); + struct qtrsp_getlink *rsp = (struct qtrsp_getlink *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + if (copy_to_user(userp->userp, req->path, strlen(req->path) + 1) || + copy_to_user(userp->userp2, rsp->path, (userp->size < MAX_PATH_LEN) ? userp->size : MAX_PATH_LEN)) { + qtfs_err("handle getlink<%s> copy to userp failed.\n", req->path); + rsp->errno = -EFAULT; + goto err_handle; + } + rsp->errno = qtfs_kern_syms.do_readlinkat(AT_FDCWD, userp->userp, userp->userp2, userp->size); + if (rsp->errno < 0) { + qtfs_err("handle getlink<%s> do readlinkat failed, errno:%d\n", req->path, rsp->errno); + goto err_handle; + } + if (copy_from_user(rsp->path, userp->userp2, rsp->errno)) { + qtfs_err("handle getlink<%s> copy from user failed, len:%d.", req->path, rsp->errno); + rsp->errno = -EFAULT; + goto err_handle; + } + rsp->ret = QTFS_OK; + qtfs_info("handle getlink<%s> ok, len:%d link:%s.", req->path, rsp->errno, rsp->path); + return sizeof(struct qtrsp_getlink) - sizeof(rsp->path) + strlen(rsp->path) + 1; + +err_handle: + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_getlink) - sizeof(rsp->path); +} + +int handle_rename(struct qtserver_arg *arg) +{ + struct qtreq_rename *req = (struct qtreq_rename *)REQ(arg); + struct qtrsp_rename *rsp = (struct qtrsp_rename *)RSP(arg); + struct qtfs_server_userp_s *userp = (struct qtfs_server_userp_s *)USERP(arg); + + if (copy_to_user(userp->userp, req->path, strlen(req->path) + 1) || + copy_to_user(userp->userp2, &req->path[req->d.oldlen], strlen(&req->path[req->d.oldlen]) + 1)) { + qtfs_err("handle rename copy to userp failed.\n"); + rsp->errno = -EFAULT; + goto err_handle; + } + rsp->errno = qtfs_kern_syms.do_renameat2(AT_FDCWD, userp->userp, AT_FDCWD, userp->userp2, 0); + +err_handle: + rsp->ret = (rsp->errno < 0) ? QTFS_ERR : QTFS_OK; + qtfs_info("handle rename oldname:%s newname:%s ret:%d %s", req->path, &req->path[req->d.oldlen], rsp->errno, + (rsp->errno < 0) ? "failed" : "successed"); + return sizeof(struct qtrsp_rename); +} + +int handle_xattrlist(struct qtserver_arg *arg) +{ + struct qtreq_xattrlist *req = (struct qtreq_xattrlist *)REQ(arg); + struct qtrsp_xattrlist *rsp = (struct qtrsp_xattrlist *)RSP(arg); + struct path path; + int ret; + ssize_t size; + int i; + + ret = kern_path(req->path, 0, &path); + if (ret) { + qtfs_err("handle xattr list path error.\n"); + rsp->d.errno = -ENOENT; + goto err_handle; + } + size = generic_listxattr(path.dentry, rsp->name, sizeof(rsp->name)); + path_put(&path); + if (size < 0) { + qtfs_err("handle list xattr failed, errno:%ld.\n", size); + rsp->d.errno = size; + goto err_handle; + } + if (size == 0) + goto err_handle; + rsp->d.ret = QTFS_OK; + rsp->d.result = true; + while (i < size) { + qtfs_info("handle list xattr result:%s\n", &rsp->name[i]); + i += strlen(&rsp->name[i]) + 1; + } + return sizeof(struct qtrsp_xattrlist); + +err_handle: + rsp->d.ret = QTFS_ERR; + rsp->d.result = false; + return sizeof(struct qtrsp_xattrlist); +} + +int handle_xattrget(struct qtserver_arg *arg) +{ + struct qtreq_xattrget *req = (struct qtreq_xattrget *)REQ(arg); + struct qtrsp_xattrget *rsp = (struct qtrsp_xattrget *)RSP(arg); + struct path path; + int ret = 0; + ssize_t error = 0; + int len = 0; + char *kvalue = NULL; + + ret = kern_path(req->path, 0, &path); + if (ret) { + qtfs_err("handle xattrget path error.\n"); + rsp->d.errno = -ENOENT; + goto err_handle; + } + + if (req->d.size > XATTR_SIZE_MAX) + req->d.size = XATTR_SIZE_MAX; + kvalue = (char *)kvzalloc(req->d.size, GFP_KERNEL); + if (!kvalue) { + qtfs_err("handle xattrget kvzalloc failed, size:%d.\n", req->d.size); + rsp->d.ret = QTFS_ERR; + rsp->d.errno = -ENOMEM; + path_put(&path); + goto err_handle; + } + + error = vfs_getxattr(path.dentry, req->d.prefix_name, kvalue, req->d.size); + path_put(&path); + if (error > 0) { + if (req->d.pos >= error) { + rsp->d.size = 0; + rsp->d.pos = req->d.pos; + goto end; + } + qtfs_info("handle getxattr: path:%s prefix name:%s : (%s - 0x%llx), size:%ld, reqpos:%d\n", req->path, req->d.prefix_name, kvalue, (__u64)kvalue, error, req->d.pos); + len = (error - req->d.pos)>sizeof(rsp->buf)? sizeof(rsp->buf):(error - req->d.pos); + memcpy(rsp->buf, &kvalue[req->d.pos], len); + rsp->d.pos = req->d.pos + len; + rsp->d.size = len; + } else { + rsp->d.ret = QTFS_ERR; + rsp->d.errno = error; + kvfree(kvalue); + goto err_handle; + } +end: + kvfree(kvalue); + rsp->d.ret = QTFS_OK; + return sizeof(struct qtrsp_xattrget) - sizeof(rsp->buf) + len; + +err_handle: + return sizeof(struct qtrsp_xattrget) - sizeof(rsp->buf); +} + +int handle_xattrset(struct qtserver_arg *arg) +{ + return 0; +} + +long qtfs_do_mount(const char *dev_name, const char *dir_name, + const char *type_page, unsigned long flags, void *data_page) +{ + struct path path; + int ret; + + ret = kern_path(dir_name, LOOKUP_FOLLOW, &path); + if (ret) { + qtfs_err("qtfs do mount failed, ret:%d\n", ret); + return ret; + } + qtfs_info("handle path mount: dev(%s), dir(%s), type(%s), flags(0x%lx), data(%s)", dev_name, dir_name, type_page, flags, (char *)data_page); + ret = qtfs_kern_syms.path_mount(dev_name, &path, type_page, flags, data_page); + path_put(&path); + return ret; +} + +int handle_syscall_mount(struct qtserver_arg *arg) +{ + struct qtreq_sysmount *req = (struct qtreq_sysmount *)REQ(arg); + struct qtrsp_sysmount *rsp = (struct qtrsp_sysmount *)RSP(arg); + char *dev_name, *dir_name, *type; + void *data_page; + + dev_name = req->d.dev_len == 0 ? NULL : req->buf; + dir_name = &req->buf[req->d.dev_len]; + type = req->d.type_len == 0 ? NULL : &req->buf[req->d.dev_len + req->d.dir_len]; + if (req->d.data_len != 0) + data_page = &req->buf[req->d.dev_len + req->d.dir_len + req->d.type_len]; + else + data_page = NULL; + + qtfs_info("handle syscall mount devname:%s dirname:%s type:%s data:%s\n", dev_name, dir_name, type, + (data_page == NULL) ? "nil" : (char *)data_page); + rsp->errno = qtfs_do_mount(dev_name, dir_name, type, req->d.flags, data_page); + if (rsp->errno < 0) + qtfs_err("handle syscall mount failed devname:%s dirname:%s type:%s data:%s, errno:%d\n", + dev_name, dir_name, type, (char *)data_page, rsp->errno); + + return sizeof(struct qtrsp_sysmount); +} + +int handle_syscall_umount(struct qtserver_arg *arg) +{ + struct qtreq_sysumount *req = (struct qtreq_sysumount *)REQ(arg); + struct qtrsp_sysumount *rsp = (struct qtrsp_sysumount *)RSP(arg); + int lookup_flags = LOOKUP_MOUNTPOINT; + struct path path; + int ret; + + qtfs_info("handle umount path:%s\n", req->buf); + // basic validity checks done first + if (req->flags & ~(MNT_FORCE | MNT_DETACH | MNT_EXPIRE | UMOUNT_NOFOLLOW)) { + qtfs_err("handle syscall umount flags error:%x", req->flags); + rsp->errno = -EINVAL; + return sizeof(struct qtrsp_sysumount); + } + + if (!(req->flags & UMOUNT_NOFOLLOW)) + lookup_flags |= LOOKUP_FOLLOW; + ret = kern_path(req->buf, lookup_flags, &path); + if (ret) { + qtfs_err("umount(%s) failed, ret:%d\n", req->buf, ret); + rsp->errno = ret; + return sizeof(struct qtrsp_sysumount); + } + rsp->errno = qtfs_kern_syms.path_umount(&path, req->flags); + if (rsp->errno) + qtfs_err("umount(%s) failed, errno:%d\n", req->buf, rsp->errno); + //dont need to path_put here. + return sizeof(struct qtrsp_sysumount); +} + +int handle_fifopoll(struct qtserver_arg *arg) +{ + struct qtreq_poll *req = (struct qtreq_poll *)REQ(arg); + struct qtrsp_poll *rsp = (struct qtrsp_poll *)RSP(arg); + struct file *filp = NULL; + unsigned int head, tail; + struct pipe_inode_info *pipe; + __poll_t mask; + + filp = (struct file *)req->file; + pipe = filp->private_data; + if (pipe == NULL) { + qtfs_err("file :%s pipe data is NULL.", filp->f_path.dentry->d_iname); + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_poll); + } + head = READ_ONCE(pipe->head); + tail = READ_ONCE(pipe->tail); + mask = 0; + if (filp->f_mode & FMODE_READ) { + if (!pipe_empty(head, tail)) + mask |= EPOLLIN | EPOLLRDNORM; + if (!pipe->writers && filp->f_version != pipe->w_counter) + mask |= EPOLLHUP; + } + + if (filp->f_mode & FMODE_WRITE) { + if (!pipe_full(head, tail, pipe->max_usage)) + mask |= EPOLLOUT | EPOLLWRNORM; + if (!pipe->readers) + mask |= EPOLLERR; + } + rsp->mask = mask; + rsp->ret = QTFS_OK; + + qtfs_info("handle fifo poll f_mode:%o: %s get poll 0x%x\n", filp->f_mode, filp->f_path.dentry->d_iname, rsp->ret); + return sizeof(struct qtrsp_poll); +} + +int handle_epollctl(struct qtserver_arg *arg) +{ + struct qtreq_epollctl *req = (struct qtreq_epollctl *)REQ(arg); + struct qtrsp_epollctl *rsp = (struct qtrsp_epollctl *)RSP(arg); + int ret; + struct epoll_event evt; + + evt.data = (__u64)req->event.data; + evt.events = req->event.events; + ret = qtfs_kern_syms.do_epoll_ctl(qtfs_epoll.epfd, req->op, req->fd, &evt, false); + if (ret < 0) { + qtfs_err("handle do epoll ctl failed, ret:%d.", ret); + rsp->ret = QTFS_ERR; + return sizeof(struct qtrsp_epollctl); + } + qtinfo_cntinc((req->op == EPOLL_CTL_ADD) ? QTINF_EPOLL_ADDFDS : QTINF_EPOLL_DELFDS); + rsp->ret = QTFS_OK; + qtfs_info("handle do epoll ctl success, fd:%d file:%lx op:%x data:%lx poll_t:%x.", + req->fd, (unsigned long)req->file, req->op, req->event.data, (unsigned)req->event.events); + + return sizeof(struct qtrsp_epollctl); +} + +int handle_exit(struct qtserver_arg *arg) +{ + return 4; +} + +int handle_null(struct qtserver_arg *arg) +{ + qtfs_err("unknown events."); + return 4; +} + +static struct qtserver_ops qtfs_server_handles[] = { + {QTFS_REQ_NULL, handle_null, "null"}, + {QTFS_REQ_MOUNT, handle_mount, "mount"}, + {QTFS_REQ_OPEN, handle_open, "open"}, + {QTFS_REQ_CLOSE, handle_close, "close"}, + {QTFS_REQ_READ, handle_null, "read"}, + {QTFS_REQ_READITER, handle_readiter, "readiter"}, + {QTFS_REQ_WRITE, handle_write, "write"}, + {QTFS_REQ_LOOKUP, handle_lookup, "lookup"}, + {QTFS_REQ_READDIR, handle_readdir, "readdir"}, + {QTFS_REQ_MKDIR, handle_mkdir, "mkdir"}, + {QTFS_REQ_RMDIR, handle_rmdir, "rmdir"}, + {QTFS_REQ_GETATTR, handle_getattr, "getattr"}, + {QTFS_REQ_SETATTR, handle_setattr, "setattr"}, + {QTFS_REQ_ICREATE, handle_icreate, "icreate"}, + {QTFS_REQ_MKNOD, handle_mknod, "mknod"}, + {QTFS_REQ_UNLINK, handle_unlink, "unlink"}, + {QTFS_REQ_SYMLINK, handle_symlink, "symlink"}, + {QTFS_REQ_LINK, handle_link, "link"}, + {QTFS_REQ_GETLINK, handle_getlink, "getlink"}, + {QTFS_REQ_READLINK, handle_null, "readlink"}, + {QTFS_REQ_RENAME, handle_rename, "rename"}, + + {QTFS_REQ_XATTRLIST, handle_xattrlist, "xattrlist"}, + {QTFS_REQ_XATTRGET, handle_xattrget, "xattrget"}, + {QTFS_REQ_XATTRSET, handle_xattrset, "xattrset"}, + + {QTFS_REQ_SYSMOUNT, handle_syscall_mount, "sysmount"}, + {QTFS_REQ_SYSUMOUNT, handle_syscall_umount, "sysumount"}, + {QTFS_REQ_FIFOPOLL, handle_fifopoll, "fifo_poll"}, + + {QTFS_REQ_STATFS, handle_statfs, "statfs"}, + {QTFS_REQ_IOCTL, handle_ioctl, "ioctl"}, + + {QTFS_REQ_EPOLL_CTL, handle_epollctl, "epollctl"}, + {QTFS_REQ_EPOLL_EVENT, NULL, "epollevent"}, + + {QTFS_REQ_EXIT, handle_exit, "exit"}, // keep this handle at the end +}; + +int qtfs_sock_server_run(struct qtfs_sock_var_s *pvar) +{ + int ret; + struct qtreq *req; + struct qtreq *rsp; + unsigned long totalproc = 0; + + req = pvar->vec_recv.iov_base; + rsp = pvar->vec_send.iov_base; + do { + ret = qtfs_conn_recv_block(QTFS_CONN_SOCKET, pvar); + if (ret == -EPIPE) { + qtfs_err("qtfs server thread recv EPIPE, restart the connection."); + qtfs_sm_reconnect(pvar); + break; + } + if (ret < 0) + break; + pvar->recv_valid = ret + 1; + if (req->type >= QTFS_REQ_INV) { + qtfs_err("qtfs server recv unknown operate type:%d\n", req->type); + rsp->type = req->type; + rsp->len = 0; + rsp->err = QTFS_ERR; + } else { + struct qtserver_arg arg; + arg.data = req->data; + arg.out = rsp->data; + arg.userp = &qtfs_userps[pvar->cur_threadidx]; + if (arg.userp->userp == NULL || arg.userp->userp2 == NULL) + qtfs_err("server run userp:%lx userp2:%lx", (unsigned long)arg.userp->userp, (unsigned long)arg.userp->userp2); + rsp->len = qtfs_server_handles[req->type].handle(&arg); + rsp->type = req->type; + rsp->err = QTFS_OK; + totalproc++; + qtinfo_recvinc(req->type); + } + if (rsp->len >= QTFS_REQ_MAX_LEN) { + qtfs_err("handle rsp len error type:%d len:%lu", rsp->type, rsp->len); + WARN_ON(1); + continue; + } + rsp->seq_num = req->seq_num; + pvar->vec_send.iov_len = QTFS_MSG_LEN - QTFS_REQ_MAX_LEN + rsp->len; + pvar->send_valid = pvar->vec_send.iov_len + 1; + qtfs_debug("Server thread:%d count:%lu recv len:%d type:%d(%s) seq_num:%lu, reqlen:%lu, resp len:%lu, rsp threadidx:%d.\n", + pvar->cur_threadidx, totalproc, ret, req->type, qtfs_server_handles[req->type].str, req->seq_num, + req->len, pvar->vec_send.iov_len, pvar->cur_threadidx); + ret = qtfs_conn_send(QTFS_CONN_SOCKET, pvar); + if (ret == -EPIPE) { + qtfs_err("qtfs server send get EPIPE, just restart the connection\n"); + qtfs_sm_reconnect(pvar); + break; + } + if (ret < 0) { + qtfs_err("conn send failed, ret:%d\n", ret); + WARN_ON(1); + } + qtinfo_sendinc(rsp->type); + } while(0); + + qtfs_sock_msg_clear(pvar); + return (ret < 0) ? QTERROR : QTOK; +} diff --git a/qtfs/qtfs_server/fsops.h b/qtfs/qtfs_server/fsops.h new file mode 100644 index 0000000..e15f267 --- /dev/null +++ b/qtfs/qtfs_server/fsops.h @@ -0,0 +1,12 @@ +#ifndef __QTFS_SERVER_FSOPS_H__ +#define __QTFS_SERVER_FSOPS_H__ + +struct qtfs_getdents { + struct dir_context ctx; + int vldcnt; + struct qtfs_dirent64 * dir; + int prev_reclen; + int count; +}; + +#endif diff --git a/qtfs/qtfs_server/qtfs-server.c b/qtfs/qtfs_server/qtfs-server.c new file mode 100644 index 0000000..bcd60b7 --- /dev/null +++ b/qtfs/qtfs_server/qtfs-server.c @@ -0,0 +1,287 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include +#include + +#include "conn.h" +#include "qtfs-server.h" +#include "comm.h" +#include "log.h" +#include "req.h" + +#define QTFS_EPOLL_TIMEO 1000 // unit ms + +int qtfs_server_thread_run = 1; + +long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg); + +struct file_operations qtfs_misc_fops = { + .owner=THIS_MODULE, + .unlocked_ioctl = qtfs_server_misc_ioctl, +}; + +struct qtfs_server_epoll_s qtfs_epoll = { + .epfd = -1, + .event_nums = 0, + .events = NULL, +}; + +long qtfs_server_epoll_thread(struct qtfs_sock_var_s *pvar) +{ + int n; + struct qtreq_epollevt *req; + struct qtrsp_epollevt *rsp; + struct qtreq *head; + int sendlen; + int ret = 0; + int i = 0; + + if (qtfs_epoll.epfd == -1) { + qtfs_err("qtfs epoll wait error, epfd is invalid."); + return QTERROR; + } + if (false == qtfs_sock_connected(pvar)) { + qtfs_warn("qtfs epoll thread disconnected, now try to reconnect, sock state:%u.", + (pvar->client_sock == NULL) ? (unsigned int)-1 : pvar->client_sock->state); + ret = qtfs_sm_reconnect(pvar); + } else { + ret = qtfs_sm_active(pvar); + } + if (ret != QTOK) { + qtfs_err("qtfs epoll thread connect state error, can't work."); + msleep(500); + return QTERROR; + } + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + rsp = qtfs_sock_msg_buf(pvar, QTFS_RECV); + head = pvar->vec_send.iov_base; + do { + n = qtfs_kern_syms.do_epoll_wait(qtfs_epoll.epfd, qtfs_epoll.events, qtfs_epoll.event_nums, 0); + if (n == 0) { + msleep(1); + break; + } + if (n < 0) { + msleep(100); + qtfs_err("epoll get new events number failed:%d ", n); + break; + } + qtfs_info(">>epoll get new events number:%d.", n); + if (copy_from_user(qtfs_epoll.kevents, qtfs_epoll.events, sizeof(struct epoll_event) * n)) { + qtfs_err("qtfs copy epoll events failed, events lost."); + WARN_ON(1); + break; + } + for (i = 0; i < n; i++) { + req->events[i].data = qtfs_epoll.kevents[i].data; + req->events[i].events = qtfs_epoll.kevents[i].events; + qtfs_info("epoll thread head:%lx req:%lx data:%lx(offset:%lx).", (unsigned long)pvar->vec_send.iov_base, + (unsigned long)req, req->events[i].data, (unsigned long)&req->events[i].data); + } + req->event_nums = n; + sendlen = sizeof(struct qtreq_epollevt) - sizeof(req->events) + n * sizeof(struct qtreq_epoll_event); + pvar->vec_send.iov_len = QTFS_MSG_LEN - (QTFS_REQ_MAX_LEN - sendlen); + head->len = sendlen; + head->type = QTFS_REQ_EPOLL_EVENT; + ret = qtfs_conn_send(QTFS_CONN_SOCKET, pvar); + qtfs_info("qtfs send msg conn: %s:%u sendlen:%lu ret:%d.", + pvar->addr,pvar->port, (unsigned long)pvar->vec_send.iov_len, ret); + if (ret == -EPIPE) { + qtfs_err("epoll wait send events failed get EPIPE, just wait new connection."); + qtfs_sm_reconnect(pvar); + break; + } + if (ret < 0) { + qtfs_err("epoll wait send events failed, ret:%d.", ret); + WARN_ON(1); + } +retry: + ret = qtfs_conn_recv_block(QTFS_CONN_SOCKET, pvar); + if (ret == -EAGAIN) { + if (qtfs_server_thread_run == 0) { + qtfs_warn("qtfs module exiting, goodbye!"); + return QTEXIT; + } + goto retry; + } + if (ret == -EPIPE) { + qtfs_err("epoll recv events failed get EPIPE, just wait new connection."); + qtfs_sm_reconnect(pvar); + break; + } + }while (0); + + return (ret < 0) ? QTERROR : QTOK; +} + +long qtfs_server_epoll_init(void) +{ + struct qtfs_sock_var_s *pvar = NULL; + struct qtreq_epollevt *req; + + pvar = qtfs_epoll_establish_conn(); + if (pvar == NULL) { + return QTERROR; + } + qtfs_epoll_var = pvar; + + req = qtfs_sock_msg_buf(pvar, QTFS_SEND); + qtfs_info("qtfs epoll events req size:%lu, events size:%lu, struct:%lu.", + sizeof(struct qtreq_epollevt), + sizeof(req->events), + sizeof(struct qtreq_epoll_event)); + qtfs_info("qtfs epoll wait thread, epfd:%d nums:%d events:%lx epoll var:%lx.", + qtfs_epoll.epfd, qtfs_epoll.event_nums, (unsigned long)qtfs_epoll.events, (unsigned long)qtfs_epoll_var); + + return QTOK; +} + +long qtfs_server_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + int i; + long ret = 0; + struct qtfs_sock_var_s *pvar; + struct qtfs_thread_init_s init_userp; + switch (cmd) { + case QTFS_IOCTL_THREAD_INIT: + if (copy_from_user(&init_userp, (void __user *)arg, sizeof(struct qtfs_thread_init_s))) { + qtfs_err("qtfs ioctl thread init copy from user failed."); + return QTERROR; + } + if (qtfs_userps == NULL) { + qtfs_err("qtfs ioctl thread init userps invalid."); + return QTERROR; + } + memset(qtfs_userps, 0, QTFS_MAX_THREADS * sizeof(struct qtfs_server_userp_s)); + if (copy_from_user(qtfs_userps, (void __user *)init_userp.userp, + init_userp.thread_nums * sizeof(struct qtfs_server_userp_s))) { + qtfs_err("qtfs ioctl thread init copy from userp failed."); + return QTERROR; + } + for (i = 0; i < init_userp.thread_nums; i++) + qtfs_info("userp set idx:%d size:%lu user pointer:%lx %lx", i, qtfs_userps[i].size, + (unsigned long)qtfs_userps[i].userp, (unsigned long)qtfs_userps[i].userp2); + break; + case QTFS_IOCTL_THREAD_RUN: + pvar = qtfs_conn_get_param(); + if (pvar == NULL) + break; + ret = qtfs_sock_server_run(pvar); + if (ret == QTEXIT) { + qtfs_warn("qtfs thread idx:%d exit.", pvar->cur_threadidx); + qtfs_sm_exit(pvar); + qtinfo_cntdec(QTINF_ACTIV_CONN); + } + qtfs_conn_put_param(pvar); + break; + case QTFS_IOCTL_EPFDSET: + if (copy_from_user(&qtfs_epoll, (void __user *)arg, sizeof(struct qtfs_server_epoll_s))) { + qtfs_err("copy epoll struct from arg failed."); + break; + } + qtfs_info("epoll arg set, epfd:%d event nums:%d events:%lx.", + qtfs_epoll.epfd, qtfs_epoll.event_nums, (unsigned long)qtfs_epoll.events); + qtfs_epoll.kevents = (struct epoll_event *)kmalloc(sizeof(struct epoll_event) * + qtfs_epoll.event_nums, GFP_KERNEL); + if (qtfs_epoll.kevents == NULL) { + qtfs_err("epoll kernel events kmalloc failed."); + ret = QTERROR; + break; + } + break; + case QTFS_IOCTL_EPOLL_THREAD_INIT: + ret = qtfs_server_epoll_init(); + break; + case QTFS_IOCTL_EPOLL_THREAD_RUN: + if (qtfs_epoll_var == NULL) { + qtfs_err("qtfs epoll thread run failed, var is invalid."); + ret = QTERROR; + break; + } + ret = qtfs_server_epoll_thread(qtfs_epoll_var); + if (ret == QTEXIT) { + qtfs_info("qtfs epoll thread exit."); + qtfs_epoll_cut_conn(qtfs_epoll_var); + } + break; + case QTFS_IOCTL_EXIT: + qtfs_info("qtfs server threads run set to:%lu.", arg); + qtfs_server_thread_run = arg; + break; + + case QTFS_IOCTL_ALLINFO: + case QTFS_IOCTL_CLEARALL: + case QTFS_IOCTL_LOGLEVEL: + ret = qtfs_misc_ioctl(file, cmd, arg); + break; + default: + qtfs_err("qtfs misc ioctl unknown cmd:%u.", cmd); + break; + } + + return ret; +} + +static int __init qtfs_server_init(void) +{ + qtfs_log_init(qtfs_log_level); + qtfs_diag_info = (struct qtinfo *)kmalloc(sizeof(struct qtinfo), GFP_KERNEL); + if (qtfs_diag_info == NULL) + qtfs_err("kmalloc qtfs diag info failed."); + else + memset(qtfs_diag_info, 0, sizeof(struct qtinfo)); + qtfs_userps = (struct qtfs_server_userp_s *)kmalloc( + QTFS_MAX_THREADS * sizeof(struct qtfs_server_userp_s), GFP_KERNEL); + if (qtfs_userps == NULL) + qtfs_err("kmalloc qtfs userps failed, nums:%d", QTFS_MAX_THREADS); + else + memset(qtfs_userps, 0, QTFS_MAX_THREADS * sizeof(struct qtfs_server_userp_s)); + qtfs_conn_param_init(); + qtfs_kallsyms_hack_init(); + qtfs_misc_register(); + return 0; +} + +static void __exit qtfs_server_exit(void) +{ + qtfs_mod_exiting = true; + qtfs_server_thread_run = 0; + + qtfs_conn_param_fini(); + + if (qtfs_epoll_var != NULL) { + qtfs_epoll_cut_conn(qtfs_epoll_var); + if (qtfs_epoll_var->sock != NULL) { + sock_release(qtfs_epoll_var->sock); + qtfs_epoll_var->sock = NULL; + } + qtfs_sock_var_fini(qtfs_epoll_var); + kfree(qtfs_epoll_var); + qtfs_epoll_var = NULL; + } + if (qtfs_diag_info != NULL) { + kfree(qtfs_diag_info); + qtfs_diag_info = NULL; + } + if (qtfs_userps != NULL) { + kfree(qtfs_userps); + qtfs_userps = NULL; + } + qtfs_misc_destroy(); + qtfs_info("qtfs server exit done.\n"); + return; +} + +module_param_string(qtfs_server_ip, qtfs_server_ip, sizeof(qtfs_server_ip), 0600); +MODULE_PARM_DESC(qtfs_server_ip, "qtfs server ip"); +module_param(qtfs_server_port, int, 0644); +module_param(qtfs_sock_max_conn, int, 0644); +module_param_string(qtfs_log_level, qtfs_log_level, sizeof(qtfs_log_level), 0600); + +module_init(qtfs_server_init); +module_exit(qtfs_server_exit); +MODULE_AUTHOR("liqiang64@huawei.com"); +MODULE_LICENSE("GPL"); + diff --git a/qtfs/qtfs_server/qtfs-server.h b/qtfs/qtfs_server/qtfs-server.h new file mode 100644 index 0000000..8bcadf6 --- /dev/null +++ b/qtfs/qtfs_server/qtfs-server.h @@ -0,0 +1,27 @@ +#ifndef __QTFS_SERVER_H__ +#define __QTFS_SERVER_H__ + +extern int qtfs_server_thread_run; +extern struct qtfs_server_epoll_s qtfs_epoll; +extern int qtfs_mod_exiting; + +struct qtserver_arg { + char *data; + char *out; + struct qtfs_server_userp_s *userp; +}; + +struct qtserver_ops { + int type; + // return int is output len. + int (*handle) (struct qtserver_arg *); + char str[32]; +}; + +int qtfs_sock_server_run(struct qtfs_sock_var_s *pvar); +long qtfs_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg); +int qtfs_misc_register(void); +void qtfs_misc_destroy(void); +long qtfs_misc_ioctl(struct file *file, unsigned int cmd, unsigned long arg); + +#endif diff --git a/qtfs/qtfs_server/user_engine.c b/qtfs/qtfs_server/user_engine.c new file mode 100644 index 0000000..a062b63 --- /dev/null +++ b/qtfs/qtfs_server/user_engine.c @@ -0,0 +1,236 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "comm.h" + +#define engine_out(info, ...) \ + do {\ + printf("[Engine::%s:%3d]"info"\n", __func__, __LINE__, ##__VA_ARGS__);\ + } while (0); + +#define engine_out2(info, ...) \ + do {\ + printf("[Engine::%s:%3d]"info, __func__, __LINE__, ##__VA_ARGS__);\ + } while (0); + +#define engine_err(info, ...) \ + do {\ + printf("[ERROR:Engine::%s:%3d]"info"\n", __func__, __LINE__, ##__VA_ARGS__);\ + } while (0); + +struct engine_arg { + int psize; + int fd; + int thread_idx; +}; + +#define QTFS_USERP_MAXSIZE 65536 +#define QTFS_USERP_SIZE 4096 +#define QTFS_SERVER_FILE "/dev/qtfs_server" + +int qtfs_fd; +int engine_run = 1; +static void qtfs_engine_userp_free(struct qtfs_server_userp_s *userp, int thread_nums) +{ + for (int i = 0; i < thread_nums; i++) { + if (userp[i].userp != NULL) + free(userp[i].userp); + if (userp[i].userp2 != NULL) + free(userp[i].userp2); + } + free(userp); + return; +} + +static struct qtfs_server_userp_s *qtfs_engine_thread_init(int fd, int thread_nums, int psize) +{ + struct qtfs_server_userp_s *userp; + userp = (struct qtfs_server_userp_s *)malloc(thread_nums * sizeof(struct qtfs_server_userp_s)); + if (userp == NULL) { + engine_out("engine thread init malloc failed."); + return NULL; + } + for (int i = 0; i < thread_nums; i++) { + userp[i].size = psize; + userp[i].userp = (void *)malloc(psize); + if (userp[i].userp == NULL) { + engine_out("engine userp malloc failed."); + goto rollback; + } + userp[i].userp2 = (void *)malloc(psize); + if (userp[i].userp2 == NULL) { + engine_out("engine userp2 malloc failed."); + goto rollback; + } + } + struct qtfs_thread_init_s init_userp; + init_userp.thread_nums = thread_nums; + init_userp.userp = userp; + (void)ioctl(fd, QTFS_IOCTL_THREAD_INIT, (unsigned long)&init_userp); + return userp; +rollback: + qtfs_engine_userp_free(userp, thread_nums); + return NULL; +} + +static void *qtfs_engine_kthread(void *arg) +{ + struct engine_arg *parg = (struct engine_arg *)arg; + int psize = parg->psize; + long ret; + + while (engine_run) { + ret = ioctl(parg->fd, QTFS_IOCTL_THREAD_RUN, 0); + if (ret == QTEXIT) { + engine_out("qtfs server thread:%d exit.", parg->thread_idx); + break; + } + } + +end: + engine_out("qtfs user engine over."); + return NULL; +} + +static void qtfs_signal_int(int signum) +{ + engine_out("qtfs engine recv signal number:%d.", signum); + + if (qtfs_fd < 0) { + engine_err("qtfs engine signal int file:%s open failed, fd:%d.", QTFS_SERVER_FILE, qtfs_fd); + return; + } + long ret = ioctl(qtfs_fd, QTFS_IOCTL_EXIT, 0); + engine_run = 0; + + return; +} + +static void *qtfs_engine_epoll_thread(void *data) +{ + struct engine_arg *arg = (struct engine_arg *)data; + int fd = arg->fd; + long ret; + + engine_out("qtfs server epoll thread run."); + + do { + ret = ioctl(fd, QTFS_IOCTL_EPOLL_THREAD_INIT, NULL); + if (ret == QTEXIT) { + engine_out("qtfs server epoll thread init exit."); + goto end; + } + } while (ret != 0 && engine_run); + while (engine_run) { + ret = ioctl(fd, QTFS_IOCTL_EPOLL_THREAD_RUN, NULL); + if (ret == QTEXIT) { + engine_out("qtfs server epoll thread exit."); + break; + } + } +end: + engine_out("qtfs server epoll thread exit, ret:%d.", ret); + + return NULL; +} + +int qtfs_epoll_init(int fd) +{ +#define MAX_EVENTS 64 + int epfd = epoll_create1(0); + if (epfd < 0) { + engine_err("epoll create error, ret:%d.", epfd); + return -1; + } + + struct qtfs_server_epoll_s ep; + struct epoll_event *evts; + evts = calloc(MAX_EVENTS, sizeof(struct epoll_event)); + if (evts == NULL) { + engine_err("calloc events failed."); + close(epfd); + return -1; + } + engine_out("qtfs engine set epoll arg, fd:%d event nums:%d events:%lx.", epfd, MAX_EVENTS, evts); + ep.epfd = epfd; + ep.event_nums = MAX_EVENTS; + ep.events = evts; + int ret = ioctl(fd, QTFS_IOCTL_EPFDSET, &ep); + + return epfd; +} + +int main(int argc, char *argv[]) +{ + if (argc != 3) { + engine_out("Usage: %s .", argv[0]); + engine_out(" Example: %s 4096 16.", argv[0]); + return -1; + } + int psize = atoi(argv[1]); + int thread_nums = atoi(argv[2]); + int fd = open(QTFS_SERVER_FILE, O_RDONLY); + if (fd < 0) { + engine_err("qtfs server file:%s open failed, fd:%d.", QTFS_SERVER_FILE, fd); + return 0; + } + qtfs_fd = fd; + // init epoll + int epfd = qtfs_epoll_init(fd); + if (epfd < 0) { + close(fd); + return -1; + } + + umask(0); + + pthread_t texec[QTFS_MAX_THREADS]; + pthread_t tepoll; + if (psize > QTFS_USERP_MAXSIZE || thread_nums > QTFS_MAX_THREADS) { + engine_err("qtfs engine param invalid, size:%d(must <= %d) thread_nums:%d(must <= %d).", + psize, QTFS_USERP_MAXSIZE, thread_nums, QTFS_MAX_THREADS); + goto end; + } + (void)ioctl(fd, QTFS_IOCTL_EXIT, 1); + signal(SIGINT, qtfs_signal_int); + signal(SIGKILL, qtfs_signal_int); + signal(SIGTERM, qtfs_signal_int); + + struct qtfs_server_userp_s *userp = qtfs_engine_thread_init(fd, thread_nums, psize); + if (userp == NULL) { + engine_out("qtfs engine userp init failed."); + goto end; + } + + struct engine_arg arg[QTFS_MAX_THREADS]; + for (int i = 0; i < thread_nums; i++) { + arg[i].psize = psize; + arg[i].fd = fd; + arg[i].thread_idx = i; + (void)pthread_create(&texec[i], NULL, qtfs_engine_kthread, &arg[i]); + } + (void)pthread_create(&tepoll, NULL, qtfs_engine_epoll_thread, &arg[0]); + for (int i = 0; i < thread_nums; i++) { + pthread_join(texec[i], NULL); + engine_out("qtfs engine join thread %d.", i); + } + pthread_join(tepoll, NULL); + qtfs_engine_userp_free(userp, thread_nums); + engine_out("qtfs engine join epoll thread."); +end: + close(epfd); + close(fd); + engine_out("qtfs engine over."); + return 0; +} diff --git a/qtfs/qtinfo/Makefile b/qtfs/qtinfo/Makefile new file mode 100644 index 0000000..9c3ea65 --- /dev/null +++ b/qtfs/qtinfo/Makefile @@ -0,0 +1,12 @@ + +all: qtinfo + +qtinfo: + gcc -g -O2 -D$(role) -o qtinfo qtinfo.c -I../ + +install: + cp qtinfo /usr/bin/ + +clean: + rm -rf qtinfo qtinfo.o + diff --git a/qtfs/qtinfo/qtinfo.c b/qtfs/qtinfo/qtinfo.c new file mode 100644 index 0000000..665d9a8 --- /dev/null +++ b/qtfs/qtinfo/qtinfo.c @@ -0,0 +1,343 @@ +#include +#include +#include +#include +#include +#include + +#include "qtinfo.h" +#include "comm.h" + +#ifdef client +#define QTFS_DEV_NAME "/dev/qtfs_client" +#else +#define QTFS_DEV_NAME "/dev/qtfs_server" +#endif + +#define qtinfo_out(info, ...) \ + do {\ + printf(info"\n", ##__VA_ARGS__);\ + } while (0); + +#define qtinfo_out2(info, ...) \ + do {\ + printf(info, ##__VA_ARGS__);\ + } while (0); + +#define qtinfo_err(info, ...) \ + do {\ + printf("ERROR: "info"\n", ##__VA_ARGS__);\ + } while (0); + +struct qtinfo_type_str qtinfo_all_events[] = { + {QTFS_REQ_NULL, "null"}, + {QTFS_REQ_MOUNT, "mount"}, + {QTFS_REQ_OPEN, "open"}, + {QTFS_REQ_CLOSE, "close"}, + {QTFS_REQ_READ, "read"}, + {QTFS_REQ_READITER, "readiter"}, //5 + {QTFS_REQ_WRITE, "write"}, + {QTFS_REQ_LOOKUP, "lookup"}, + {QTFS_REQ_READDIR, "readdir"}, + {QTFS_REQ_MKDIR, "mkdir"}, + {QTFS_REQ_RMDIR, "rmdir"}, //10 + {QTFS_REQ_GETATTR, "getattr"}, + {QTFS_REQ_SETATTR, "setattr"}, + {QTFS_REQ_ICREATE, "icreate"}, + {QTFS_REQ_MKNOD, "mknod"}, + {QTFS_REQ_UNLINK, "unlink"}, //15 + {QTFS_REQ_SYMLINK, "symlink"}, + {QTFS_REQ_LINK, "link"}, + {QTFS_REQ_GETLINK, "getlink"}, + {QTFS_REQ_READLINK, "readlink"}, + {QTFS_REQ_RENAME, "rename"}, //20 + + {QTFS_REQ_XATTRLIST, "xattrlist"}, + {QTFS_REQ_XATTRGET, "xattrget"}, + {QTFS_REQ_XATTRSET, "xattrset"}, + + {QTFS_REQ_SYSMOUNT, "sysmount"}, + {QTFS_REQ_SYSUMOUNT, "sysumount"}, //25 + {QTFS_REQ_FIFOPOLL, "fifo_poll"}, + + {QTFS_REQ_STATFS, "statfs"}, + {QTFS_REQ_IOCTL, "ioctl"}, + {QTFS_REQ_EPOLL_CTL, "epollctl"}, + + {QTFS_REQ_EPOLL_EVENT, "epollevent"}, +}; + +static void qtinfo_events_count(struct qtinfo *evts) +{ + unsigned long total = 0; + int i; +#ifdef client + qtinfo_out("++++++++++++++++++++++++++send events count++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->c.o_events[i], + qtinfo_all_events[i+1].str, evts->c.o_events[i+1], + qtinfo_all_events[i+2].str, evts->c.o_events[i+2]); + total += evts->c.o_events[i] + evts->c.o_events[i+1] + evts->c.o_events[i+2]; + } + for (; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->c.o_events[i]); + total += evts->c.o_events[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Send events total: %lu", total); + total = 0; + + qtinfo_out("++++++++++++++++++++++++++recv events count++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->c.i_events[i], + qtinfo_all_events[i+1].str, evts->c.i_events[i+1], + qtinfo_all_events[i+2].str, evts->c.i_events[i+2]); + total += evts->c.i_events[i] + evts->c.i_events[i+1] + evts->c.i_events[i+2]; + } + for(; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->c.i_events[i]); + total += evts->c.i_events[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Recv events total: %lu", total); + total = 0; + qtinfo_out("++++++++++++++++++++++++++send error count+++++++++++++++++++++++++++"); + for(i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->c.send_err[i], + qtinfo_all_events[i+1].str, evts->c.send_err[i+1], + qtinfo_all_events[i+2].str, evts->c.send_err[i+2]); + total += evts->c.send_err[i] + evts->c.send_err[i+1] + evts->c.send_err[i+2]; + } + for(; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->c.send_err[i]); + total += evts->c.send_err[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Send events error total: %lu", total); + total = 0; + + qtinfo_out("++++++++++++++++++++++++++recv error count+++++++++++++++++++++++++++"); + for(i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->c.recv_err[i], + qtinfo_all_events[i+1].str, evts->c.recv_err[i+1], + qtinfo_all_events[i+2].str, evts->c.recv_err[i+2]); + total += evts->c.recv_err[i] + evts->c.recv_err[i+1] + evts->c.recv_err[i+2]; + } + for(; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->c.recv_err[i]); + total += evts->c.recv_err[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Recv events error total: %lu", total); + total = 0; +#endif +#ifdef server + qtinfo_out("++++++++++++++++++++++++++send events count++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->s.o_events[i], + qtinfo_all_events[i+1].str, evts->s.o_events[i+1], + qtinfo_all_events[i+2].str, evts->s.o_events[i+2]); + total += evts->s.o_events[i] + evts->s.o_events[i+1] + evts->s.o_events[i+2]; + } + for (; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->s.o_events[i]); + total += evts->s.o_events[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Send events total: %lu", total); + total = 0; + + qtinfo_out("++++++++++++++++++++++++++recv events count++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s: %-10lu %-10s: %-10lu %-10s: %-10lu", + qtinfo_all_events[i].str, evts->s.i_events[i], + qtinfo_all_events[i+1].str, evts->s.i_events[i+1], + qtinfo_all_events[i+2].str, evts->s.i_events[i+2]); + total += evts->s.i_events[i] + evts->s.i_events[i+1] + evts->s.i_events[i+2]; + } + for(; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s: %-10lu ", qtinfo_all_events[i].str, evts->s.i_events[i]); + total += evts->s.i_events[i]; + } + qtinfo_out2("\n"); + qtinfo_out("Recv events total: %lu", total); + total=0; +#endif +} + +static void qtinfo_misc_count(struct qtinfo *info) +{ + qtinfo_out("++++++++++++++++++++++++++++++Misc count+++++++++++++++++++++++++++++"); +#ifdef client + qtinfo_out("Active connects: %-8lu Seq err count : %-8lu Restartsys : %-8lu", + info->c.cnts[QTINF_ACTIV_CONN], info->c.cnts[QTINF_SEQ_ERR], info->c.cnts[QTINF_RESTART_SYS]); + qtinfo_out("Type mismatch : %-8lu Epoll add fds : %-8lu Epoll del fds: %-8lu", + info->c.cnts[QTINF_TYPE_MISMATCH], info->c.cnts[QTINF_EPOLL_ADDFDS], info->c.cnts[QTINF_EPOLL_DELFDS]); + qtinfo_out("Epoll err fds : %-8lu", + info->c.cnts[QTINF_EPOLL_FDERR]); +#else + qtinfo_out("Active connects: %-8lu Epoll add fds: %-8lu Epoll del fds: %-8lu", + info->s.cnts[QTINF_ACTIV_CONN], info->s.cnts[QTINF_EPOLL_ADDFDS], info->s.cnts[QTINF_EPOLL_DELFDS]); +#endif +} + +static void qtinfo_thread_state(struct qtinfo *info) +{ + int i = 0; + + qtinfo_out("+++++++++++++++++++++++++++Connection state++++++++++++++++++++++++++"); + for (i = 0; i < QTFS_MAX_THREADS - 3; i+=3) { + qtinfo_out("Conn%-2d: %-10s Conn%-2d: %-10s Conn%-2d: %-10s", + i+1, QTINFO_STATE(info->thread_state[i]), + i+2, QTINFO_STATE(info->thread_state[i+1]), + i+3, QTINFO_STATE(info->thread_state[i+2])); + } + for (; i < QTFS_MAX_THREADS; i++) { + qtinfo_out2("Conn%-2d: %-10s ", i+1, QTINFO_STATE(info->thread_state[i])); + } + qtinfo_out("Epoll state: %-10s", QTINFO_STATE(info->epoll_state)); + return; +} + +static void qtinfo_pvar_count(struct qtinfo *info) +{ + int i = 0; + qtinfo_out("+++++++++++++++++++++++++++++Param count+++++++++++++++++++++++++++++"); + qtinfo_out("Parameter valid count: %-2d Parameter busy count: %-2d", + info->pvar_vld, info->pvar_busy); + for (i = 0; i < QTFS_MAX_THREADS; i++) { + qtinfo_out("Conn%-2d holder: [%-20s]", i+1, (info->who_using[i][0] == '\0') ? "No one" : info->who_using[i]); + } +} + +static void qtinfo_log_level(struct qtinfo *info) +{ + qtinfo_out("Log level: %d", info->log_level); +} + +static void qtinfo_opt_a(int fd) +{ + struct qtinfo *diag = (struct qtinfo *)malloc(sizeof(struct qtinfo)); + if (diag == NULL) { + qtinfo_err("malloc failed."); + return; + } + memset(diag, 0, sizeof(struct qtinfo)); + int ret = ioctl(fd, QTFS_IOCTL_ALLINFO, diag); + if (ret < 0) { + qtinfo_err("ioctl failed, ret:%d.", ret); + goto end; + } + qtinfo_events_count(diag); + qtinfo_misc_count(diag); + qtinfo_log_level(diag); + qtinfo_thread_state(diag); + qtinfo_pvar_count(diag); +end: + free(diag); + return; +} + +void qtinfo_opt_c(int fd) +{ + int ret = ioctl(fd, QTFS_IOCTL_CLEARALL, NULL); + return; +} + +void qtinfo_opt_l(int fd, char *level) +{ + int ret; + + ret = ioctl(fd, QTFS_IOCTL_LOGLEVEL, level); + if (ret != 0) { + qtinfo_out("Set qtfs log level:%s failed.", level); + return; + } + qtinfo_out("Set qtfs log level to %s success.", level); + return; +} + +void qtinfo_opt_t(int fd) +{ + int i; + struct qtinfo *diag = (struct qtinfo *)malloc(sizeof(struct qtinfo)); + if (diag == NULL) { + qtinfo_err("malloc failed."); + return; + } + int ret = ioctl(fd, QTFS_IOCTL_ALLINFO, (unsigned long)diag); + qtinfo_out("++++++++++++++++++++++++++qtreq_xxx size++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s req: %-10lu %-10s req: %-10lu %-10s req: %-10lu", + qtinfo_all_events[i].str, diag->req_size[i], + qtinfo_all_events[i+1].str, diag->req_size[i+1], + qtinfo_all_events[i+2].str, diag->req_size[i+2]); + } + for (; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s req: %-10lu ", qtinfo_all_events[i].str, diag->req_size[i]); + } + qtinfo_out2("\n"); + qtinfo_out("++++++++++++++++++++++++++qtrsp_xxx size++++++++++++++++++++++++++"); + for (i = 0; i < (sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str)) - 3; i+=3) { + qtinfo_out("%-10s rsp: %-10lu %-10s rsp: %-10lu %-10s rsp: %-10lu", + qtinfo_all_events[i].str, diag->rsp_size[i], + qtinfo_all_events[i+1].str, diag->rsp_size[i+1], + qtinfo_all_events[i+2].str, diag->rsp_size[i+2]); + } + for (; i < sizeof(qtinfo_all_events)/sizeof(struct qtinfo_type_str); i++) { + qtinfo_out2("%-10s rsp: %-10lu ", qtinfo_all_events[i].str, diag->rsp_size[i]); + } + qtinfo_out2("\n"); + + free(diag); + return; +} + +static void qtinfo_help(char *exec) +{ + qtinfo_out("Usage: %s [OPTION].", exec); + qtinfo_out("Display qtfs client/server diagnostic information."); + qtinfo_out(" -a, All diag info."); + qtinfo_out(" -c, Clear all diag info."); + qtinfo_out(" -l, Set log level(valid param: \"NONE\", \"ERROR\", \"WARN\", \"INFO\", \"DEBUG\")."); + qtinfo_out(" -t, For test informations."); +} + +int main(int argc, char *argv[]) +{ + int ch; + if ((argc == 1) || (argc == 2 && strcmp(argv[1], "--help") == 0)) + qtinfo_help(argv[0]); + int fd = open(QTFS_DEV_NAME, O_RDONLY|O_NONBLOCK); + if (fd < 0) { + qtinfo_err("open file %s failed.", QTFS_DEV_NAME); + return 0; + } + while ((ch = getopt(argc, argv, "acl:t")) != -1) { + switch (ch) { + case 'a': + qtinfo_opt_a(fd); + break; + case 'c': + qtinfo_opt_c(fd); + break; + case 'l': + qtinfo_opt_l(fd, optarg); + break; + case 't': + qtinfo_opt_t(fd); + break; + default: + qtinfo_help(argv[0]); + break; + } + } + + close(fd); + return 0; +} diff --git a/qtfs/qtinfo/qtinfo.h b/qtfs/qtinfo/qtinfo.h new file mode 100644 index 0000000..0244a6e --- /dev/null +++ b/qtfs/qtinfo/qtinfo.h @@ -0,0 +1,54 @@ +#ifndef __QTINFO_H__ +#define __QTINFO_H__ + + +enum qtfs_req_type +{ + QTFS_REQ_NULL, + QTFS_REQ_MOUNT, + QTFS_REQ_OPEN, + QTFS_REQ_CLOSE, + QTFS_REQ_READ, + QTFS_REQ_READITER, //5 + QTFS_REQ_WRITE, + QTFS_REQ_LOOKUP, + QTFS_REQ_READDIR, + QTFS_REQ_MKDIR, + QTFS_REQ_RMDIR, //10 + QTFS_REQ_GETATTR, + QTFS_REQ_SETATTR, + QTFS_REQ_ICREATE, + QTFS_REQ_MKNOD, + QTFS_REQ_UNLINK, //15 + QTFS_REQ_SYMLINK, + QTFS_REQ_LINK, + QTFS_REQ_GETLINK, + QTFS_REQ_READLINK, + QTFS_REQ_RENAME, //20 + + QTFS_REQ_XATTRLIST, + QTFS_REQ_XATTRGET, + QTFS_REQ_XATTRSET, + + QTFS_REQ_SYSMOUNT, + QTFS_REQ_SYSUMOUNT, //25 + QTFS_REQ_FIFOPOLL, + + QTFS_REQ_STATFS, + QTFS_REQ_IOCTL, + + QTFS_REQ_EPOLL_CTL, + + QTFS_REQ_EPOLL_EVENT, + + QTFS_REQ_EXIT, + QTFS_REQ_INV, +}; + +struct qtinfo_type_str { + enum qtfs_req_type type; + char str[20]; +}; + +#endif + diff --git a/qtfs/req.h b/qtfs/req.h new file mode 100644 index 0000000..431342f --- /dev/null +++ b/qtfs/req.h @@ -0,0 +1,519 @@ +#ifndef __QTFS_REQ_STRUCT_DEF_H__ +#define __QTFS_REQ_STRUCT_DEF_H__ + +#include +#include +#include +#include "log.h" + +enum qtreq_type { + QTFS_REQ_NULL, + QTFS_REQ_MOUNT, + QTFS_REQ_OPEN, + QTFS_REQ_CLOSE, + QTFS_REQ_READ, + QTFS_REQ_READITER, //5 + QTFS_REQ_WRITE, + QTFS_REQ_LOOKUP, + QTFS_REQ_READDIR, + QTFS_REQ_MKDIR, + QTFS_REQ_RMDIR, //10 + QTFS_REQ_GETATTR, + QTFS_REQ_SETATTR, + QTFS_REQ_ICREATE, + QTFS_REQ_MKNOD, + QTFS_REQ_UNLINK, //15 + QTFS_REQ_SYMLINK, + QTFS_REQ_LINK, + QTFS_REQ_GETLINK, + QTFS_REQ_READLINK, + QTFS_REQ_RENAME, //20 + + QTFS_REQ_XATTRLIST, + QTFS_REQ_XATTRGET, + QTFS_REQ_XATTRSET, + + QTFS_REQ_SYSMOUNT, + QTFS_REQ_SYSUMOUNT, //25 + QTFS_REQ_FIFOPOLL, + + QTFS_REQ_STATFS, + QTFS_REQ_IOCTL, + + QTFS_REQ_EPOLL_CTL, + + QTFS_REQ_EPOLL_EVENT, + + QTFS_REQ_EXIT, // exit server thread + QTFS_REQ_INV, +}; +#define QTFS_REQ_TYPEVALID(type) (type < QTFS_REQ_INV && type >= QTFS_REQ_NULL) + + +enum qtreq_ret { + QTFS_OK, + QTFS_ERR, +}; + +enum qtfs_type { + QTFS_NORMAL, + QTFS_PROC, +}; + +struct qtfs_dirent64 { + u64 d_ino; + s64 d_off; + unsigned short d_reclen; + unsigned char d_type; + unsigned char resv[5]; + char d_name[]; +}; + +#define NBYTES 256 +#define ISCHR(x) ((x >= 32 && x <= 126)) +static inline void qtfs_nbytes_print(unsigned char *buf, int bytes) +{ + int i = 0; + qtfs_info("nbyts:%d->", bytes); + for (; i < bytes; i++) { + if (ISCHR(buf[i])) { + qtfs_info("addr:0x%lx, %x(%c)\n", (unsigned long)&buf[i], buf[i], buf[i]); + } else + qtfs_info("addr:0x%lx, %x\n", (unsigned long)&buf[i], buf[i]); + } +} + + +#define QTFS_SEND 0 +#define QTFS_RECV 1 + +// maximum possible length, can be increased according to the actual situation +#define NAME_MAX 255 +#define MAX_PATH_LEN PATH_MAX +#define MAX_ELSE_LEN 4096 +#define QTFS_REQ_MAX_LEN (MAX_PATH_LEN + MAX_ELSE_LEN) + +#define MAX_BUF 4096 + +// QTFS_TAIL_LEN解释： +// 私有数据结构最大长度为QTFS_REQ_MAX_LEN，超出就越界了 +// 一般有变长buf要求的，把变长buf放在末尾 +// 其长度定义为QTFS_REQ_MAX_LEN减去前面所有成员结构长度 +// 尾部变长数组长度自定义的，整体结构体长度不能超出最大长度 +#define QTFS_TAIL_LEN(head) (QTFS_REQ_MAX_LEN - sizeof(head)) + +// QTFS_SEND_SIZE解释： +// 用来发送的数据结构buf为固定大小QTFS_REQ_MAX_LEN +// 但是我们大多数时候只使用了少量的bytes +// 只需要发送有效数据，所以私有数据结构一般采取一些关键 +// 字段，加一个动态buf的组合方式，buf放在结构体末尾 +// 当传输时，只传输关键字段和动态buf的有效长度，可以用这个宏 +// 来计算所需发送的有效长度 +// 如果结构体定义不是：关键字段+字符串buf的模式，则不能用这个宏 +// 因为这个宏使用了strlen来测量末尾有效长度 +#define QTFS_SEND_SIZE(stru, tailstr) sizeof(stru) - sizeof(tailstr) + strlen(tailstr) + 1 + +struct qtreq { + unsigned int type; // operation type + unsigned int err; + unsigned long seq_num; // check code + size_t len; + char data[QTFS_REQ_MAX_LEN]; // operation's private data +}; + +#define QTFS_MSG_LEN sizeof(struct qtreq) +#define QTFS_MSG_HEAD_LEN (QTFS_MSG_LEN - QTFS_REQ_MAX_LEN) + +struct qtreq_ioctl { + struct qtreq_ioctl_len { + unsigned int cmd; + unsigned int size; + unsigned int offset; + } d; + + char path[QTFS_TAIL_LEN(struct qtreq_ioctl_len)]; +}; + +struct qtrsp_ioctl { + int ret; + int errno; + unsigned int size; + + char buf[MAX_PATH_LEN]; +}; + +struct qtreq_statfs { + char path[MAX_PATH_LEN]; // include file name +}; + +struct qtrsp_statfs { + struct kstatfs kstat; + int ret; + int errno; +}; + +struct qtreq_mount { + char path[MAX_PATH_LEN]; // include file name +}; +struct qtrsp_mount { + int ret; +}; + +struct qtreq_open { + __u64 flags; + unsigned int mode; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_open { + __u64 file; + int fd; + int ret; +}; + +struct qtreq_close { + int fd; +}; + +struct qtrsp_close { + int ret; +}; + +struct qtreq_read { + size_t len; + long long pos; + __u64 file; +}; + +struct qtrsp_read { + struct qtrsp_read_len { + int ret; + ssize_t len; + int errno; + } d; + char readbuf[QTFS_TAIL_LEN(struct qtrsp_read_len)]; +}; + +struct qtreq_readiter { + size_t len; + long long pos; + __u64 file; +}; + +struct qtrsp_readiter { + struct qtrsp_readiter_len { + int ret; + ssize_t len; + int errno; + } d; + char readbuf[QTFS_TAIL_LEN(struct qtrsp_readiter_len)]; +}; + +struct qtreq_write { + struct qtreq_write_len { + int buflen; + long long pos; + __u64 file; + long long flags; + long long mode; + } d; + // fullname and writebuf + char path_buf[QTFS_TAIL_LEN(struct qtreq_write_len)]; +}; + +struct qtrsp_write { + int ret; + ssize_t len; // 成功写入的长度 +}; + +struct qtreq_mmap { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_mmap { + int ret; +}; + +struct qtreq_lookup { + char fullname[MAX_PATH_LEN]; +}; + +struct inode_info { + unsigned int mode; + unsigned short i_opflags; + kuid_t i_uid; + kgid_t i_gid; + unsigned int i_flags; + unsigned long i_ino; + + dev_t i_rdev; + long long i_size; + + struct timespec64 atime; + struct timespec64 mtime; + struct timespec64 ctime; + + unsigned short i_bytes; + u8 i_blkbits; + u8 i_write_hint; + blkcnt_t i_blocks; + + unsigned long i_state; + unsigned long dirtied_when; /* jiffies of first dirtying */ + unsigned long dirtied_time_when; + + __u32 i_generation; +}; + +struct qtrsp_lookup { + int ret; + struct inode_info inode_info; +}; + +struct qtreq_readdir { + int count; + loff_t pos; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_readdir { + struct qtrsp_readdir_len { + int ret; + int vldcnt; + int over; // 是否已经全部获取完成 + loff_t pos; + } d; + char dirent[QTFS_TAIL_LEN(struct qtrsp_readdir_len)]; +}; + +struct qtreq_mkdir { + umode_t mode; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_mkdir { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_rmdir { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_rmdir { + int ret; + int errno; +}; + +struct qtreq_getattr { + u32 request_mask; + unsigned int query_flags; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_getattr { + int ret; + int errno; + struct kstat stat; +}; + +struct qtreq_setattr { + struct iattr attr; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_setattr { + int ret; + int errno; +}; + +struct qtreq_icreate { + umode_t mode; + bool excl; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_icreate { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_mknod { + umode_t mode; + dev_t dev; + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_mknod { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_unlink { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_unlink { + int errno; +}; + +struct qtreq_symlink { + struct qtreq_symlink_len { + int newlen; + int oldlen; + } d; + char path[QTFS_TAIL_LEN(struct qtreq_symlink_len)]; +}; + +struct qtrsp_symlink { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_link { + struct qtreq_link_len { + int newlen; + int oldlen; + } d; + char path[QTFS_TAIL_LEN(struct qtreq_symlink_len)]; +}; + +struct qtrsp_link { + int ret; + int errno; + struct inode_info inode_info; +}; + +struct qtreq_getlink { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_getlink { + int ret; + int errno; + char path[MAX_PATH_LEN]; +}; + +struct qtreq_readlink { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_readlink { + int ret; + int errno; + int len; + char path[MAX_PATH_LEN]; +}; + +struct qtreq_rename { + struct qtreq_rename_len { + int oldlen; + int newlen; + unsigned int flags; + }d; + char path[QTFS_TAIL_LEN(struct qtreq_rename_len)]; +}; + +struct qtrsp_rename { + int ret; + int errno; +}; + +// xattr def +#define QTFS_XATTR_LEN 64 +struct qtreq_xattrlist { + char path[MAX_PATH_LEN]; +}; + +struct qtrsp_xattrlist { + struct qtrsp_xattrlist_len { + int ret; + int errno; + bool result; + }d; + char name[QTFS_TAIL_LEN(struct qtrsp_xattrlist_len)]; +}; + +struct qtreq_xattrget { + struct qtreq_xattrget_len { + int pos; + int size; // 请求最多可以读取多少字节 + char prefix_name[QTFS_XATTR_LEN]; + }d; + char path[QTFS_TAIL_LEN(struct qtreq_xattrget_len)]; +}; + +struct qtrsp_xattrget { + struct qtrsp_xattrget_len { + int ret; + int errno; + ssize_t size; + int pos; + }d; + char buf[QTFS_TAIL_LEN(struct qtrsp_xattrget_len)]; +}; +// xattr end + +struct qtreq_sysmount { + struct qtreq_sysmount_len { + int dev_len; + int dir_len; + int type_len; + int data_len; + unsigned long flags; + } d; + char buf[QTFS_TAIL_LEN(struct qtreq_sysmount_len)]; +}; + +struct qtrsp_sysmount { + int errno; +}; + +struct qtreq_sysumount { + int flags; + char buf[MAX_PATH_LEN]; +}; + +struct qtrsp_sysumount { + int errno; +}; + +struct qtreq_poll { + int fd; + __u64 file; + int qproc; +}; + +struct qtrsp_poll { + int ret; + __poll_t mask; +}; + + +struct qtreq_epollctl { + __u64 file; + int fd; + int op; + struct qtreq_epoll_event event; +}; + +struct qtrsp_epollctl { + int ret; +}; + + +// server epoll 通知 client +#define QTFS_EPOLL_MAX_EVENTS 128 +struct qtreq_epollevt { + int event_nums; + struct qtreq_epoll_event events[QTFS_EPOLL_MAX_EVENTS]; +}; + +struct qtrsp_epollevt { + int ret; +}; +#endif diff --git a/qtfs/test/cgroup.go b/qtfs/test/cgroup.go new file mode 100644 index 0000000..fcc62ae --- /dev/null +++ b/qtfs/test/cgroup.go @@ -0,0 +1,53 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "path" + "sync" + "time" + "strconv" +) + +func do_watch(dir, prefix string) { + entries, err := ioutil.ReadDir(dir) + if err != nil { + fmt.Printf("read cgroup dir(%s) failed: %s\n", dir, err.Error()) + return + } + + for _, entry := range entries { + if entry.IsDir() { + entryPath := path.Join(dir, entry.Name()) + prefix = prefix + " " + do_watch(entryPath, prefix) + } else { + filePath := path.Join(dir, entry.Name()) + file, err := os.Open(filePath) + if err == nil { + file.Close() + } + } + } +} + +func watch(dir, prefix string, wg *sync.WaitGroup) { + do_watch(dir, prefix) + wg.Done() +} + +func main() { + var wg sync.WaitGroup + begin := time.Now() + threads, _ := strconv.Atoi(os.Args[2]) + fmt.Printf("watch %s\n", os.Args[1]) + for i := 0; i < threads; i++ { + wg.Add(1) + go watch(os.Args[1], "", &wg) + fmt.Printf("Thread run %d\n", i) + } + wg.Wait() + dlt := time.Since(begin) + fmt.Printf("All thread over, %d threads cost time:%v\n", threads, dlt) +} -- Gitee