diff --git a/0001-nfs_add_api_to_support_enfs_registe_and_handle_mount_option.patch b/0001-nfs_add_api_to_support_enfs_registe_and_handle_mount_option.patch new file mode 100644 index 0000000000000000000000000000000000000000..f9e18033e788e54127a0545f2b726c56db599ced --- /dev/null +++ b/0001-nfs_add_api_to_support_enfs_registe_and_handle_mount_option.patch @@ -0,0 +1,794 @@ +diff --git a/fs/nfs/client.c b/fs/nfs/client.c +index 7d02dc522..b1194a3fb 100644 +--- a/fs/nfs/client.c ++++ b/fs/nfs/client.c +@@ -48,7 +48,7 @@ + #include "callback.h" + #include "delegation.h" + #include "iostat.h" +-#include "internal.h" ++#include "enfs_adapter.h" + #include "fscache.h" + #include "pnfs.h" + #include "nfs.h" +@@ -255,6 +255,7 @@ void nfs_free_client(struct nfs_client *clp) + put_nfs_version(clp->cl_nfs_mod); + kfree(clp->cl_hostname); + kfree(clp->cl_acceptor); ++ nfs_free_multi_path_client(clp); + kfree(clp); + } + EXPORT_SYMBOL_GPL(nfs_free_client); +@@ -324,11 +325,26 @@ static struct nfs_client *nfs_match_client(const struct nfs_client_initdata *dat + continue; + /* Match the full socket address */ + if (!rpc_cmp_addr_port(sap, clap)) ++#if IS_ENABLED(CONFIG_ENFS) ++ { ++ if (data->enfs_option != NULL) ++ continue; ++ else { ++ if (IS_ERR(clp->cl_rpcclient) || ++ !rpc_clnt_xprt_switch_has_addr( ++ clp->cl_rpcclient, sap)) ++ continue; ++ } ++ } ++#else + /* Match all xprt_switch full socket addresses */ + if (IS_ERR(clp->cl_rpcclient) || + !rpc_clnt_xprt_switch_has_addr(clp->cl_rpcclient, + sap)) + continue; ++#endif ++ if (!nfs_multipath_client_match(clp, data)) ++ continue; + + refcount_inc(&clp->cl_count); + return clp; +@@ -512,6 +528,9 @@ int nfs_create_rpc_client(struct nfs_client *clp, + .program = &nfs_program, + .version = clp->rpc_ops->version, + .authflavor = flavor, ++#if IS_ENABLED(CONFIG_ENFS) ++ .multipath_option = cl_init->enfs_option, ++#endif + }; + + if (test_bit(NFS_CS_DISCRTRY, &clp->cl_flags)) +@@ -634,6 +653,13 @@ struct nfs_client *nfs_init_client(struct nfs_client *clp, + /* the client is already initialised */ + if (clp->cl_cons_state == NFS_CS_READY) + return clp; ++ error = nfs_create_multi_path_client(clp, cl_init); ++ if (error < 0) { ++ dprintk("%s: create failed.%d!\n", __func__, error); ++ nfs_put_client(clp); ++ clp = ERR_PTR(error); ++ return clp; ++ } + + /* + * Create a client RPC handle for doing FSSTAT with UNIX auth only +@@ -666,6 +692,9 @@ static int nfs_init_server(struct nfs_server *server, + .net = data->net, + .timeparms = &timeparms, + .init_flags = (1UL << NFS_CS_REUSEPORT), ++#if IS_ENABLED(CONFIG_ENFS) ++ .enfs_option = data->enfs_option, ++#endif + }; + struct nfs_client *clp; + int error; +diff --git a/fs/nfs/enfs_adapter.c b/fs/nfs/enfs_adapter.c +new file mode 100644 +index 000000000..e0f3841c1 +--- /dev/null ++++ b/fs/nfs/enfs_adapter.c +@@ -0,0 +1,276 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Client-side ENFS adapter. ++ * ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "enfs_adapter.h" ++#include "iostat.h" ++ ++struct enfs_adapter_ops __rcu *enfs_adapter; ++ ++int enfs_adapter_register(struct enfs_adapter_ops *ops) ++{ ++ struct enfs_adapter_ops *old; ++ ++ old = cmpxchg((struct enfs_adapter_ops **)&enfs_adapter, NULL, ops); ++ if (old == NULL || old == ops) ++ return 0; ++ pr_err("regist %s ops %p failed. old %p\n", __func__, ops, old); ++ return -EPERM; ++} ++EXPORT_SYMBOL_GPL(enfs_adapter_register); ++ ++int enfs_adapter_unregister(struct enfs_adapter_ops *ops) ++{ ++ struct enfs_adapter_ops *old; ++ ++ old = cmpxchg((struct enfs_adapter_ops **)&enfs_adapter, ops, NULL); ++ if (old == ops || old == NULL) ++ return 0; ++ pr_err("unregist %s ops %p failed. old %p\n", __func__, ops, old); ++ return -EPERM; ++} ++EXPORT_SYMBOL_GPL(enfs_adapter_unregister); ++ ++struct enfs_adapter_ops *nfs_multipath_router_get(void) ++{ ++ struct enfs_adapter_ops *ops; ++ ++ rcu_read_lock(); ++ ops = rcu_dereference(enfs_adapter); ++ if (ops == NULL) { ++ rcu_read_unlock(); ++ return NULL; ++ } ++ if (!try_module_get(ops->owner)) ++ ops = NULL; ++ rcu_read_unlock(); ++ return ops; ++} ++ ++void nfs_multipath_router_put(struct enfs_adapter_ops *ops) ++{ ++ if (ops) ++ module_put(ops->owner); ++} ++ ++bool is_valid_option(enum nfs_multi_path_options option) ++{ ++ if (option < REMOTEADDR || option >= INVALID_OPTION) { ++ pr_warn("%s: ENFS invalid option %d\n", __func__, option); ++ return false; ++ } ++ ++ return true; ++} ++ ++int enfs_parse_mount_options(enum nfs_multi_path_options option, char *str, ++ struct nfs_parsed_mount_data *mnt) ++{ ++ int rc; ++ struct enfs_adapter_ops *ops; ++ ++ ops = nfs_multipath_router_get(); ++ if ((ops == NULL) || (ops->parse_mount_options == NULL) || ++ !is_valid_option(option)) { ++ nfs_multipath_router_put(ops); ++ dfprintk(MOUNT, ++ "NFS: parsing nfs mount option enfs not load[%s]\n" ++ , __func__); ++ return -EOPNOTSUPP; ++ } ++ // nfs_multipath_parse_options ++ dfprintk(MOUNT, "NFS: parsing nfs mount option '%s' type: %d[%s]\n" ++ , str, option, __func__); ++ rc = ops->parse_mount_options(option, str, &mnt->enfs_option, mnt->net); ++ nfs_multipath_router_put(ops); ++ return rc; ++} ++ ++void enfs_free_mount_options(struct nfs_parsed_mount_data *data) ++{ ++ struct enfs_adapter_ops *ops; ++ ++ if (data->enfs_option == NULL) ++ return; ++ ++ ops = nfs_multipath_router_get(); ++ if ((ops == NULL) || (ops->free_mount_options == NULL)) { ++ nfs_multipath_router_put(ops); ++ return; ++ } ++ ops->free_mount_options((void *)&data->enfs_option); ++ nfs_multipath_router_put(ops); ++} ++ ++int nfs_create_multi_path_client(struct nfs_client *client, ++ const struct nfs_client_initdata *cl_init) ++{ ++ int ret = 0; ++ struct enfs_adapter_ops *ops; ++ ++ if (cl_init->enfs_option == NULL) ++ return 0; ++ ++ ops = nfs_multipath_router_get(); ++ if (ops != NULL && ops->client_info_init != NULL) ++ ret = ops->client_info_init( ++ (void *)&client->cl_multipath_data, cl_init); ++ nfs_multipath_router_put(ops); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(nfs_create_multi_path_client); ++ ++void nfs_free_multi_path_client(struct nfs_client *clp) ++{ ++ struct enfs_adapter_ops *ops; ++ ++ if (clp->cl_multipath_data == NULL) ++ return; ++ ++ ops = nfs_multipath_router_get(); ++ if (ops != NULL && ops->client_info_free != NULL) ++ ops->client_info_free(clp->cl_multipath_data); ++ nfs_multipath_router_put(ops); ++} ++ ++int nfs_multipath_client_match(struct nfs_client *clp, ++ const struct nfs_client_initdata *sap) ++{ ++ bool ret = true; ++ struct enfs_adapter_ops *ops; ++ ++ pr_info("%s src %p dst %p\n.", __func__, ++ clp->cl_multipath_data, sap->enfs_option); ++ ++ if (clp->cl_multipath_data == NULL && sap->enfs_option == NULL) ++ return true; ++ ++ if ((clp->cl_multipath_data == NULL && sap->enfs_option) || ++ (clp->cl_multipath_data && sap->enfs_option == NULL)) { ++ pr_err("not match client src %p dst %p\n.", ++ clp->cl_multipath_data, sap->enfs_option); ++ return false; ++ } ++ ++ ops = nfs_multipath_router_get(); ++ if (ops != NULL && ops->client_info_match != NULL) ++ ret = ops->client_info_match(clp->cl_multipath_data, ++ sap->enfs_option); ++ nfs_multipath_router_put(ops); ++ ++ return ret; ++} ++ ++int nfs4_multipath_client_match(struct nfs_client *src, struct nfs_client *dst) ++{ ++ int ret = true; ++ struct enfs_adapter_ops *ops; ++ ++ if (src->cl_multipath_data == NULL && dst->cl_multipath_data == NULL) ++ return true; ++ ++ if (src->cl_multipath_data == NULL || dst->cl_multipath_data == NULL) ++ return false; ++ ++ ops = nfs_multipath_router_get(); ++ if (ops != NULL && ops->nfs4_client_info_match != NULL) ++ ret = ops->nfs4_client_info_match(src->cl_multipath_data, ++ src->cl_multipath_data); ++ nfs_multipath_router_put(ops); ++ ++ return ret; ++} ++EXPORT_SYMBOL_GPL(nfs4_multipath_client_match); ++ ++void nfs_multipath_show_client_info(struct seq_file *mount_option, ++ struct nfs_server *server) ++{ ++ struct enfs_adapter_ops *ops; ++ ++ if (mount_option == NULL || server == NULL || ++ server->client == NULL || ++ server->nfs_client->cl_multipath_data == NULL) ++ return; ++ ++ ops = nfs_multipath_router_get(); ++ if (ops != NULL && ops->client_info_show != NULL) ++ ops->client_info_show(mount_option, server); ++ nfs_multipath_router_put(ops); ++} ++ ++int nfs_remount_iplist(struct nfs_client *nfs_client, void *data) ++{ ++ int ret = 0; ++ struct enfs_adapter_ops *ops; ++ struct nfs_parsed_mount_data *parsed_data = ++ (struct nfs_parsed_mount_data *)data; ++ ++ if (!parsed_data->enfs_option) ++ return 0; ++ ++ if (nfs_client == NULL || nfs_client->cl_rpcclient == NULL) ++ return 0; ++ ++ ops = nfs_multipath_router_get(); ++ if (ops != NULL && ops->remount_ip_list != NULL) ++ ret = ops->remount_ip_list(nfs_client, ++ parsed_data->enfs_option); ++ nfs_multipath_router_put(ops); ++ return ret; ++} ++EXPORT_SYMBOL_GPL(nfs_remount_iplist); ++ ++/* ++ * Error-check and convert a string of mount options from ++ * user space into a data structure. The whole mount string ++ * is processed; bad options are skipped as they are encountered. ++ * If there were no errors, return 1; otherwise return zero(0) ++ */ ++int enfs_check_mount_parse_info(char *p, int token, ++ struct nfs_parsed_mount_data *mnt, const substring_t *args) ++{ ++ char *string; ++ int rc; ++ ++ string = match_strdup(args); ++ if (string == NULL) { ++ pr_info("NFS: not enough memory to parse option\n"); ++ return 0; ++ } ++ rc = enfs_parse_mount_options(get_nfs_multi_path_opt(token), ++ string, mnt); ++ ++ kfree(string); ++ switch (rc) { ++ case 0: ++ return 1; ++ case -ENOMEM: ++ pr_info("NFS: not enough memory to parse option\n"); ++ return 0; ++ case -ENOSPC: ++ pr_info("NFS: param is more than supported limit: %d\n", rc); ++ return 0; ++ case -EINVAL: ++ pr_info("NFS: bad IP address specified: %s\n", p); ++ return 0; ++ case -ENOTSUPP: ++ pr_info("NFS: bad IP address specified: %s\n", p); ++ return 0; ++ case -EOPNOTSUPP: ++ pr_info("NFS: bad IP address specified: %s\n", p); ++ return 0; ++ } ++ return 1; ++} +diff --git a/fs/nfs/enfs_adapter.h b/fs/nfs/enfs_adapter.h +new file mode 100644 +index 000000000..98b4e3292 +--- /dev/null ++++ b/fs/nfs/enfs_adapter.h +@@ -0,0 +1,116 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Client-side ENFS adapt header. ++ * ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#ifndef _NFS_MULTIPATH_H_ ++#define _NFS_MULTIPATH_H_ ++ ++#include ++#include "internal.h" ++ ++#if IS_ENABLED(CONFIG_ENFS) ++enum nfs_multi_path_options { ++ REMOTEADDR, ++ LOCALADDR, ++ REMOTEDNSNAME, ++ REMOUNTREMOTEADDR, ++ REMOUNTLOCALADDR, ++ INVALID_OPTION ++}; ++ ++ ++struct enfs_adapter_ops { ++ const char *name; ++ struct module *owner; ++ int (*parse_mount_options)(enum nfs_multi_path_options option, ++ char *str, void **enfs_option, struct net *net_ns); ++ ++ void (*free_mount_options)(void **data); ++ ++ int (*client_info_init)(void **data, ++ const struct nfs_client_initdata *cl_init); ++ void (*client_info_free)(void *data); ++ int (*client_info_match)(void *src, void *dst); ++ int (*nfs4_client_info_match)(void *src, void *dst); ++ void (*client_info_show)(struct seq_file *mount_option, void *data); ++ int (*remount_ip_list)(struct nfs_client *nfs_client, ++ void *enfs_option); ++}; ++ ++int enfs_parse_mount_options(enum nfs_multi_path_options option, char *str, ++ struct nfs_parsed_mount_data *mnt); ++void enfs_free_mount_options(struct nfs_parsed_mount_data *data); ++int nfs_create_multi_path_client(struct nfs_client *client, ++ const struct nfs_client_initdata *cl_init); ++void nfs_free_multi_path_client(struct nfs_client *clp); ++int nfs_multipath_client_match(struct nfs_client *clp, ++ const struct nfs_client_initdata *sap); ++int nfs4_multipath_client_match(struct nfs_client *src, struct nfs_client *dst); ++void nfs_multipath_show_client_info(struct seq_file *mount_option, ++ struct nfs_server *server); ++int enfs_adapter_register(struct enfs_adapter_ops *ops); ++int enfs_adapter_unregister(struct enfs_adapter_ops *ops); ++int nfs_remount_iplist(struct nfs_client *nfs_client, void *data); ++int nfs4_create_multi_path(struct nfs_server *server, ++ struct nfs_parsed_mount_data *data, ++ const struct rpc_timeout *timeparms); ++int enfs_check_mount_parse_info(char *p, int token, ++ struct nfs_parsed_mount_data *mnt, const substring_t *args); ++ ++#else ++static inline ++void nfs_free_multi_path_client(struct nfs_client *clp) ++{ ++ ++} ++ ++static inline ++int nfs_multipath_client_match(struct nfs_client *clp, ++ const struct nfs_client_initdata *sap) ++{ ++ return 1; ++} ++ ++static inline ++int nfs_create_multi_path_client(struct nfs_client *client, ++ const struct nfs_client_initdata *cl_init) ++{ ++ return 0; ++} ++ ++static inline ++void nfs_multipath_show_client_info(struct seq_file *mount_option, ++ struct nfs_server *server) ++{ ++ ++} ++ ++static inline ++int nfs4_multipath_client_match(struct nfs_client *src, ++ struct nfs_client *dst) ++{ ++ return 1; ++} ++ ++static inline ++void enfs_free_mount_options(struct nfs_parsed_mount_data *data) ++{ ++ ++} ++ ++static inline ++int enfs_check_mount_parse_info(char *p, int token, ++ struct nfs_parsed_mount_data *mnt, const substring_t *args) ++{ ++ return 1; ++} ++ ++static inline ++int nfs_remount_iplist(struct nfs_client *nfs_client, void *data) ++{ ++ return 0; ++} ++#endif // CONFIG_ENFS ++#endif // _NFS_MULTIPATH_H_ +diff --git a/fs/nfs/internal.h b/fs/nfs/internal.h +index 0ce5a9064..84ac82dbb 100644 +--- a/fs/nfs/internal.h ++++ b/fs/nfs/internal.h +@@ -93,6 +93,9 @@ struct nfs_client_initdata { + u32 minorversion; + struct net *net; + const struct rpc_timeout *timeparms; ++#if IS_ENABLED(CONFIG_ENFS) ++ void *enfs_option; /* struct multipath_mount_options * */ ++#endif + }; + + /* +@@ -135,6 +138,9 @@ struct nfs_parsed_mount_data { + + struct security_mnt_opts lsm_opts; + struct net *net; ++#if IS_ENABLED(CONFIG_ENFS) ++ void *enfs_option; /* struct multipath_mount_options * */ ++#endif + }; + + /* mount_clnt.c */ +@@ -430,6 +436,10 @@ extern void nfs_sb_deactive(struct super_block *sb); + extern int nfs_client_for_each_server(struct nfs_client *clp, + int (*fn)(struct nfs_server *, void *), + void *data); ++#if IS_ENABLED(CONFIG_ENFS) ++extern enum nfs_multi_path_options get_nfs_multi_path_opt(int token); ++#endif ++ + /* io.c */ + extern void nfs_start_io_read(struct inode *inode); + extern void nfs_end_io_read(struct inode *inode); +diff --git a/fs/nfs/nfs4client.c b/fs/nfs/nfs4client.c +index 1350ea673..f97646b98 100644 +--- a/fs/nfs/nfs4client.c ++++ b/fs/nfs/nfs4client.c +@@ -10,7 +10,7 @@ + #include + #include + #include +-#include "internal.h" ++#include "enfs_adapter.h" + #include "callback.h" + #include "delegation.h" + #include "nfs4session.h" +@@ -225,6 +225,14 @@ struct nfs_client *nfs4_alloc_client(const struct nfs_client_initdata *cl_init) + __set_bit(NFS_CS_DISCRTRY, &clp->cl_flags); + __set_bit(NFS_CS_NO_RETRANS_TIMEOUT, &clp->cl_flags); + ++ err = nfs_create_multi_path_client(clp, cl_init); ++ if (err < 0) { ++ dprintk("%s: create failed.%d\n", __func__, err); ++ nfs_put_client(clp); ++ clp = ERR_PTR(err); ++ return clp; ++ } ++ + /* + * Set up the connection to the server before we add add to the + * global list. +@@ -529,6 +537,9 @@ static int nfs4_match_client(struct nfs_client *pos, struct nfs_client *new, + if (!nfs4_match_client_owner_id(pos, new)) + return 1; + ++ if (!nfs4_multipath_client_match(pos, new)) ++ return 1; ++ + return 0; + } + +@@ -860,7 +871,7 @@ static int nfs4_set_client(struct nfs_server *server, + const size_t addrlen, + const char *ip_addr, + int proto, const struct rpc_timeout *timeparms, +- u32 minorversion, struct net *net) ++ u32 minorversion, struct net *net, void *enfs_option) + { + struct nfs_client_initdata cl_init = { + .hostname = hostname, +@@ -872,6 +883,9 @@ static int nfs4_set_client(struct nfs_server *server, + .minorversion = minorversion, + .net = net, + .timeparms = timeparms, ++#if IS_ENABLED(CONFIG_ENFS) ++ .enfs_option = enfs_option, ++#endif + }; + struct nfs_client *clp; + +@@ -1050,6 +1064,7 @@ static int nfs4_init_server(struct nfs_server *server, + { + struct rpc_timeout timeparms; + int error; ++ void *enfs_option = NULL; + + nfs_init_timeout_values(&timeparms, data->nfs_server.protocol, + data->timeo, data->retrans); +@@ -1067,6 +1082,10 @@ static int nfs4_init_server(struct nfs_server *server, + else + data->selected_flavor = RPC_AUTH_UNIX; + ++#if IS_ENABLED(CONFIG_ENFS) ++ enfs_option = data->enfs_option; ++#endif ++ + /* Get a client record */ + error = nfs4_set_client(server, + data->nfs_server.hostname, +@@ -1076,7 +1095,7 @@ static int nfs4_init_server(struct nfs_server *server, + data->nfs_server.protocol, + &timeparms, + data->minorversion, +- data->net); ++ data->net, enfs_option); + if (error < 0) + return error; + +@@ -1161,7 +1180,7 @@ struct nfs_server *nfs4_create_referral_server(struct nfs_clone_mount *data, + XPRT_TRANSPORT_RDMA, + parent_server->client->cl_timeout, + parent_client->cl_mvops->minor_version, +- parent_client->cl_net); ++ parent_client->cl_net, NULL); + if (!error) + goto init_server; + #endif /* IS_ENABLED(CONFIG_SUNRPC_XPRT_RDMA) */ +@@ -1174,7 +1193,7 @@ struct nfs_server *nfs4_create_referral_server(struct nfs_clone_mount *data, + XPRT_TRANSPORT_TCP, + parent_server->client->cl_timeout, + parent_client->cl_mvops->minor_version, +- parent_client->cl_net); ++ parent_client->cl_net, NULL); + if (error < 0) + goto error; + +@@ -1269,7 +1288,7 @@ int nfs4_update_server(struct nfs_server *server, const char *hostname, + set_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status); + error = nfs4_set_client(server, hostname, sap, salen, buf, + clp->cl_proto, clnt->cl_timeout, +- clp->cl_minorversion, net); ++ clp->cl_minorversion, net, NULL); + clear_bit(NFS_MIG_TSM_POSSIBLE, &server->mig_status); + if (error != 0) { + nfs_server_insert_lists(server); +diff --git a/fs/nfs/super.c b/fs/nfs/super.c +index a05e1eb2c..168da9700 100644 +--- a/fs/nfs/super.c ++++ b/fs/nfs/super.c +@@ -51,7 +51,6 @@ + #include + #include + #include +-#include + #include + #include + +@@ -61,7 +60,7 @@ + #include "callback.h" + #include "delegation.h" + #include "iostat.h" +-#include "internal.h" ++#include "enfs_adapter.h" + #include "fscache.h" + #include "nfs4session.h" + #include "pnfs.h" +@@ -113,6 +112,12 @@ enum { + + /* Special mount options */ + Opt_userspace, Opt_deprecated, Opt_sloppy, ++#if IS_ENABLED(CONFIG_ENFS) ++ Opt_remote_iplist, ++ Opt_local_iplist, ++ Opt_remote_dnslist, ++ Opt_enfs_info, ++#endif + + Opt_err + }; +@@ -183,6 +188,13 @@ static const match_table_t nfs_mount_option_tokens = { + { Opt_fscache_uniq, "fsc=%s" }, + { Opt_local_lock, "local_lock=%s" }, + ++#if IS_ENABLED(CONFIG_ENFS) ++ { Opt_remote_iplist, "remoteaddrs=%s" }, ++ { Opt_local_iplist, "localaddrs=%s" }, ++ { Opt_remote_dnslist, "remotednsname=%s" }, ++ { Opt_enfs_info, "enfs_info=%s" }, ++#endif ++ + /* The following needs to be listed after all other options */ + { Opt_nfsvers, "v%s" }, + +@@ -365,6 +377,21 @@ static struct shrinker acl_shrinker = { + .seeks = DEFAULT_SEEKS, + }; + ++#if IS_ENABLED(CONFIG_ENFS) ++enum nfs_multi_path_options get_nfs_multi_path_opt(int token) ++{ ++ switch (token) { ++ case Opt_remote_iplist: ++ return REMOUNTREMOTEADDR; ++ case Opt_local_iplist: ++ return REMOUNTLOCALADDR; ++ case Opt_remote_dnslist: ++ return REMOTEDNSNAME; ++ } ++ return INVALID_OPTION; ++} ++#endif ++ + /* + * Register the NFS filesystems + */ +@@ -758,6 +785,9 @@ int nfs_show_options(struct seq_file *m, struct dentry *root) + seq_printf(m, ",addr=%s", + rpc_peeraddr2str(nfss->nfs_client->cl_rpcclient, + RPC_DISPLAY_ADDR)); ++ ++ nfs_multipath_show_client_info(m, nfss); ++ + rcu_read_unlock(); + + return 0; +@@ -853,6 +883,8 @@ int nfs_show_stats(struct seq_file *m, struct dentry *root) + seq_puts(m, root->d_sb->s_flags & SB_NODIRATIME ? ",nodiratime" : ""); + nfs_show_mount_options(m, nfss, 1); + ++ nfs_multipath_show_client_info(m, nfss); ++ + seq_printf(m, "\n\tage:\t%lu", (jiffies - nfss->mount_time) / HZ); + + show_implementation_id(m, nfss); +@@ -977,6 +1009,7 @@ static void nfs_free_parsed_mount_data(struct nfs_parsed_mount_data *data) + kfree(data->nfs_server.export_path); + kfree(data->nfs_server.hostname); + kfree(data->fscache_uniq); ++ enfs_free_mount_options(data); + security_free_mnt_opts(&data->lsm_opts); + kfree(data); + } +@@ -1641,7 +1674,6 @@ static int nfs_parse_mount_options(char *raw, + return 0; + }; + break; +- + /* + * Special options + */ +@@ -1654,7 +1686,18 @@ static int nfs_parse_mount_options(char *raw, + dfprintk(MOUNT, "NFS: ignoring mount option " + "'%s'\n", p); + break; +- ++#if IS_ENABLED(CONFIG_ENFS) ++ case Opt_remote_iplist: ++ case Opt_local_iplist: ++ case Opt_remote_dnslist: ++ rc = enfs_check_mount_parse_info(p, ++ token, mnt, args); ++ if (rc != 1) ++ return rc; ++ break; ++ case Opt_enfs_info: ++ break; ++#endif + default: + invalid_option = 1; + dfprintk(MOUNT, "NFS: unrecognized mount option " +@@ -2335,6 +2378,10 @@ nfs_remount(struct super_block *sb, int *flags, char *raw_data) + if (!nfs_parse_mount_options((char *)options, data)) + goto out; + ++ error = nfs_remount_iplist(nfss->nfs_client, data); ++ if (error) ++ goto out; ++ + /* + * noac is a special case. It implies -o sync, but that's not + * necessarily reflected in the mtab options. do_remount_sb +@@ -2347,6 +2394,8 @@ nfs_remount(struct super_block *sb, int *flags, char *raw_data) + /* compare new mount options with old ones */ + error = nfs_compare_remount_data(nfss, data); + out: ++ /* release remount option member */ ++ enfs_free_mount_options(data); + nfs_free_parsed_mount_data(data); + return error; + } +diff --git a/include/linux/nfs_fs_sb.h b/include/linux/nfs_fs_sb.h +index 7023ae64e..2c19678af 100644 +--- a/include/linux/nfs_fs_sb.h ++++ b/include/linux/nfs_fs_sb.h +@@ -123,6 +123,11 @@ struct nfs_client { + + struct net *cl_net; + struct list_head pending_cb_stateids; ++ ++#if IS_ENABLED(CONFIG_ENFS) ++ /* multi path private structure (struct multipath_client_info *) */ ++ void *cl_multipath_data; ++#endif + }; + + /* diff --git a/0002-sunrpc_add_api_to_support_enfs_registe_and_create_multipath_then_dispatch_IO.patch b/0002-sunrpc_add_api_to_support_enfs_registe_and_create_multipath_then_dispatch_IO.patch new file mode 100644 index 0000000000000000000000000000000000000000..98af7d8538887b82d73dd526dd173a0ed045c090 --- /dev/null +++ b/0002-sunrpc_add_api_to_support_enfs_registe_and_create_multipath_then_dispatch_IO.patch @@ -0,0 +1,832 @@ +diff --git a/include/linux/sunrpc/clnt.h b/include/linux/sunrpc/clnt.h +index 8aa865bce..0425aaabf 100644 +--- a/include/linux/sunrpc/clnt.h ++++ b/include/linux/sunrpc/clnt.h +@@ -70,6 +70,10 @@ struct rpc_clnt { + struct dentry *cl_debugfs; /* debugfs directory */ + #endif + struct rpc_xprt_iter cl_xpi; ++ ++#if IS_ENABLED(CONFIG_ENFS) ++ bool cl_enfs; ++#endif + }; + + /* +@@ -124,6 +128,9 @@ struct rpc_create_args { + unsigned long flags; + char *client_name; + struct svc_xprt *bc_xprt; /* NFSv4.1 backchannel */ ++#if IS_ENABLED(CONFIG_ENFS) ++ void *multipath_option; ++#endif + }; + + struct rpc_add_xprt_test { +@@ -221,6 +228,13 @@ bool rpc_clnt_xprt_switch_has_addr(struct rpc_clnt *clnt, + const struct sockaddr *sap); + void rpc_cleanup_clids(void); + ++#if IS_ENABLED(CONFIG_ENFS) ++int ++rpc_clnt_test_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt, ++ const struct rpc_call_ops *ops, void *data, int flags); ++struct rpc_xprt *rpc_task_get_next_xprt(struct rpc_clnt *clnt); ++#endif /* CONFIG_ENFS */ ++ + static inline int rpc_reply_expected(struct rpc_task *task) + { + return (task->tk_msg.rpc_proc != NULL) && +diff --git a/include/linux/sunrpc/sched.h b/include/linux/sunrpc/sched.h +index ad2e243f3..124f5a0fa 100644 +--- a/include/linux/sunrpc/sched.h ++++ b/include/linux/sunrpc/sched.h +@@ -90,6 +90,9 @@ struct rpc_task { + tk_garb_retry : 2, + tk_cred_retry : 2, + tk_rebind_retry : 2; ++#if IS_ENABLED(CONFIG_ENFS) ++ unsigned long tk_major_timeo; /* major timeout ticks */ ++#endif + }; + + typedef void (*rpc_action)(struct rpc_task *); +@@ -118,6 +121,9 @@ struct rpc_task_setup { + */ + #define RPC_TASK_ASYNC 0x0001 /* is an async task */ + #define RPC_TASK_SWAPPER 0x0002 /* is swapping in/out */ ++#if IS_ENABLED(CONFIG_ENFS) ++#define RPC_TASK_FIXED 0x0004 /* detect xprt status task */ ++#endif + #define RPC_CALL_MAJORSEEN 0x0020 /* major timeout seen */ + #define RPC_TASK_ROOTCREDS 0x0040 /* force root creds */ + #define RPC_TASK_DYNAMIC 0x0080 /* task was kmalloc'ed */ +@@ -257,6 +263,9 @@ void rpc_destroy_mempool(void); + extern struct workqueue_struct *rpciod_workqueue; + extern struct workqueue_struct *xprtiod_workqueue; + void rpc_prepare_task(struct rpc_task *task); ++#if IS_ENABLED(CONFIG_ENFS) ++void rpc_init_task_retry_counters(struct rpc_task *task); ++#endif + + static inline int rpc_wait_for_completion_task(struct rpc_task *task) + { +diff --git a/include/linux/sunrpc/sunrpc_enfs_adapter.h b/include/linux/sunrpc/sunrpc_enfs_adapter.h +new file mode 100644 +index 000000000..cdd7fa699 +--- /dev/null ++++ b/include/linux/sunrpc/sunrpc_enfs_adapter.h +@@ -0,0 +1,135 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* Client-side SUNRPC ENFS adapter header. ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#ifndef _SUNRPC_ENFS_ADAPTER_H_ ++#define _SUNRPC_ENFS_ADAPTER_H_ ++#include ++ ++#if IS_ENABLED(CONFIG_ENFS) ++ ++static inline void rpc_xps_nactive_add_one(struct rpc_xprt_switch *xps) ++{ ++ xps->xps_nactive--; ++} ++ ++static inline void rpc_xps_nactive_sub_one(struct rpc_xprt_switch *xps) ++{ ++ xps->xps_nactive--; ++} ++ ++struct rpc_xprt *rpc_task_get_xprt ++(struct rpc_clnt *clnt, struct rpc_xprt *xprt); ++ ++struct rpc_multipath_ops { ++ struct module *owner; ++ void (*create_clnt)(struct rpc_create_args *args, ++ struct rpc_clnt *clnt); ++ void (*releas_clnt)(struct rpc_clnt *clnt); ++ void (*create_xprt)(struct rpc_xprt *xprt); ++ void (*destroy_xprt)(struct rpc_xprt *xprt); ++ void (*xprt_iostat)(struct rpc_task *task); ++ void (*failover_handle)(struct rpc_task *task); ++ bool (*task_need_call_start_again)(struct rpc_task *task); ++ void (*adjust_task_timeout)(struct rpc_task *task, void *condition); ++ void (*init_task_req)(struct rpc_task *task, struct rpc_rqst *req); ++ bool (*prepare_transmit)(struct rpc_task *task); ++ void (*set_transport)(struct rpc_task *task, struct rpc_clnt *clnt); ++}; ++ ++extern struct rpc_multipath_ops __rcu *multipath_ops; ++void rpc_init_task_retry_counters(struct rpc_task *task); ++int rpc_multipath_ops_register(struct rpc_multipath_ops *ops); ++int rpc_multipath_ops_unregister(struct rpc_multipath_ops *ops); ++struct rpc_multipath_ops *rpc_multipath_ops_get(void); ++void rpc_multipath_ops_put(struct rpc_multipath_ops *ops); ++void rpc_task_release_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt); ++void rpc_multipath_ops_create_clnt(struct rpc_create_args *args, ++ struct rpc_clnt *clnt); ++void rpc_multipath_ops_releas_clnt(struct rpc_clnt *clnt); ++bool rpc_multipath_ops_create_xprt(struct rpc_xprt *xprt); ++void rpc_multipath_ops_destroy_xprt(struct rpc_xprt *xprt); ++void rpc_multipath_ops_xprt_iostat(struct rpc_task *task); ++void rpc_multipath_ops_failover_handle(struct rpc_task *task); ++bool rpc_multipath_ops_task_need_call_start_again(struct rpc_task *task); ++void rpc_multipath_ops_adjust_task_timeout(struct rpc_task *task, ++ void *condition); ++void rpc_multipath_ops_init_task_req(struct rpc_task *task, ++ struct rpc_rqst *req); ++bool rpc_multipath_ops_prepare_transmit(struct rpc_task *task); ++void rpc_multipath_ops_set_transport(struct rpc_task *task, ++ struct rpc_clnt *clnt); ++#else ++static inline struct rpc_xprt *rpc_task_get_xprt(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt) ++{ ++ return NULL; ++} ++ ++static inline void rpc_task_release_xprt(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt) ++{ ++} ++ ++static inline void rpc_xps_nactive_add_one(struct rpc_xprt_switch *xps) ++{ ++} ++ ++static inline void rpc_xps_nactive_sub_one(struct rpc_xprt_switch *xps) ++{ ++} ++ ++static inline void rpc_multipath_ops_create_clnt ++(struct rpc_create_args *args, struct rpc_clnt *clnt) ++{ ++} ++ ++static inline void rpc_multipath_ops_releas_clnt(struct rpc_clnt *clnt) ++{ ++} ++ ++static inline bool rpc_multipath_ops_create_xprt(struct rpc_xprt *xprt) ++{ ++ return false; ++} ++ ++static inline void rpc_multipath_ops_destroy_xprt(struct rpc_xprt *xprt) ++{ ++} ++ ++static inline void rpc_multipath_ops_xprt_iostat(struct rpc_task *task) ++{ ++} ++ ++static inline void rpc_multipath_ops_failover_handle(struct rpc_task *task) ++{ ++} ++ ++static inline ++bool rpc_multipath_ops_task_need_call_start_again(struct rpc_task *task) ++{ ++ return false; ++} ++ ++static inline void ++rpc_multipath_ops_adjust_task_timeout(struct rpc_task *task, void *condition) ++{ ++} ++ ++static inline void ++rpc_multipath_ops_init_task_req(struct rpc_task *task, struct rpc_rqst *req) ++{ ++} ++ ++static inline bool rpc_multipath_ops_prepare_transmit(struct rpc_task *task) ++{ ++ return false; ++} ++ ++static inline void ++rpc_multipath_ops_set_transport(struct rpc_task *task, struct rpc_clnt *clnt) ++{ ++} ++ ++#endif ++#endif // _SUNRPC_ENFS_ADAPTER_H_ +diff --git a/include/linux/sunrpc/xprt.h b/include/linux/sunrpc/xprt.h +index ccfacca1e..2e47b3577 100644 +--- a/include/linux/sunrpc/xprt.h ++++ b/include/linux/sunrpc/xprt.h +@@ -279,6 +279,10 @@ struct rpc_xprt { + atomic_t inject_disconnect; + #endif + struct rcu_head rcu; ++#if IS_ENABLED(CONFIG_ENFS) ++ atomic_long_t queuelen; ++ void *multipath_context; ++#endif + }; + + #if defined(CONFIG_SUNRPC_BACKCHANNEL) +diff --git a/include/linux/sunrpc/xprtmultipath.h b/include/linux/sunrpc/xprtmultipath.h +index af1257c03..5677a46ee 100644 +--- a/include/linux/sunrpc/xprtmultipath.h ++++ b/include/linux/sunrpc/xprtmultipath.h +@@ -22,6 +22,11 @@ struct rpc_xprt_switch { + const struct rpc_xprt_iter_ops *xps_iter_ops; + + struct rcu_head xps_rcu; ++#if IS_ENABLED(CONFIG_ENFS) ++ unsigned int xps_nactive; ++ atomic_long_t xps_queuelen; ++ unsigned long xps_tmp_time; ++#endif + }; + + struct rpc_xprt_iter { +@@ -69,4 +74,8 @@ extern struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi); + + extern bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps, + const struct sockaddr *sap); ++#if IS_ENABLED(CONFIG_ENFS) ++extern void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps, ++ struct rpc_xprt *xprt); ++#endif + #endif +diff --git a/net/sunrpc/clnt.c b/net/sunrpc/clnt.c +index 0fc540b0d..ca10bc61a 100644 +--- a/net/sunrpc/clnt.c ++++ b/net/sunrpc/clnt.c +@@ -37,6 +37,7 @@ + #include + #include + #include ++#include + #include + + #include "sunrpc.h" +@@ -490,6 +491,8 @@ static struct rpc_clnt *rpc_create_xprt(struct rpc_create_args *args, + } + } + ++ rpc_multipath_ops_create_clnt(args, clnt); ++ + clnt->cl_softrtry = 1; + if (args->flags & RPC_CLNT_CREATE_HARDRTRY) + clnt->cl_softrtry = 0; +@@ -869,6 +872,8 @@ void rpc_shutdown_client(struct rpc_clnt *clnt) + list_empty(&clnt->cl_tasks), 1*HZ); + } + ++ rpc_multipath_ops_releas_clnt(clnt); ++ + rpc_release_client(clnt); + } + EXPORT_SYMBOL_GPL(rpc_shutdown_client); +@@ -981,6 +986,12 @@ void rpc_task_release_transport(struct rpc_task *task) + + if (xprt) { + task->tk_xprt = NULL; ++#if IS_ENABLED(CONFIG_ENFS) ++ if (task->tk_client) { ++ rpc_task_release_xprt(task->tk_client, xprt); ++ return; ++ } ++#endif + xprt_put(xprt); + } + } +@@ -990,6 +1001,10 @@ void rpc_task_release_client(struct rpc_task *task) + { + struct rpc_clnt *clnt = task->tk_client; + ++#if IS_ENABLED(CONFIG_ENFS) ++ rpc_task_release_transport(task); ++#endif ++ + if (clnt != NULL) { + /* Remove from client task list */ + spin_lock(&clnt->cl_lock); +@@ -999,14 +1014,31 @@ void rpc_task_release_client(struct rpc_task *task) + + rpc_release_client(clnt); + } +- rpc_task_release_transport(task); ++ ++ if (!IS_ENABLED(CONFIG_ENFS)) ++ rpc_task_release_transport(task); ++ + } + ++#if IS_ENABLED(CONFIG_ENFS) ++struct rpc_xprt * ++rpc_task_get_next_xprt(struct rpc_clnt *clnt) ++{ ++ return rpc_task_get_xprt(clnt, xprt_iter_get_next(&clnt->cl_xpi)); ++} ++EXPORT_SYMBOL_GPL(rpc_task_get_next_xprt); ++#endif ++ + static + void rpc_task_set_transport(struct rpc_task *task, struct rpc_clnt *clnt) + { ++ rpc_multipath_ops_set_transport(task, clnt); + if (!task->tk_xprt) ++#if IS_ENABLED(CONFIG_ENFS) ++ task->tk_xprt = rpc_task_get_next_xprt(clnt); ++#else + task->tk_xprt = xprt_iter_get_next(&clnt->cl_xpi); ++#endif + } + + static +@@ -1597,6 +1629,14 @@ call_reserveresult(struct rpc_task *task) + return; + case -EIO: /* probably a shutdown */ + break; ++#if IS_ENABLED(CONFIG_ENFS) ++ case -ETIMEDOUT: /* woken up; restart */ ++ if (rpc_multipath_ops_task_need_call_start_again(task)) { ++ rpc_task_release_transport(task); ++ task->tk_action = call_start; ++ return; ++ } ++#endif + default: + printk(KERN_ERR "%s: unrecognized error %d, exiting\n", + __func__, status); +@@ -1962,6 +2002,10 @@ call_transmit(struct rpc_task *task) + return; + if (!xprt_prepare_transmit(task)) + return; ++ ++ if (rpc_multipath_ops_prepare_transmit(task)) ++ return; ++ + task->tk_action = call_transmit_status; + /* Encode here so that rpcsec_gss can use correct sequence number. */ + if (rpc_task_need_encode(task)) { +@@ -2277,6 +2321,7 @@ call_timeout(struct rpc_task *task) + + retry: + task->tk_action = call_bind; ++ rpc_multipath_ops_failover_handle(task); + task->tk_status = 0; + } + +@@ -2798,8 +2843,8 @@ int rpc_clnt_add_xprt(struct rpc_clnt *clnt, + xprt->ops->set_connect_timeout(xprt, + connect_timeout, + reconnect_timeout); +- +- rpc_xprt_switch_set_roundrobin(xps); ++ if (!clnt->cl_enfs) ++ rpc_xprt_switch_set_roundrobin(xps); + if (setup) { + ret = setup(clnt, xps, xprt, data); + if (ret != 0) +@@ -2961,3 +3006,30 @@ rpc_clnt_swap_deactivate(struct rpc_clnt *clnt) + } + EXPORT_SYMBOL_GPL(rpc_clnt_swap_deactivate); + #endif /* CONFIG_SUNRPC_SWAP */ ++ ++#if IS_ENABLED(CONFIG_ENFS) ++/* rpc_clnt_test_xprt - Test and add a new transport to a rpc_clnt ++ * @clnt: pointer to struct rpc_clnt ++ * @xprt: pointer struct rpc_xprt ++ * @ops: async operation ++ */ ++int ++rpc_clnt_test_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt, ++ const struct rpc_call_ops *ops, void *data, int flags) ++{ ++ struct rpc_cred *cred; ++ struct rpc_task *task; ++ ++ cred = authnull_ops.lookup_cred(NULL, NULL, 0); ++ task = rpc_call_null_helper(clnt, xprt, cred, ++ RPC_TASK_SOFT | RPC_TASK_SOFTCONN | flags, ++ ops, data); ++ put_rpccred(cred); ++ if (IS_ERR(task)) ++ return PTR_ERR(task); ++ ++ rpc_put_task(task); ++ return 1; ++} ++EXPORT_SYMBOL_GPL(rpc_clnt_test_xprt); ++#endif +diff --git a/net/sunrpc/sched.c b/net/sunrpc/sched.c +index a873c92a4..2254fea0e 100644 +--- a/net/sunrpc/sched.c ++++ b/net/sunrpc/sched.c +@@ -20,7 +20,7 @@ + #include + #include + +-#include ++#include + + #include "sunrpc.h" + +@@ -962,7 +962,12 @@ static void rpc_init_task(struct rpc_task *task, const struct rpc_task_setup *ta + /* Initialize workqueue for async tasks */ + task->tk_workqueue = task_setup_data->workqueue; + ++#if IS_ENABLED(CONFIG_ENFS) ++ task->tk_xprt = rpc_task_get_xprt(task_setup_data->rpc_client, ++ xprt_get(task_setup_data->rpc_xprt)); ++#else + task->tk_xprt = xprt_get(task_setup_data->rpc_xprt); ++#endif + + if (task->tk_ops->rpc_call_prepare != NULL) + task->tk_action = rpc_prepare_task; +diff --git a/net/sunrpc/sunrpc_enfs_adapter.c b/net/sunrpc/sunrpc_enfs_adapter.c +new file mode 100644 +index 000000000..106ad7309 +--- /dev/null ++++ b/net/sunrpc/sunrpc_enfs_adapter.c +@@ -0,0 +1,225 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* Client-side SUNRPC ENFS adapter header. ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#include ++ ++struct rpc_multipath_ops __rcu *multipath_ops; ++ ++void rpc_init_task_retry_counters(struct rpc_task *task) ++{ ++ /* Initialize retry counters */ ++ task->tk_garb_retry = 2; ++ task->tk_cred_retry = 2; ++ task->tk_rebind_retry = 2; ++} ++EXPORT_SYMBOL_GPL(rpc_init_task_retry_counters); ++ ++struct rpc_xprt * ++rpc_task_get_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt) ++{ ++ struct rpc_xprt_switch *xps; ++ ++ if (!xprt) ++ return NULL; ++ rcu_read_lock(); ++ xps = rcu_dereference(clnt->cl_xpi.xpi_xpswitch); ++ atomic_long_inc(&xps->xps_queuelen); ++ rcu_read_unlock(); ++ atomic_long_inc(&xprt->queuelen); ++ ++ return xprt; ++} ++ ++int rpc_multipath_ops_register(struct rpc_multipath_ops *ops) ++{ ++ struct rpc_multipath_ops *old; ++ ++ old = cmpxchg((struct rpc_multipath_ops **)&multipath_ops, NULL, ops); ++ if (!old || old == ops) ++ return 0; ++ pr_err("regist rpc_multipath ops %p fail. old %p\n", ops, old); ++ return -EPERM; ++} ++EXPORT_SYMBOL_GPL(rpc_multipath_ops_register); ++ ++int rpc_multipath_ops_unregister(struct rpc_multipath_ops *ops) ++{ ++ struct rpc_multipath_ops *old; ++ ++ old = cmpxchg((struct rpc_multipath_ops **)&multipath_ops, ops, NULL); ++ if (!old || old == ops) ++ return 0; ++ pr_err("regist rpc_multipath ops %p fail. old %p\n", ops, old); ++ return -EPERM; ++} ++EXPORT_SYMBOL_GPL(rpc_multipath_ops_unregister); ++ ++struct rpc_multipath_ops *rpc_multipath_ops_get(void) ++{ ++ struct rpc_multipath_ops *ops; ++ ++ rcu_read_lock(); ++ ops = rcu_dereference(multipath_ops); ++ if (!ops) { ++ rcu_read_unlock(); ++ return NULL; ++ } ++ if (!try_module_get(ops->owner)) ++ ops = NULL; ++ rcu_read_unlock(); ++ return ops; ++} ++EXPORT_SYMBOL_GPL(rpc_multipath_ops_get); ++ ++void rpc_multipath_ops_put(struct rpc_multipath_ops *ops) ++{ ++ if (ops) ++ module_put(ops->owner); ++} ++EXPORT_SYMBOL_GPL(rpc_multipath_ops_put); ++ ++void rpc_task_release_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt) ++{ ++ struct rpc_xprt_switch *xps; ++ ++ atomic_long_dec(&xprt->queuelen); ++ rcu_read_lock(); ++ xps = rcu_dereference(clnt->cl_xpi.xpi_xpswitch); ++ atomic_long_dec(&xps->xps_queuelen); ++ rcu_read_unlock(); ++ ++ xprt_put(xprt); ++} ++ ++void rpc_multipath_ops_create_clnt(struct rpc_create_args *args, ++ struct rpc_clnt *clnt) ++{ ++ struct rpc_multipath_ops *mops; ++ ++ if (args->multipath_option) { ++ mops = rpc_multipath_ops_get(); ++ if (mops && mops->create_clnt) ++ mops->create_clnt(args, clnt); ++ rpc_multipath_ops_put(mops); ++ } ++} ++ ++void rpc_multipath_ops_releas_clnt(struct rpc_clnt *clnt) ++{ ++ struct rpc_multipath_ops *mops; ++ ++ mops = rpc_multipath_ops_get(); ++ if (mops && mops->releas_clnt) ++ mops->releas_clnt(clnt); ++ ++ rpc_multipath_ops_put(mops); ++} ++ ++bool rpc_multipath_ops_create_xprt(struct rpc_xprt *xprt) ++{ ++ struct rpc_multipath_ops *mops = NULL; ++ ++ mops = rpc_multipath_ops_get(); ++ if (mops && mops->create_xprt) { ++ mops->create_xprt(xprt); ++ if (!xprt->multipath_context) { ++ rpc_multipath_ops_put(mops); ++ return true; ++ } ++ } ++ rpc_multipath_ops_put(mops); ++ return false; ++} ++ ++void rpc_multipath_ops_destroy_xprt(struct rpc_xprt *xprt) ++{ ++ struct rpc_multipath_ops *mops; ++ ++ if (xprt->multipath_context) { ++ mops = rpc_multipath_ops_get(); ++ if (mops && mops->destroy_xprt) ++ mops->destroy_xprt(xprt); ++ rpc_multipath_ops_put(mops); ++ } ++} ++ ++void rpc_multipath_ops_xprt_iostat(struct rpc_task *task) ++{ ++ struct rpc_multipath_ops *mops; ++ ++ mops = rpc_multipath_ops_get(); ++ if (task->tk_client && mops && mops->xprt_iostat) ++ mops->xprt_iostat(task); ++ rpc_multipath_ops_put(mops); ++} ++ ++void rpc_multipath_ops_failover_handle(struct rpc_task *task) ++{ ++ struct rpc_multipath_ops *mpath_ops = NULL; ++ ++ mpath_ops = rpc_multipath_ops_get(); ++ if (mpath_ops && mpath_ops->failover_handle) ++ mpath_ops->failover_handle(task); ++ rpc_multipath_ops_put(mpath_ops); ++} ++ ++bool rpc_multipath_ops_task_need_call_start_again(struct rpc_task *task) ++{ ++ struct rpc_multipath_ops *mpath_ops = NULL; ++ bool ret = false; ++ ++ mpath_ops = rpc_multipath_ops_get(); ++ if (mpath_ops && mpath_ops->task_need_call_start_again) ++ ret = mpath_ops->task_need_call_start_again(task); ++ rpc_multipath_ops_put(mpath_ops); ++ return ret; ++} ++ ++void rpc_multipath_ops_adjust_task_timeout(struct rpc_task *task, ++ void *condition) ++{ ++ struct rpc_multipath_ops *mops = NULL; ++ ++ mops = rpc_multipath_ops_get(); ++ if (mops && mops->adjust_task_timeout) ++ mops->adjust_task_timeout(task, NULL); ++ rpc_multipath_ops_put(mops); ++} ++ ++void rpc_multipath_ops_init_task_req(struct rpc_task *task, ++ struct rpc_rqst *req) ++{ ++ struct rpc_multipath_ops *mops = NULL; ++ ++ mops = rpc_multipath_ops_get(); ++ if (mops && mops->init_task_req) ++ mops->init_task_req(task, req); ++ rpc_multipath_ops_put(mops); ++} ++ ++bool rpc_multipath_ops_prepare_transmit(struct rpc_task *task) ++{ ++ struct rpc_multipath_ops *mops = NULL; ++ ++ mops = rpc_multipath_ops_get(); ++ if (mops && mops->prepare_transmit) { ++ if (!(mops->prepare_transmit(task))) { ++ rpc_multipath_ops_put(mops); ++ return true; ++ } ++ } ++ rpc_multipath_ops_put(mops); ++ return false; ++} ++ ++void rpc_multipath_ops_set_transport(struct rpc_task *task, ++ struct rpc_clnt *clnt) ++{ ++ struct rpc_multipath_ops *mops = NULL; ++ ++ mops = rpc_multipath_ops_get(); ++ if (mops && mops->set_transport) ++ mops->set_transport(task, clnt); ++ rpc_multipath_ops_put(mops); ++} +diff --git a/net/sunrpc/xprt.c b/net/sunrpc/xprt.c +index c912bf20f..c2b63b3d5 100644 +--- a/net/sunrpc/xprt.c ++++ b/net/sunrpc/xprt.c +@@ -48,6 +48,7 @@ + #include + #include + #include ++#include + #include + + #include +@@ -259,6 +260,9 @@ int xprt_reserve_xprt(struct rpc_xprt *xprt, struct rpc_task *task) + dprintk("RPC: %5u failed to lock transport %p\n", + task->tk_pid, xprt); + task->tk_timeout = 0; ++ ++ rpc_multipath_ops_adjust_task_timeout(task, NULL); ++ + task->tk_status = -EAGAIN; + if (req == NULL) + priority = RPC_PRIORITY_LOW; +@@ -560,6 +564,9 @@ void xprt_wait_for_buffer_space(struct rpc_task *task, rpc_action action) + struct rpc_xprt *xprt = req->rq_xprt; + + task->tk_timeout = RPC_IS_SOFT(task) ? req->rq_timeout : 0; ++ ++ rpc_multipath_ops_adjust_task_timeout(task, NULL); ++ + rpc_sleep_on(&xprt->pending, task, action); + } + EXPORT_SYMBOL_GPL(xprt_wait_for_buffer_space); +@@ -1347,6 +1354,9 @@ xprt_request_init(struct rpc_task *task) + req->rq_rcv_buf.buflen = 0; + req->rq_release_snd_buf = NULL; + xprt_reset_majortimeo(req); ++ ++ rpc_multipath_ops_init_task_req(task, req); ++ + dprintk("RPC: %5u reserved req %p xid %08x\n", task->tk_pid, + req, ntohl(req->rq_xid)); + } +@@ -1427,6 +1437,9 @@ void xprt_release(struct rpc_task *task) + task->tk_ops->rpc_count_stats(task, task->tk_calldata); + else if (task->tk_client) + rpc_count_iostats(task, task->tk_client->cl_metrics); ++ ++ rpc_multipath_ops_xprt_iostat(task); ++ + spin_lock(&xprt->recv_lock); + if (!list_empty(&req->rq_list)) { + list_del_init(&req->rq_list); +@@ -1455,6 +1468,7 @@ void xprt_release(struct rpc_task *task) + else + xprt_free_bc_request(req); + } ++EXPORT_SYMBOL_GPL(xprt_release); + + static void xprt_init(struct rpc_xprt *xprt, struct net *net) + { +@@ -1528,6 +1542,10 @@ struct rpc_xprt *xprt_create_transport(struct xprt_create *args) + return ERR_PTR(-ENOMEM); + } + ++if (rpc_multipath_ops_create_xprt(xprt)) { ++ xprt_destroy(xprt); ++ return ERR_PTR(-ENOMEM); ++} + rpc_xprt_debugfs_register(xprt); + + dprintk("RPC: created transport %p with %u slots\n", xprt, +@@ -1547,6 +1565,9 @@ static void xprt_destroy_cb(struct work_struct *work) + rpc_destroy_wait_queue(&xprt->sending); + rpc_destroy_wait_queue(&xprt->backlog); + kfree(xprt->servername); ++ ++ rpc_multipath_ops_destroy_xprt(xprt); ++ + /* + * Tear down transport state and free the rpc_xprt + */ +diff --git a/net/sunrpc/xprtmultipath.c b/net/sunrpc/xprtmultipath.c +index 6ebaa58b4..f6a004ee7 100644 +--- a/net/sunrpc/xprtmultipath.c ++++ b/net/sunrpc/xprtmultipath.c +@@ -18,6 +18,7 @@ + #include + #include + #include ++#include + + typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct list_head *head, + const struct rpc_xprt *cur); +@@ -26,8 +27,8 @@ static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular; + static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin; + static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall; + +-static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps, +- struct rpc_xprt *xprt) ++void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps, ++ struct rpc_xprt *xprt) + { + if (unlikely(xprt_get(xprt) == NULL)) + return; +@@ -36,7 +37,9 @@ static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps, + if (xps->xps_nxprts == 0) + xps->xps_net = xprt->xprt_net; + xps->xps_nxprts++; ++ rpc_xps_nactive_add_one(xps); + } ++EXPORT_SYMBOL(xprt_switch_add_xprt_locked); + + /** + * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch +@@ -63,6 +66,7 @@ static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps, + if (unlikely(xprt == NULL)) + return; + xps->xps_nxprts--; ++ rpc_xps_nactive_sub_one(xps); + if (xps->xps_nxprts == 0) + xps->xps_net = NULL; + smp_wmb(); +@@ -84,7 +88,7 @@ void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps, + spin_unlock(&xps->xps_lock); + xprt_put(xprt); + } +- ++EXPORT_SYMBOL(rpc_xprt_switch_remove_xprt); + /** + * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch + * @xprt: pointer to struct rpc_xprt +@@ -103,6 +107,10 @@ struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt, + spin_lock_init(&xps->xps_lock); + kref_init(&xps->xps_kref); + xps->xps_nxprts = 0; ++#if IS_ENABLED(CONFIG_ENFS) ++ xps->xps_nactive = 0; ++ atomic_long_set(&xps->xps_queuelen, 0); ++#endif + INIT_LIST_HEAD(&xps->xps_xprt_list); + xps->xps_iter_ops = &rpc_xprt_iter_singular; + xprt_switch_add_xprt_locked(xps, xprt); +@@ -148,6 +156,7 @@ struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps) + return xps; + return NULL; + } ++EXPORT_SYMBOL(xprt_switch_get); + + /** + * xprt_switch_put - Release a reference to a rpc_xprt_switch +@@ -160,6 +169,7 @@ void xprt_switch_put(struct rpc_xprt_switch *xps) + if (xps != NULL) + kref_put(&xps->xps_kref, xprt_switch_free); + } ++EXPORT_SYMBOL(xprt_switch_put); + + /** + * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch diff --git a/0003-add_enfs_module_for_nfs_mount_option.patch b/0003-add_enfs_module_for_nfs_mount_option.patch new file mode 100644 index 0000000000000000000000000000000000000000..7461fb62fa3083ae9782235c6bce7c5fbf1df114 --- /dev/null +++ b/0003-add_enfs_module_for_nfs_mount_option.patch @@ -0,0 +1,1208 @@ +diff --git a/fs/nfs/enfs/Makefile b/fs/nfs/enfs/Makefile +new file mode 100644 +index 000000000..6e83eb23c +--- /dev/null ++++ b/fs/nfs/enfs/Makefile +@@ -0,0 +1,18 @@ ++obj-m += enfs.o ++ ++#EXTRA_CFLAGS += -I$(PWD)/.. ++ ++enfs-y := enfs_init.o ++enfs-y += enfs_config.o ++enfs-y += mgmt_init.o ++enfs-y += enfs_multipath_client.o ++enfs-y += enfs_multipath_parse.o ++enfs-y += failover_path.o ++enfs-y += failover_time.o ++enfs-y += enfs_roundrobin.o ++enfs-y += enfs_multipath.o ++enfs-y += enfs_path.o ++enfs-y += enfs_proc.o ++enfs-y += enfs_remount.o ++enfs-y += pm_ping.o ++enfs-y += pm_state.o +diff --git a/fs/nfs/enfs/enfs.h b/fs/nfs/enfs/enfs.h +new file mode 100644 +index 000000000..be3d95220 +--- /dev/null ++++ b/fs/nfs/enfs/enfs.h +@@ -0,0 +1,62 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Client-side ENFS multipath adapt header. ++ * ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++ ++#ifndef _ENFS_H_ ++#define _ENFS_H_ ++#include ++#include ++#include ++#include ++#include ++#include ++#include "../enfs_adapter.h" ++ ++#define IP_ADDRESS_LEN_MAX 64 ++#define MAX_IP_PAIR_PER_MOUNT 8 ++#define MAX_IP_INDEX (MAX_IP_PAIR_PER_MOUNT) ++#define MAX_SUPPORTED_LOCAL_IP_COUNT 8 ++#define MAX_SUPPORTED_REMOTE_IP_COUNT 32 ++#define MAX_DNS_NAME_LEN 512 ++#define MAX_DNS_SUPPORTED 2 ++#define EXTEND_CMD_MAX_BUF_LEN 65356 ++ ++ ++struct nfs_ip_list { ++ int count; ++ struct sockaddr_storage address[MAX_SUPPORTED_REMOTE_IP_COUNT]; ++ size_t addrlen[MAX_SUPPORTED_REMOTE_IP_COUNT]; ++}; ++ ++struct NFS_ROUTE_DNS_S { ++ char dnsname[MAX_DNS_NAME_LEN]; // valid only if dnsExist is true ++}; ++ ++struct NFS_ROUTE_DNS_INFO_S { ++ int dnsNameCount; // Count of DNS name in the list ++ // valid only if dnsExist is true ++ struct NFS_ROUTE_DNS_S routeRemoteDnsList[MAX_DNS_SUPPORTED]; ++}; ++ ++struct rpc_iostats; ++struct enfs_xprt_context { ++ struct sockaddr_storage srcaddr; ++ struct rpc_iostats *stats; ++ bool main; ++ atomic_t path_state; ++ atomic_t path_check_state; ++}; ++ ++static inline bool enfs_is_main_xprt(struct rpc_xprt *xprt) ++{ ++ struct enfs_xprt_context *ctx = xprt->multipath_context; ++ ++ if (!ctx) ++ return false; ++ return ctx->main; ++} ++ ++#endif +diff --git a/fs/nfs/enfs/enfs_init.c b/fs/nfs/enfs/enfs_init.c +new file mode 100644 +index 000000000..4b5560819 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_init.c +@@ -0,0 +1,98 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Client-side ENFS adapter. ++ * ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "enfs.h" ++#include "enfs_multipath_parse.h" ++#include "enfs_multipath_client.h" ++#include "enfs_remount.h" ++#include "init.h" ++#include "enfs_log.h" ++#include "enfs_multipath.h" ++#include "mgmt_init.h" ++ ++struct enfs_adapter_ops enfs_adapter = { ++ .name = "enfs", ++ .owner = THIS_MODULE, ++ .parse_mount_options = nfs_multipath_parse_options, ++ .free_mount_options = nfs_multipath_free_options, ++ .client_info_init = nfs_multipath_client_info_init, ++ .client_info_free = nfs_multipath_client_info_free, ++ .client_info_match = nfs_multipath_client_info_match, ++ .client_info_show = nfs_multipath_client_info_show, ++ .remount_ip_list = enfs_remount_iplist, ++}; ++ ++int32_t enfs_init(void) ++{ ++ int err; ++ ++ err = enfs_multipath_init(); ++ if (err) { ++ enfs_log_error("init multipath failed.\n"); ++ goto out; ++ } ++ ++ err = mgmt_init(); ++ if (err != 0) { ++ enfs_log_error("init mgmt failed.\n"); ++ goto out_tp_exit; ++ } ++ ++ return 0; ++ ++out_tp_exit: ++ enfs_multipath_exit(); ++out: ++ return err; ++} ++ ++void enfs_fini(void) ++{ ++ mgmt_fini(); ++ ++ enfs_multipath_exit(); ++} ++ ++static int __init init_enfs(void) ++{ ++ int ret; ++ ++ ret = enfs_adapter_register(&enfs_adapter); ++ if (ret) { ++ pr_err("regist enfs_adapter fail. ret %d\n", ret); ++ return -1; ++ } ++ ++ ret = enfs_init(); ++ if (ret) { ++ enfs_adapter_unregister(&enfs_adapter); ++ return -1; ++ } ++ ++ return 0; ++} ++ ++static void __exit exit_enfs(void) ++{ ++ enfs_fini(); ++ enfs_adapter_unregister(&enfs_adapter); ++} ++ ++MODULE_LICENSE("GPL"); ++MODULE_AUTHOR("Huawei Tech. Co., Ltd."); ++MODULE_DESCRIPTION("Nfs client router"); ++MODULE_VERSION("1.0"); ++ ++module_init(init_enfs); ++module_exit(exit_enfs); +diff --git a/fs/nfs/enfs/enfs_multipath_client.c b/fs/nfs/enfs/enfs_multipath_client.c +new file mode 100644 +index 000000000..63c02898a +--- /dev/null ++++ b/fs/nfs/enfs/enfs_multipath_client.c +@@ -0,0 +1,340 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Client-side ENFS adapter. ++ * ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "enfs_multipath_client.h" ++#include "enfs_multipath_parse.h" ++ ++int ++nfs_multipath_client_mount_info_init(struct multipath_client_info *client_info, ++ const struct nfs_client_initdata *client_init_data) ++{ ++ struct multipath_mount_options *mount_options = ++ (struct multipath_mount_options *)client_init_data->enfs_option; ++ ++ if (mount_options->local_ip_list) { ++ client_info->local_ip_list = ++ kzalloc(sizeof(struct nfs_ip_list), GFP_KERNEL); ++ ++ if (!client_info->local_ip_list) ++ return -ENOMEM; ++ ++ memcpy(client_info->local_ip_list, mount_options->local_ip_list, ++ sizeof(struct nfs_ip_list)); ++ } ++ ++ if (mount_options->remote_ip_list) { ++ ++ client_info->remote_ip_list = ++ kzalloc(sizeof(struct nfs_ip_list), GFP_KERNEL); ++ ++ if (!client_info->remote_ip_list) { ++ kfree(client_info->local_ip_list); ++ client_info->local_ip_list = NULL; ++ return -ENOMEM; ++ } ++ memcpy(client_info->remote_ip_list, ++ mount_options->remote_ip_list, ++ sizeof(struct nfs_ip_list)); ++ } ++ ++ if (mount_options->pRemoteDnsInfo) { ++ client_info->pRemoteDnsInfo = ++ kzalloc(sizeof(struct NFS_ROUTE_DNS_INFO_S), GFP_KERNEL); ++ ++ if (!client_info->pRemoteDnsInfo) { ++ kfree(client_info->local_ip_list); ++ client_info->local_ip_list = NULL; ++ kfree(client_info->remote_ip_list); ++ client_info->remote_ip_list = NULL; ++ return -ENOMEM; ++ } ++ memcpy(client_info->pRemoteDnsInfo, ++ mount_options->pRemoteDnsInfo, ++ sizeof(struct NFS_ROUTE_DNS_INFO_S)); ++ } ++ return 0; ++} ++ ++void nfs_multipath_client_info_free_work(struct work_struct *work) ++{ ++ ++ struct multipath_client_info *clp_info; ++ ++ if (work == NULL) ++ return; ++ ++ clp_info = container_of(work, struct multipath_client_info, work); ++ ++ if (clp_info->local_ip_list != NULL) { ++ kfree(clp_info->local_ip_list); ++ clp_info->local_ip_list = NULL; ++ } ++ if (clp_info->remote_ip_list != NULL) { ++ kfree(clp_info->remote_ip_list); ++ clp_info->remote_ip_list = NULL; ++ } ++ kfree(clp_info); ++} ++ ++void nfs_multipath_client_info_free(void *data) ++{ ++ struct multipath_client_info *clp_info = ++ (struct multipath_client_info *)data; ++ ++ if (clp_info == NULL) ++ return; ++ pr_info("free client info %p.\n", clp_info); ++ INIT_WORK(&clp_info->work, nfs_multipath_client_info_free_work); ++ schedule_work(&clp_info->work); ++} ++ ++int nfs_multipath_client_info_init(void **data, ++ const struct nfs_client_initdata *cl_init) ++{ ++ int rc; ++ struct multipath_client_info *info; ++ struct multipath_client_info **enfs_info; ++ /* no multi path info, no need do multipath init */ ++ if (cl_init->enfs_option == NULL) ++ return 0; ++ enfs_info = (struct multipath_client_info **)data; ++ if (enfs_info == NULL) ++ return -EINVAL; ++ ++ if (*enfs_info == NULL) ++ *enfs_info = kzalloc(sizeof(struct multipath_client_info), ++ GFP_KERNEL); ++ ++ if (*enfs_info == NULL) ++ return -ENOMEM; ++ ++ info = (struct multipath_client_info *)*enfs_info; ++ pr_info("init client info %p.\n", info); ++ rc = nfs_multipath_client_mount_info_init(info, cl_init); ++ if (rc) { ++ nfs_multipath_client_info_free((void *)info); ++ return rc; ++ } ++ return rc; ++} ++ ++bool nfs_multipath_ip_list_info_match(const struct nfs_ip_list *ip_list_src, ++ const struct nfs_ip_list *ip_list_dst) ++{ ++ int i; ++ int j; ++ bool is_find; ++ /* if both are equal or NULL, then return true. */ ++ if (ip_list_src == ip_list_dst) ++ return true; ++ ++ if ((ip_list_src == NULL || ip_list_dst == NULL)) ++ return false; ++ ++ if (ip_list_src->count != ip_list_dst->count) ++ return false; ++ ++ for (i = 0; i < ip_list_src->count; i++) { ++ is_find = false; ++ for (j = 0; j < ip_list_src->count; j++) { ++ if (rpc_cmp_addr_port( ++ (const struct sockaddr *) ++ &ip_list_src->address[i], ++ (const struct sockaddr *) ++ &ip_list_dst->address[j]) ++ ) { ++ is_find = true; ++ break; ++ } ++ } ++ if (is_find == false) ++ return false; ++ } ++ return true; ++} ++ ++int ++nfs_multipath_dns_list_info_match( ++ const struct NFS_ROUTE_DNS_INFO_S *pRemoteDnsInfoSrc, ++ const struct NFS_ROUTE_DNS_INFO_S *pRemoteDnsInfoDst) ++{ ++ int i; ++ ++ /* if both are equal or NULL, then return true. */ ++ if (pRemoteDnsInfoSrc == pRemoteDnsInfoDst) ++ return true; ++ ++ if ((pRemoteDnsInfoSrc == NULL || pRemoteDnsInfoDst == NULL)) ++ return false; ++ ++ if (pRemoteDnsInfoSrc->dnsNameCount != pRemoteDnsInfoDst->dnsNameCount) ++ return false; ++ ++ for (i = 0; i < pRemoteDnsInfoSrc->dnsNameCount; i++) { ++ if (!strcmp(pRemoteDnsInfoSrc->routeRemoteDnsList[i].dnsname, ++ pRemoteDnsInfoDst->routeRemoteDnsList[i].dnsname)) ++ return false; ++ } ++ return true; ++} ++ ++int nfs_multipath_client_info_match(void *src, void *dst) ++{ ++ int ret = true; ++ ++ struct multipath_client_info *src_info; ++ struct multipath_mount_options *dst_info; ++ ++ src_info = (struct multipath_client_info *)src; ++ dst_info = (struct multipath_mount_options *)dst; ++ pr_info("try match client .\n"); ++ ret = nfs_multipath_ip_list_info_match(src_info->local_ip_list, ++ dst_info->local_ip_list); ++ if (ret == false) { ++ pr_err("local_ip not match.\n"); ++ return ret; ++ } ++ ++ ret = nfs_multipath_ip_list_info_match(src_info->remote_ip_list, ++ dst_info->remote_ip_list); ++ if (ret == false) { ++ pr_err("remote_ip not match.\n"); ++ return ret; ++ } ++ ++ ret = nfs_multipath_dns_list_info_match(src_info->pRemoteDnsInfo, ++ dst_info->pRemoteDnsInfo); ++ if (ret == false) { ++ pr_err("dns not match.\n"); ++ return ret; ++ } ++ pr_info("try match client ret %d.\n", ret); ++ return ret; ++} ++ ++void nfs_multipath_print_ip_info(struct seq_file *mount_option, ++ struct nfs_ip_list *ip_list, ++ const char *type) ++{ ++ char buf[IP_ADDRESS_LEN_MAX + 1]; ++ int len = 0; ++ int i = 0; ++ ++ seq_printf(mount_option, ",%s=", type); ++ for (i = 0; i < ip_list->count; i++) { ++ len = rpc_ntop((struct sockaddr *)&ip_list->address[i], ++ buf, IP_ADDRESS_LEN_MAX); ++ if (len > 0 && len < IP_ADDRESS_LEN_MAX) ++ buf[len] = '\0'; ++ ++ if (i == 0) ++ seq_printf(mount_option, "%s", buf); ++ else ++ seq_printf(mount_option, "~%s", buf); ++ dfprintk(MOUNT, ++ "NFS: show nfs mount option type:%s %s [%s]\n", ++ type, buf, __func__); ++ } ++} ++ ++void nfs_multipath_print_dns_info(struct seq_file *mount_option, ++ struct NFS_ROUTE_DNS_INFO_S *pRemoteDnsInfo, ++ const char *type) ++{ ++ int i = 0; ++ ++ seq_printf(mount_option, ",%s=", type); ++ for (i = 0; i < pRemoteDnsInfo->dnsNameCount; i++) { ++ if (i == 0) ++ seq_printf(mount_option, ++ "[%s", pRemoteDnsInfo->routeRemoteDnsList[i].dnsname); ++ else if (i == pRemoteDnsInfo->dnsNameCount - 1) ++ seq_printf(mount_option, ",%s]", ++ pRemoteDnsInfo->routeRemoteDnsList[i].dnsname); ++ else ++ seq_printf(mount_option, ++ ",%s", pRemoteDnsInfo->routeRemoteDnsList[i].dnsname); ++ } ++} ++ ++ ++static void multipath_print_sockaddr(struct seq_file *seq, ++ struct sockaddr *addr) ++{ ++ switch (addr->sa_family) { ++ case AF_INET: { ++ struct sockaddr_in *sin = (struct sockaddr_in *)addr; ++ ++ seq_printf(seq, "%pI4", &sin->sin_addr); ++ return; ++ } ++ case AF_INET6: { ++ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; ++ ++ seq_printf(seq, "%pI6", &sin6->sin6_addr); ++ return; ++ } ++ default: ++ break; ++ } ++ pr_err("unsupport family:%d\n", addr->sa_family); ++} ++ ++static void multipath_print_enfs_info(struct seq_file *seq, ++ struct nfs_server *server) ++{ ++ struct sockaddr_storage peeraddr; ++ struct rpc_clnt *next = server->client; ++ ++ rpc_peeraddr(server->client, ++ (struct sockaddr *)&peeraddr, sizeof(peeraddr)); ++ seq_puts(seq, ",enfs_info="); ++ multipath_print_sockaddr(seq, (struct sockaddr *)&peeraddr); ++ ++ while (next->cl_parent) { ++ if (next == next->cl_parent) ++ break; ++ next = next->cl_parent; ++ } ++ seq_printf(seq, "_%u", next->cl_clid); ++} ++ ++void nfs_multipath_client_info_show(struct seq_file *mount_option, void *data) ++{ ++ struct nfs_server *server = data; ++ struct multipath_client_info *client_info = ++ server->nfs_client->cl_multipath_data; ++ ++ dfprintk(MOUNT, "NFS: show nfs mount option[%s]\n", __func__); ++ if ((client_info->remote_ip_list) && ++ (client_info->remote_ip_list->count > 0)) ++ nfs_multipath_print_ip_info(mount_option, ++ client_info->remote_ip_list, ++ "remoteaddrs"); ++ ++ if ((client_info->local_ip_list) && ++ (client_info->local_ip_list->count > 0)) ++ nfs_multipath_print_ip_info(mount_option, ++ client_info->local_ip_list, ++ "localaddrs"); ++ ++ if ((client_info->pRemoteDnsInfo) && ++ (client_info->pRemoteDnsInfo->dnsNameCount > 0)) ++ nfs_multipath_print_dns_info(mount_option, ++ client_info->pRemoteDnsInfo, ++ "remotednsname"); ++ ++ multipath_print_enfs_info(mount_option, server); ++} +diff --git a/fs/nfs/enfs/enfs_multipath_client.h b/fs/nfs/enfs/enfs_multipath_client.h +new file mode 100644 +index 000000000..208f72606 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_multipath_client.h +@@ -0,0 +1,26 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Client-side ENFS adapter. ++ * ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#ifndef _ENFS_MULTIPATH_CLIENT_H_ ++#define _ENFS_MULTIPATH_CLIENT_H_ ++ ++#include "enfs.h" ++ ++struct multipath_client_info { ++ struct work_struct work; ++ struct nfs_ip_list *remote_ip_list; ++ struct nfs_ip_list *local_ip_list; ++ struct NFS_ROUTE_DNS_INFO_S *pRemoteDnsInfo; ++ s64 client_id; ++}; ++ ++int nfs_multipath_client_info_init(void **data, ++ const struct nfs_client_initdata *cl_init); ++void nfs_multipath_client_info_free(void *data); ++int nfs_multipath_client_info_match(void *src, void *dst); ++void nfs_multipath_client_info_show(struct seq_file *mount_option, void *data); ++ ++#endif +diff --git a/fs/nfs/enfs/enfs_multipath_parse.c b/fs/nfs/enfs/enfs_multipath_parse.c +new file mode 100644 +index 000000000..868782b0a +--- /dev/null ++++ b/fs/nfs/enfs/enfs_multipath_parse.c +@@ -0,0 +1,600 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Client-side ENFS adapter. ++ * ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "enfs_multipath_parse.h" ++#include "enfs_log.h" ++ ++#define NFSDBG_FACILITY NFSDBG_CLIENT ++ ++void nfs_multipath_parse_ip_ipv6_add(struct sockaddr_in6 *sin6, int add_num) ++{ ++ int i; ++ ++ pr_info("NFS: before %08x%08x%08x%08x add_num: %d[%s]\n", ++ ntohl(sin6->sin6_addr.in6_u.u6_addr32[0]), ++ ntohl(sin6->sin6_addr.in6_u.u6_addr32[1]), ++ ntohl(sin6->sin6_addr.in6_u.u6_addr32[2]), ++ ntohl(sin6->sin6_addr.in6_u.u6_addr32[3]), ++ add_num, __func__); ++ for (i = 0; i < add_num; i++) { ++ sin6->sin6_addr.in6_u.u6_addr32[3] = ++ htonl(ntohl(sin6->sin6_addr.in6_u.u6_addr32[3]) + 1); ++ ++ if (sin6->sin6_addr.in6_u.u6_addr32[3] != 0) ++ continue; ++ ++ sin6->sin6_addr.in6_u.u6_addr32[2] = ++ htonl(ntohl(sin6->sin6_addr.in6_u.u6_addr32[2]) + 1); ++ ++ if (sin6->sin6_addr.in6_u.u6_addr32[2] != 0) ++ continue; ++ ++ sin6->sin6_addr.in6_u.u6_addr32[1] = ++ htonl(ntohl(sin6->sin6_addr.in6_u.u6_addr32[1]) + 1); ++ ++ if (sin6->sin6_addr.in6_u.u6_addr32[1] != 0) ++ continue; ++ ++ sin6->sin6_addr.in6_u.u6_addr32[0] = ++ htonl(ntohl(sin6->sin6_addr.in6_u.u6_addr32[0]) + 1); ++ ++ if (sin6->sin6_addr.in6_u.u6_addr32[0] != 0) ++ continue; ++ } ++ ++ return; ++ ++} ++ ++static int nfs_multipath_parse_ip_range(struct net *net_ns, const char *cursor, ++ struct nfs_ip_list *ip_list, enum nfs_multi_path_options type) ++{ ++ struct sockaddr_storage addr; ++ struct sockaddr_storage tmp_addr; ++ int i; ++ size_t len; ++ int add_num = 1; ++ bool duplicate_flag = false; ++ bool is_complete = false; ++ struct sockaddr_in *sin4; ++ struct sockaddr_in6 *sin6; ++ ++ pr_info("NFS: parsing nfs mount option '%s' type: %d[%s]\n", ++ cursor, type, __func__); ++ len = rpc_pton(net_ns, cursor, strlen(cursor), ++ (struct sockaddr *)&addr, sizeof(addr)); ++ if (!len) ++ return -EINVAL; ++ ++ if (addr.ss_family != ip_list->address[ip_list->count - 1].ss_family) { ++ pr_info("NFS: %s parsing nfs mount option type: %d fail.\n", ++ __func__, type); ++ return -EINVAL; ++ } ++ ++ if (rpc_cmp_addr((const struct sockaddr *) ++ &ip_list->address[ip_list->count - 1], ++ (const struct sockaddr *)&addr)) { ++ ++ pr_info("range ip is same ip.\n"); ++ return 0; ++ ++ } ++ ++ while (true) { ++ ++ tmp_addr = ip_list->address[ip_list->count - 1]; ++ ++ switch (addr.ss_family) { ++ case AF_INET: ++ sin4 = (struct sockaddr_in *)&tmp_addr; ++ ++ sin4->sin_addr.s_addr = ++ htonl(ntohl(sin4->sin_addr.s_addr) + add_num); ++ ++ pr_info("NFS: mount option ip%08x type: %d ipcont %d [%s]\n", ++ ntohl(sin4->sin_addr.s_addr), ++ type, ip_list->count, __func__); ++ break; ++ case AF_INET6: ++ sin6 = (struct sockaddr_in6 *)&tmp_addr; ++ nfs_multipath_parse_ip_ipv6_add(sin6, add_num); ++ pr_info("NFS: mount option ip %08x%08x%08x%08x type: %d ipcont %d [%s]\n", ++ ntohl(sin6->sin6_addr.in6_u.u6_addr32[0]), ++ ntohl(sin6->sin6_addr.in6_u.u6_addr32[1]), ++ ntohl(sin6->sin6_addr.in6_u.u6_addr32[2]), ++ ntohl(sin6->sin6_addr.in6_u.u6_addr32[3]), ++ type, ip_list->count, __func__); ++ break; ++ // return -EOPNOTSUPP; ++ default: ++ return -EOPNOTSUPP; ++ } ++ ++ if (rpc_cmp_addr((const struct sockaddr *)&tmp_addr, ++ (const struct sockaddr *)&addr)) { ++ is_complete = true; ++ } ++ // delete duplicate ip, continuosly repeat, skip it ++ for (i = 0; i < ip_list->count; i++) { ++ duplicate_flag = false; ++ if (rpc_cmp_addr((const struct sockaddr *) ++ &ip_list->address[i], ++ (const struct sockaddr *)&tmp_addr)) { ++ add_num++; ++ duplicate_flag = true; ++ break; ++ } ++ } ++ ++ if (duplicate_flag == false) { ++ pr_info("this ip not duplicate;"); ++ add_num = 1; ++ // if not repeat but omit limit return false ++ if ((type == LOCALADDR && ++ ip_list->count >= MAX_SUPPORTED_LOCAL_IP_COUNT) || ++ (type == REMOTEADDR && ++ ip_list->count >= MAX_SUPPORTED_REMOTE_IP_COUNT)) { ++ ++ pr_info("[MULTIPATH:%s] iplist for type %d reached %d, more than supported limit %d\n", ++ __func__, type, ip_list->count, ++ type == LOCALADDR ? ++ MAX_SUPPORTED_LOCAL_IP_COUNT : ++ MAX_SUPPORTED_REMOTE_IP_COUNT); ++ ip_list->count = 0; ++ return -ENOSPC; ++ } ++ ip_list->address[ip_list->count] = tmp_addr; ++ ++ ip_list->addrlen[ip_list->count] = ++ ip_list->addrlen[ip_list->count - 1]; ++ ++ ip_list->count += 1; ++ } ++ if (is_complete == true) ++ break; ++ } ++ return 0; ++} ++ ++int nfs_multipath_parse_ip_list_inter(struct nfs_ip_list *ip_list, ++ struct net *net_ns, ++ char *cursor, enum nfs_multi_path_options type) ++{ ++ int i = 0; ++ struct sockaddr_storage addr; ++ struct sockaddr_storage swap; ++ int len; ++ ++ pr_info("NFS: parsing nfs mount option '%s' type: %d[%s]\n", ++ cursor, type, __func__); ++ ++ len = rpc_pton(net_ns, cursor, ++ strlen(cursor), ++ (struct sockaddr *)&addr, sizeof(addr)); ++ if (!len) ++ return -EINVAL; ++ ++ // check repeated ip ++ for (i = 0; i < ip_list->count; i++) { ++ if (rpc_cmp_addr((const struct sockaddr *) ++ &ip_list->address[i], ++ (const struct sockaddr *)&addr)) { ++ ++ pr_info("NFS: mount option '%s' type:%d index %d same as before index %d [%s]\n", ++ cursor, type, ip_list->count, i, __func__); ++ // prevent this ip is beginning ++ // if repeated take it to the end of list ++ swap = ip_list->address[i]; ++ ++ ip_list->address[i] = ++ ip_list->address[ip_list->count-1]; ++ ++ ip_list->address[ip_list->count-1] = swap; ++ return 0; ++ } ++ } ++ // if not repeated, check exceed limit ++ if ((type == LOCALADDR && ++ ip_list->count >= MAX_SUPPORTED_LOCAL_IP_COUNT) || ++ (type == REMOTEADDR && ++ ip_list->count >= MAX_SUPPORTED_REMOTE_IP_COUNT)) { ++ ++ pr_info("[MULTIPATH:%s] iplist for type %d reached %d, more than supported limit %d\n", ++ __func__, type, ip_list->count, ++ type == LOCALADDR ? ++ MAX_SUPPORTED_LOCAL_IP_COUNT : ++ MAX_SUPPORTED_REMOTE_IP_COUNT); ++ ++ ip_list->count = 0; ++ return -ENOSPC; ++ } ++ ip_list->address[ip_list->count] = addr; ++ ip_list->addrlen[ip_list->count] = len; ++ ip_list->count++; ++ ++ return 0; ++} ++ ++char *nfs_multipath_parse_ip_list_get_cursor(char **buf_to_parse, bool *single) ++{ ++ char *cursor = NULL; ++ const char *single_sep = strchr(*buf_to_parse, '~'); ++ const char *range_sep = strchr(*buf_to_parse, '-'); ++ ++ *single = true; ++ if (range_sep) { ++ if (range_sep > single_sep) { // A-B or A~B-C ++ if (single_sep == NULL) { // A-B ++ cursor = strsep(buf_to_parse, "-"); ++ if (cursor) ++ *single = false; ++ } else// A~B-C ++ cursor = strsep(buf_to_parse, "~"); ++ } else { // A-B~C ++ cursor = strsep(buf_to_parse, "-"); ++ if (cursor) ++ *single = false; ++ } ++ } else { // A~B~C ++ cursor = strsep(buf_to_parse, "~"); ++ } ++ return cursor; ++} ++ ++bool nfs_multipath_parse_param_check(enum nfs_multi_path_options type, ++ struct multipath_mount_options *options) ++{ ++ if (type == REMOUNTREMOTEADDR && options->remote_ip_list->count != 0) { ++ memset(options->remote_ip_list, 0, sizeof(struct nfs_ip_list)); ++ return true; ++ } ++ if (type == REMOUNTLOCALADDR && options->local_ip_list->count != 0) { ++ memset(options->local_ip_list, 0, sizeof(struct nfs_ip_list)); ++ return true; ++ } ++ if ((type == REMOTEADDR || type == REMOTEDNSNAME) && ++ options->pRemoteDnsInfo->dnsNameCount != 0) { ++ ++ pr_info("[MULTIPATH:%s] parse for %d ,already have dns\n", ++ __func__, type); ++ return false; ++ } else if ((type == REMOTEADDR || type == REMOTEDNSNAME) && ++ options->remote_ip_list->count != 0) { ++ ++ pr_info("[MULTIPATH:%s] parse for %d ,already have iplist\n", ++ __func__, type); ++ return false; ++ } ++ return true; ++} ++ ++int nfs_multipath_parse_ip_list(char *buffer, struct net *net_ns, ++ struct multipath_mount_options *options, ++ enum nfs_multi_path_options type) ++{ ++ char *buf_to_parse = NULL; ++ bool prev_range = false; ++ int ret = 0; ++ char *cursor = NULL; ++ bool single = true; ++ struct nfs_ip_list *ip_list_tmp = NULL; ++ ++ if (!nfs_multipath_parse_param_check(type, options)) ++ return -ENOTSUPP; ++ ++ if (type == REMOUNTREMOTEADDR) ++ type = REMOTEADDR; ++ ++ if (type == REMOUNTLOCALADDR) ++ type = LOCALADDR; ++ ++ if (type == LOCALADDR) ++ ip_list_tmp = options->local_ip_list; ++ else ++ ip_list_tmp = options->remote_ip_list; ++ ++ pr_info("NFS: parsing nfs mount option '%s' type: %d[%s]\n", ++ buffer, type, __func__); ++ ++ buf_to_parse = buffer; ++ while (buf_to_parse != NULL) { ++ cursor = ++ nfs_multipath_parse_ip_list_get_cursor(&buf_to_parse, &single); ++ if (!cursor) ++ break; ++ ++ if (single == false && prev_range == true) { ++ pr_info("NFS: mount option type: %d fail. Multiple Range.[%s]\n", ++ type, __func__); ++ ++ ret = -EINVAL; ++ goto out; ++ } ++ ++ if (prev_range == false) { ++ ret = nfs_multipath_parse_ip_list_inter(ip_list_tmp, ++ net_ns, cursor, type); ++ if (ret) ++ goto out; ++ if (single == false) ++ prev_range = true; ++ } else { ++ ret = nfs_multipath_parse_ip_range(net_ns, cursor, ++ ip_list_tmp, type); ++ if (ret != 0) ++ goto out; ++ prev_range = false; ++ } ++ } ++ ++out: ++ if (ret) ++ memset(ip_list_tmp, 0, sizeof(struct nfs_ip_list)); ++ ++ return ret; ++} ++ ++int nfs_multipath_parse_dns_list(char *buffer, struct net *net_ns, ++ struct multipath_mount_options *options) ++{ ++ struct NFS_ROUTE_DNS_INFO_S *dns_name_list_tmp = NULL; ++ char *cursor = NULL; ++ char *bufToParse; ++ ++ if (!nfs_multipath_parse_param_check(REMOTEDNSNAME, options)) ++ return -ENOTSUPP; ++ ++ pr_info("[MULTIPATH:%s] buffer %s\n", __func__, buffer); ++ // freed in nfs_free_parsed_mount_data ++ dns_name_list_tmp = kmalloc(sizeof(struct NFS_ROUTE_DNS_INFO_S), ++ GFP_KERNEL); ++ if (!dns_name_list_tmp) ++ return -ENOMEM; ++ ++ dns_name_list_tmp->dnsNameCount = 0; ++ bufToParse = buffer; ++ while (bufToParse) { ++ if (dns_name_list_tmp->dnsNameCount >= MAX_DNS_SUPPORTED) { ++ pr_err("%s: dnsname for %s reached %d,more than supported limit %d\n", ++ __func__, cursor, ++ dns_name_list_tmp->dnsNameCount, ++ MAX_DNS_SUPPORTED); ++ dns_name_list_tmp->dnsNameCount = 0; ++ return -ENOSPC; ++ } ++ cursor = strsep(&bufToParse, "~"); ++ if (!cursor) ++ break; ++ ++ strcpy(dns_name_list_tmp->routeRemoteDnsList ++ [dns_name_list_tmp->dnsNameCount].dnsname, ++ cursor); ++ dns_name_list_tmp->dnsNameCount++; ++ } ++ if (dns_name_list_tmp->dnsNameCount == 0) ++ return -EINVAL; ++ options->pRemoteDnsInfo = dns_name_list_tmp; ++ return 0; ++} ++ ++int nfs_multipath_parse_options_check_ipv4_valid(struct sockaddr_in *addr) ++{ ++ if (addr->sin_addr.s_addr == 0 || addr->sin_addr.s_addr == 0xffffffff) ++ return -EINVAL; ++ return 0; ++} ++ ++int nfs_multipath_parse_options_check_ipv6_valid(struct sockaddr_in6 *addr) ++{ ++ if (addr->sin6_addr.in6_u.u6_addr32[0] == 0 && ++ addr->sin6_addr.in6_u.u6_addr32[1] == 0 && ++ addr->sin6_addr.in6_u.u6_addr32[2] == 0 && ++ addr->sin6_addr.in6_u.u6_addr32[3] == 0) ++ return -EINVAL; ++ ++ if (addr->sin6_addr.in6_u.u6_addr32[0] == 0xffffffff && ++ addr->sin6_addr.in6_u.u6_addr32[1] == 0xffffffff && ++ addr->sin6_addr.in6_u.u6_addr32[2] == 0xffffffff && ++ addr->sin6_addr.in6_u.u6_addr32[3] == 0xffffffff) ++ return -EINVAL; ++ return 0; ++} ++ ++int nfs_multipath_parse_options_check_ip_valid(struct sockaddr_storage *address) ++{ ++ int rc = 0; ++ ++ if (address->ss_family == AF_INET) ++ rc = nfs_multipath_parse_options_check_ipv4_valid( ++ (struct sockaddr_in *)address); ++ else if (address->ss_family == AF_INET6) ++ rc = nfs_multipath_parse_options_check_ipv6_valid( ++ (struct sockaddr_in6 *)address); ++ else ++ rc = -EINVAL; ++ ++ return rc; ++} ++ ++int nfs_multipath_parse_options_check_valid( ++ struct multipath_mount_options *options) ++{ ++ int rc; ++ int i; ++ ++ if (options == NULL) ++ return 0; ++ ++ for (i = 0; i < options->local_ip_list->count; i++) { ++ rc = nfs_multipath_parse_options_check_ip_valid( ++ &options->local_ip_list->address[i]); ++ if (rc != 0) ++ return rc; ++ } ++ ++ for (i = 0; i < options->remote_ip_list->count; i++) { ++ rc = nfs_multipath_parse_options_check_ip_valid( ++ &options->remote_ip_list->address[i]); ++ if (rc != 0) ++ return rc; ++ } ++ ++ return 0; ++} ++int nfs_multipath_parse_options_check_duplicate( ++ struct multipath_mount_options *options) ++{ ++ int i; ++ int j; ++ ++ if (options == NULL || ++ options->local_ip_list->count == 0 || ++ options->remote_ip_list->count == 0) ++ ++ return 0; ++ ++ for (i = 0; i < options->local_ip_list->count; i++) { ++ for (j = 0; j < options->remote_ip_list->count; j++) { ++ if (rpc_cmp_addr((const struct sockaddr *) ++ &options->local_ip_list->address[i], ++ (const struct sockaddr *) ++ &options->remote_ip_list->address[j])) ++ return -ENOTSUPP; ++ } ++ } ++ return 0; ++} ++ ++int nfs_multipath_parse_options_check(struct multipath_mount_options *options) ++{ ++ int rc = 0; ++ ++ rc = nfs_multipath_parse_options_check_valid(options); ++ ++ if (rc != 0) { ++ pr_err("has invaild ip.\n"); ++ return rc; ++ } ++ ++ rc = nfs_multipath_parse_options_check_duplicate(options); ++ if (rc != 0) ++ return rc; ++ return rc; ++} ++ ++int nfs_multipath_alloc_options(void **enfs_option) ++{ ++ struct multipath_mount_options *options = NULL; ++ ++ options = kzalloc(sizeof(struct multipath_mount_options), GFP_KERNEL); ++ ++ if (options == NULL) ++ return -ENOMEM; ++ ++ options->local_ip_list = ++ kzalloc(sizeof(struct nfs_ip_list), GFP_KERNEL); ++ if (options->local_ip_list == NULL) { ++ kfree(options); ++ return -ENOMEM; ++ } ++ ++ options->remote_ip_list = ++ kzalloc(sizeof(struct nfs_ip_list), GFP_KERNEL); ++ if (options->remote_ip_list == NULL) { ++ kfree(options->local_ip_list); ++ kfree(options); ++ return -ENOMEM; ++ } ++ ++ options->pRemoteDnsInfo = kzalloc(sizeof(struct NFS_ROUTE_DNS_INFO_S), ++ GFP_KERNEL); ++ if (options->pRemoteDnsInfo == NULL) { ++ kfree(options->remote_ip_list); ++ kfree(options->local_ip_list); ++ kfree(options); ++ return -ENOMEM; ++ } ++ ++ *enfs_option = options; ++ return 0; ++} ++ ++int nfs_multipath_parse_options(enum nfs_multi_path_options type, ++ char *str, void **enfs_option, struct net *net_ns) ++{ ++ int rc; ++ struct multipath_mount_options *options = NULL; ++ ++ if ((str == NULL) || (enfs_option == NULL) || (net_ns == NULL)) ++ return -EINVAL; ++ ++ if (*enfs_option == NULL) { ++ rc = nfs_multipath_alloc_options(enfs_option); ++ if (rc != 0) { ++ enfs_log_error( ++ "alloc enfs_options failed! errno:%d\n", rc); ++ return rc; ++ } ++ } ++ ++ options = (struct multipath_mount_options *)*enfs_option; ++ ++ if (type == LOCALADDR || type == REMOUNTLOCALADDR || ++ type == REMOTEADDR || type == REMOUNTREMOTEADDR) { ++ rc = nfs_multipath_parse_ip_list(str, net_ns, options, type); ++ } else if (type == REMOTEDNSNAME) { ++ /* alloc and release need to modify */ ++ rc = nfs_multipath_parse_dns_list(str, net_ns, options); ++ } else { ++ rc = -EOPNOTSUPP; ++ } ++ ++ // after parsing cmd, need checking local and remote ++ // IP is same. if not means illegal cmd ++ if (rc == 0) ++ rc = nfs_multipath_parse_options_check_duplicate(options); ++ ++ if (rc == 0) ++ rc = nfs_multipath_parse_options_check(options); ++ ++ return rc; ++} ++ ++void nfs_multipath_free_options(void **enfs_option) ++{ ++ struct multipath_mount_options *options; ++ ++ if (enfs_option == NULL || *enfs_option == NULL) ++ return; ++ ++ options = (struct multipath_mount_options *)*enfs_option; ++ ++ if (options->remote_ip_list != NULL) { ++ kfree(options->remote_ip_list); ++ options->remote_ip_list = NULL; ++ } ++ ++ if (options->local_ip_list != NULL) { ++ kfree(options->local_ip_list); ++ options->local_ip_list = NULL; ++ } ++ ++ if (options->pRemoteDnsInfo != NULL) { ++ kfree(options->pRemoteDnsInfo); ++ options->pRemoteDnsInfo = NULL; ++ } ++ ++ kfree(options); ++ *enfs_option = NULL; ++} +diff --git a/fs/nfs/enfs/enfs_multipath_parse.h b/fs/nfs/enfs/enfs_multipath_parse.h +new file mode 100644 +index 000000000..d5a36b2eb +--- /dev/null ++++ b/fs/nfs/enfs/enfs_multipath_parse.h +@@ -0,0 +1,22 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Client-side ENFS adapter. ++ * ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#ifndef _ENFS_MULTIPATH_PARSE_H_ ++#define _ENFS_MULTIPATH_PARSE_H_ ++ ++#include "enfs.h" ++ ++struct multipath_mount_options { ++ struct nfs_ip_list *remote_ip_list; ++ struct nfs_ip_list *local_ip_list; ++ struct NFS_ROUTE_DNS_INFO_S *pRemoteDnsInfo; ++}; ++ ++int nfs_multipath_parse_options(enum nfs_multi_path_options type, ++ char *str, void **enfs_option, struct net *net_ns); ++void nfs_multipath_free_options(void **enfs_option); ++ ++#endif diff --git a/0004-add_enfs_module_for_sunrpc_multipatch.patch b/0004-add_enfs_module_for_sunrpc_multipatch.patch new file mode 100644 index 0000000000000000000000000000000000000000..eb9acf8c7781a48078c787554c02ec2e4ff1b003 --- /dev/null +++ b/0004-add_enfs_module_for_sunrpc_multipatch.patch @@ -0,0 +1,1860 @@ +diff --git a/fs/nfs/enfs/enfs_multipath.c b/fs/nfs/enfs/enfs_multipath.c +new file mode 100644 +index 000000000..d49dc91e1 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_multipath.c +@@ -0,0 +1,654 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ */ ++#include "enfs_multipath.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "enfs.h" ++#include "enfs_log.h" ++#include "enfs_multipath_parse.h" ++#include "enfs_path.h" ++#include "enfs_proc.h" ++#include "enfs_remount.h" ++#include "enfs_roundrobin.h" ++#include "failover_path.h" ++#include "failover_time.h" ++#include "pm_ping.h" ++#include "pm_state.h" ++ ++struct xprt_attach_callback_data { ++ atomic_t *conditon; ++ wait_queue_head_t *waitq; ++}; ++ ++struct xprt_attach_info { ++ struct sockaddr_storage *localAddress; ++ struct sockaddr_storage *remoteAddress; ++ struct rpc_xprt *xprt; ++ struct xprt_attach_callback_data *data; ++}; ++ ++static DECLARE_WAIT_QUEUE_HEAD(path_attach_wait_queue); ++ ++/** ++ * set socket port. ++ * @ns: need transform to network byte order ++ */ ++static void sockaddr_set_port(struct sockaddr *addr, __be16 port, bool ns) ++{ ++ switch (addr->sa_family) { ++ case AF_INET: ++ ((struct sockaddr_in *)addr)->sin_port = ++ ns ? htons(port) : port; ++ break; ++ case AF_INET6: ++ ((struct sockaddr_in6 *)addr)->sin6_port = ++ ns ? htons(port) : port; ++ break; ++ } ++} ++ ++static __be16 get_rpc_clnt_port(struct rpc_clnt *clnt) ++{ ++ struct sockaddr_storage ss; ++ struct sockaddr *addr = (struct sockaddr *)&ss; ++ ++ rpc_peeraddr(clnt, (struct sockaddr *)&ss, sizeof(ss)); ++ switch (addr->sa_family) { ++ case AF_INET: ++ return ((struct sockaddr_in *)addr)->sin_port; ++ ++ case AF_INET6: ++ return ((struct sockaddr_in6 *)addr)->sin6_port; ++ ++ default: ++ enfs_log_error("not support family:%d.\n", addr->sa_family); ++ return -1; ++ } ++} ++ ++static int sockaddr_ip_to_str(struct sockaddr *addr, char *buf, int len) ++{ ++ switch (addr->sa_family) { ++ case AF_INET: { ++ struct sockaddr_in *sin = (struct sockaddr_in *)addr; ++ ++ snprintf(buf, len, "%pI4", &sin->sin_addr); ++ return 0; ++ } ++ case AF_INET6: { ++ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; ++ ++ snprintf(buf, len, "%pI6", &sin6->sin6_addr); ++ return 0; ++ } ++ default: ++ break; ++ } ++ return 1; ++} ++ ++void print_enfs_multipath_addr(struct sockaddr *local, struct sockaddr *remote) ++{ ++ char buf1[128]; ++ char buf2[128]; ++ ++ sockaddr_ip_to_str(local, buf1, sizeof(buf1)); ++ sockaddr_ip_to_str(remote, buf2, sizeof(buf2)); ++ ++ pr_info("local:%s remote:%s\n", buf1, buf2); ++} ++ ++static int enfs_servername(char *servername, unsigned long long len, ++ struct rpc_create_args *args) ++{ ++ struct sockaddr_un *sun = (struct sockaddr_un *)args->address; ++ struct sockaddr_in *sin = (struct sockaddr_in *)args->address; ++ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)args->address; ++ ++ servername[0] = '\0'; ++ switch (args->address->sa_family) { ++ case AF_LOCAL: ++ snprintf(servername, len, "%s", sun->sun_path); ++ break; ++ case AF_INET: ++ snprintf(servername, len, "%pI4", &sin->sin_addr.s_addr); ++ break; ++ case AF_INET6: ++ snprintf(servername, len, "%pI6", &sin6->sin6_addr); ++ break; ++ default: ++ pr_info("invalid family:%d\n", ++ args->address->sa_family); ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++static void pm_xprt_ping_callback(void *data) ++{ ++ struct xprt_attach_callback_data *attach_callback_data = ++ (struct xprt_attach_callback_data *)data; ++ ++ atomic_dec(attach_callback_data->conditon); ++ wake_up(attach_callback_data->waitq); ++} ++ ++static ++int enfs_add_xprt_setup(struct rpc_clnt *clnt, struct rpc_xprt_switch *xps, ++ struct rpc_xprt *xprt, void *data) ++{ ++ int ret; ++ struct enfs_xprt_context *ctx; ++ struct xprt_attach_info *attach_info = data; ++ struct sockaddr_storage *srcaddr = attach_info->localAddress; ++ ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ ctx->stats = rpc_alloc_iostats(clnt); ++ ctx->main = false; ++ ctx->srcaddr = *srcaddr; ++ pm_set_path_state(xprt, PM_STATE_INIT); ++ pm_ping_set_path_check_state(xprt, PM_CHECK_INIT); ++ ++ attach_info->xprt = xprt; ++ xprt_get(xprt); ++ ++ ret = pm_ping_rpc_test_xprt_with_callback(clnt, xprt, ++ pm_xprt_ping_callback, attach_info->data); ++ if (ret != 1) { ++ enfs_log_error("Failed to add ping task.\n"); ++ atomic_dec(attach_info->data->conditon); ++ } ++ ++ // so that rpc_clnt_add_xprt ++ // does not call rpc_xprt_switch_add_xprt ++ return 1; ++} ++ ++int enfs_configure_xprt_to_clnt(struct xprt_create *xprtargs, ++ struct rpc_clnt *clnt, ++ struct xprt_attach_info *attach_info) ++{ ++ int err = 0; ++ __be16 port; ++ ++ xprtargs->srcaddr = (struct sockaddr *)attach_info->localAddress; ++ xprtargs->dstaddr = (struct sockaddr *)attach_info->remoteAddress; ++ ++ port = get_rpc_clnt_port(clnt); ++ sockaddr_set_port((struct sockaddr *)attach_info->remoteAddress, port, ++ false); ++ print_enfs_multipath_addr((struct sockaddr *)attach_info->localAddress, ++ (struct sockaddr *)attach_info->remoteAddress); ++ ++ err = rpc_clnt_add_xprt(clnt, xprtargs, ++ enfs_add_xprt_setup, ++ attach_info); ++ if (err != 1) { ++ enfs_log_error("clnt add xprt err:%d\n", err); ++ return err; ++ } ++ return 0; ++} ++ ++// Calculate the greatest common divisor of two numbers ++static int enfs_cal_gcd(int num1, int num2) ++{ ++ if (num2 == 0) ++ return num1; ++ return enfs_cal_gcd(num2, num1 % num2); ++} ++ ++bool enfs_cmp_addrs(struct sockaddr_storage *srcaddr, ++ struct sockaddr_storage *dstaddr, ++ struct sockaddr_storage *localAddress, ++ struct sockaddr_storage *remoteAddress) ++{ ++ if (rpc_cmp_addr((struct sockaddr *)srcaddr, ++ (struct sockaddr *)localAddress)) { ++ ++ if (rpc_cmp_addr((struct sockaddr *)dstaddr, ++ (struct sockaddr *)remoteAddress)) ++ return true; ++ ++ } ++ ++ return false; ++} ++ ++bool enfs_xprt_addrs_is_same(struct rpc_xprt *xprt, ++ struct sockaddr_storage *localAddress, ++ struct sockaddr_storage *remoteAddress) ++{ ++ struct enfs_xprt_context *xprt_local_info = NULL; ++ struct sockaddr_storage *srcaddr = NULL; ++ struct sockaddr_storage *dstaddr = NULL; ++ ++ if (xprt == NULL) ++ return true; ++ xprt_local_info = (struct enfs_xprt_context *)xprt->multipath_context; ++ srcaddr = &xprt_local_info->srcaddr; ++ dstaddr = &xprt->addr; ++ ++ return enfs_cmp_addrs(srcaddr, dstaddr, localAddress, remoteAddress); ++} ++ ++bool enfs_already_have_xprt(struct rpc_clnt *clnt, ++ struct sockaddr_storage *localAddress, ++ struct sockaddr_storage *remoteAddress) ++{ ++ struct rpc_xprt *pos = NULL; ++ struct rpc_xprt_switch *xps = NULL; ++ ++ rcu_read_lock(); ++ xps = xprt_switch_get(rcu_dereference(clnt->cl_xpi.xpi_xpswitch)); ++ if (xps == NULL) { ++ rcu_read_unlock(); ++ return false; ++ } ++ list_for_each_entry_rcu(pos, &xps->xps_xprt_list, xprt_switch) { ++ if (enfs_xprt_addrs_is_same(pos, localAddress, remoteAddress)) { ++ xprt_switch_put(xps); ++ rcu_read_unlock(); ++ return true; ++ } ++ } ++ xprt_switch_put(xps); ++ rcu_read_unlock(); ++ return false; ++} ++ ++static void enfs_xprt_switch_add_xprt(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt) ++{ ++ struct rpc_xprt_switch *xps; ++ ++ rcu_read_lock(); ++ xps = rcu_dereference(clnt->cl_xpi.xpi_xpswitch); ++ spin_lock(&xps->xps_lock); ++ if (xps->xps_net == xprt->xprt_net || xps->xps_net == NULL) ++ xprt_switch_add_xprt_locked(xps, xprt); ++ spin_unlock(&xps->xps_lock); ++ rcu_read_unlock(); ++} ++ ++static void enfs_add_xprts_to_clnt(struct rpc_clnt *clnt, ++ struct xprt_attach_info *attach_infos, ++ int total_combinations) ++{ ++ struct rpc_xprt *xprt; ++ enum pm_path_state state; ++ int i; ++ int link_count = 0; ++ ++ for (i = 0; i < total_combinations; i++) { ++ xprt = attach_infos[i].xprt; ++ ++ if (xprt == NULL) ++ continue; ++ ++ state = pm_get_path_state(xprt); ++ ++ if (link_count < MAX_XPRT_NUM_PER_CLIENT && ++ state == PM_STATE_NORMAL) { ++ ++ enfs_xprt_switch_add_xprt(clnt, xprt); ++ link_count++; ++ continue; ++ ++ } ++ ++ xprt_put(xprt); ++ } ++} ++ ++static void enfs_combine_addr(struct xprt_create *xprtargs, ++ struct rpc_clnt *clnt, ++ struct nfs_ip_list *local, ++ struct nfs_ip_list *remote) ++{ ++ int i; ++ int err; ++ int local_index; ++ int remote_index; ++ int link_count = 0; ++ int local_total = local->count; ++ int remote_total = remote->count; ++ int local_remote_total_lcm; ++ int total_combinations = local_total * remote_total; ++ struct xprt_attach_info *attach_infos; ++ atomic_t wait_queue_condition; ++ ++ struct xprt_attach_callback_data attach_callback_data = { ++ &wait_queue_condition, &path_attach_wait_queue}; ++ ++ atomic_set(&wait_queue_condition, total_combinations); ++ ++ pr_info("local count:%d remote count:%d\n", local_total, remote_total); ++ if (local_total == 0 || remote_total == 0) { ++ pr_err("no ip list is present.\n"); ++ return; ++ } ++ ++ attach_infos = kzalloc( ++ (total_combinations) * sizeof(struct xprt_attach_info), ++ GFP_KERNEL); ++ if (attach_infos == NULL) { ++ enfs_log_error("Failed to kzalloc memory\n"); ++ return; ++ } ++ // Calculate the least common multiple of local_total and remote_total ++ local_remote_total_lcm = total_combinations / ++ enfs_cal_gcd(local_total, remote_total); ++ ++ // It needs to be offset one for lcm times of ++ // cycle so that all possible link setup method would be traversed ++ for (i = 0; i < total_combinations; i++) { ++ local_index = i % local_total; ++ remote_index = (i + link_count / local_remote_total_lcm) % ++ remote_total; ++ ++ if (enfs_already_have_xprt(clnt, ++ &local->address[local_index], ++ &remote->address[remote_index])) { ++ ++ atomic_dec(&wait_queue_condition); ++ link_count++; ++ continue; ++ ++ } ++ ++ attach_infos[i].localAddress = &local->address[local_index]; ++ attach_infos[i].remoteAddress = &remote->address[remote_index]; ++ attach_infos[i].data = &attach_callback_data; ++ ++ err = enfs_configure_xprt_to_clnt(xprtargs, ++ clnt, &attach_infos[i]); ++ if (err) { ++ pr_err("add xprt ippair err:%d\n", err); ++ atomic_dec(&wait_queue_condition); ++ } ++ link_count++; ++ } ++ ++ wait_event(path_attach_wait_queue, ++ atomic_read(&wait_queue_condition) == 0); ++ ++ enfs_add_xprts_to_clnt(clnt, attach_infos, total_combinations); ++ ++ kfree(attach_infos); ++} ++ ++void enfs_xprt_ippair_create(struct xprt_create *xprtargs, ++ struct rpc_clnt *clnt, ++ void *data) ++{ ++ struct multipath_mount_options *mopt = ++ (struct multipath_mount_options *)data; ++ ++ if (mopt == NULL) { ++ pr_err("ip list is NULL.\n"); ++ return; ++ } ++ ++ enfs_combine_addr(xprtargs, clnt, ++ mopt->local_ip_list, ++ mopt->remote_ip_list); ++ enfs_lb_set_policy(clnt); ++} ++ ++struct xprts_options_and_clnt { ++ struct rpc_create_args *args; ++ struct rpc_clnt *clnt; ++ void *data; ++}; ++ ++static void set_clnt_enfs_flag(struct rpc_clnt *clnt) ++{ ++ clnt->cl_enfs = true; ++} ++ ++int enfs_config_xprt_create_args(struct xprt_create *xprtargs, ++ struct rpc_create_args *args, ++ char *servername, size_t length) ++{ ++ int errno = 0; ++ ++ xprtargs->ident = args->protocol; ++ xprtargs->net = args->net; ++ xprtargs->addrlen = args->addrsize; ++ xprtargs->servername = args->servername; ++ ++ if (args->flags & RPC_CLNT_CREATE_INFINITE_SLOTS) ++ xprtargs->flags |= XPRT_CREATE_INFINITE_SLOTS; ++ if (args->flags & RPC_CLNT_CREATE_NO_IDLE_TIMEOUT) ++ xprtargs->flags |= XPRT_CREATE_NO_IDLE_TIMEOUT; ++ ++ if (xprtargs->servername == NULL) { ++ errno = enfs_servername(servername, length, args); ++ if (errno) ++ return errno; ++ xprtargs->servername = servername; ++ } ++ ++ return 0; ++} ++ ++int enfs_multipath_create_thread(void *data) ++{ ++ int errno; ++ char servername[48]; ++ struct xprts_options_and_clnt *create_args = ++ (struct xprts_options_and_clnt *)data; ++ struct multipath_mount_options *mount_options = ++ (struct multipath_mount_options *) ++ create_args->args->multipath_option; ++ struct xprt_create xprtargs; ++ ++ memset(&xprtargs, 0, sizeof(struct xprt_create)); ++ ++ if (mount_options == NULL) { ++ enfs_log_error("enfs: local and remot addr empty\n"); ++ return -EINVAL; ++ } ++ ++ errno = enfs_config_xprt_create_args(&xprtargs, ++ create_args->args, servername, ++ sizeof(servername)); ++ if (errno) { ++ enfs_log_error("enfs: create xprt failed! errno:%d\n", ++ errno); ++ return errno; ++ } ++ ++ //mount : localaddrs or remoteaddrs is empty ++ if (mount_options->local_ip_list->count == 0) { ++ errno = rpc_localaddr(create_args->clnt, ++ (struct sockaddr *) ++ &mount_options->local_ip_list->address[0], ++ sizeof(struct sockaddr_storage)); ++ if (errno) { ++ enfs_log_error("enfs: get clnt srcaddr errno:%d\n", ++ errno); ++ return errno; ++ } ++ mount_options->local_ip_list->count = 1; ++ } ++ ++ if (mount_options->remote_ip_list->count == 0) { ++ errno = rpc_peeraddr(create_args->clnt, ++ (struct sockaddr *) ++ &mount_options->remote_ip_list->address[0], ++ sizeof(struct sockaddr_storage)); ++ if (errno == 0) { ++ enfs_log_error("enfs: get clnt dstaddr errno:%d\n", ++ errno); ++ return errno; ++ } ++ mount_options->remote_ip_list->count = 1; ++ } ++ ++ errno = enfs_proc_create_clnt(create_args->clnt); ++ if (errno != 0) ++ pr_err("create clnt proc failed.\n"); ++ ++ set_clnt_enfs_flag(create_args->clnt); ++ enfs_xprt_ippair_create(&xprtargs, create_args->clnt, mount_options); ++ ++ kfree(create_args->args); ++ kfree(data); ++ return 0; ++} ++ ++static int set_main_xprt_ctx(struct rpc_clnt *clnt, struct rpc_xprt *xprt, ++ struct sockaddr_storage *srcaddr) ++{ ++ struct enfs_xprt_context *ctx = xprt->multipath_context; ++ ++ if (!ctx) { ++ enfs_log_error("main xprt not multipath ctx.\n"); ++ return -1; ++ } ++ ++ ctx->main = true; ++ ctx->stats = rpc_alloc_iostats(clnt); ++ ctx->srcaddr = *srcaddr; ++ pm_set_path_state(xprt, PM_STATE_NORMAL); ++ pm_ping_set_path_check_state(xprt, PM_CHECK_INIT); ++ ++ return 0; ++} ++ ++static int alloc_main_xprt_multicontext(struct rpc_create_args *args, ++ struct rpc_clnt *clnt) ++{ ++ int err; ++ struct sockaddr_storage srcaddr; ++ ++ // avoid main xprt multicontex local addr is empty. ++ err = rpc_localaddr(clnt, (struct sockaddr *)&srcaddr, sizeof(srcaddr)); ++ if (err) { ++ enfs_log_error("get clnt localaddr err:%d\n", err); ++ return err; ++ } ++ ++ err = set_main_xprt_ctx(clnt, clnt->cl_xprt, &srcaddr); ++ if (err) ++ enfs_log_error("alloc main xprt failed.\n"); ++ ++ return err; ++} ++ ++void enfs_create_multi_xprt(struct rpc_create_args *args, struct rpc_clnt *clnt) ++{ ++ // struct task_struct *th; ++ struct xprts_options_and_clnt *thargs; ++ struct rpc_create_args *cargs; ++ int err; ++ ++ enfs_log_info("%s %p\n", __func__, clnt); ++ ++ cargs = kmalloc(sizeof(struct rpc_create_args), GFP_KERNEL); ++ if (cargs == NULL) ++ return; ++ ++ *cargs = *args; ++ ++ thargs = kmalloc(sizeof(struct xprts_options_and_clnt), GFP_KERNEL); ++ if (thargs == NULL) { ++ kfree(cargs); ++ return; ++ } ++ ++ alloc_main_xprt_multicontext(args, clnt); ++ ++ thargs->args = cargs; ++ thargs->clnt = clnt; ++ thargs->data = args->multipath_option; ++ ++ err = enfs_multipath_create_thread(thargs); ++ ++ if (err != 0) { ++ kfree(cargs); ++ kfree(thargs); ++ } ++} ++ ++static void enfs_create_xprt_ctx(struct rpc_xprt *xprt) ++{ ++ int err; ++ ++ err = enfs_alloc_xprt_ctx(xprt); ++ if (err) ++ enfs_log_error("alloc xprt failed.\n"); ++} ++ ++static struct rpc_multipath_ops ops = { ++ .owner = THIS_MODULE, ++ .create_clnt = enfs_create_multi_xprt, ++ .releas_clnt = enfs_proc_delete_clnt, ++ .create_xprt = enfs_create_xprt_ctx, ++ .destroy_xprt = enfs_free_xprt_ctx, ++ .xprt_iostat = enfs_count_iostat, ++ .failover_handle = failover_handle, ++ .task_need_call_start_again = failover_task_need_call_start_again, ++ .adjust_task_timeout = failover_adjust_task_timeout, ++ .init_task_req = failover_init_task_req, ++ .prepare_transmit = failover_prepare_transmit, ++}; ++ ++int enfs_multipath_init(void) ++{ ++ int err; ++ ++ enfs_log_info("multipath init.\n"); ++ ++ err = pm_ping_init(); ++ if (err != 0) { ++ enfs_log_error("pm ping init err:%d\n", err); ++ return err; ++ } ++ ++ err = enfs_proc_init(); ++ if (err != 0) { ++ enfs_log_error("enfs proc init err:%d\n", err); ++ pm_ping_fini(); ++ return err; ++ } ++ ++ rpc_multipath_ops_register(&ops); ++ ++ return 0; ++} ++ ++void enfs_multipath_exit(void) ++{ ++ enfs_log_info("multipath exit.\n"); ++ rpc_multipath_ops_unregister(&ops); ++ enfs_proc_exit(); ++ pm_ping_fini(); ++} +diff --git a/fs/nfs/enfs/enfs_multipath.h b/fs/nfs/enfs/enfs_multipath.h +new file mode 100644 +index 000000000..3c58b0c4c +--- /dev/null ++++ b/fs/nfs/enfs/enfs_multipath.h +@@ -0,0 +1,23 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: enfs multipath ++ * Create: 2023-07-31 ++ */ ++ ++#ifndef ENFS_MULTIPATH_H ++#define ENFS_MULTIPATH_H ++#include ++ ++#define MAX_XPRT_NUM_PER_CLIENT 32 ++ ++int enfs_multipath_init(void); ++void enfs_multipath_exit(void); ++void enfs_xprt_ippair_create(struct xprt_create *xprtargs, ++ struct rpc_clnt *clnt, void *data); ++int enfs_config_xprt_create_args(struct xprt_create *xprtargs, ++ struct rpc_create_args *args, ++ char *servername, size_t length); ++void print_enfs_multipath_addr(struct sockaddr *local, struct sockaddr *remote); ++ ++#endif // ENFS_MULTIPATH_H +diff --git a/fs/nfs/enfs/enfs_path.c b/fs/nfs/enfs/enfs_path.c +new file mode 100644 +index 000000000..7355f8c2f +--- /dev/null ++++ b/fs/nfs/enfs/enfs_path.c +@@ -0,0 +1,47 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ */ ++ ++#include ++#include ++ ++#include "enfs.h" ++#include "enfs_log.h" ++#include "enfs_path.h" ++ ++// only create ctx in this function ++// alloc iostat memory in create_clnt ++int enfs_alloc_xprt_ctx(struct rpc_xprt *xprt) ++{ ++ struct enfs_xprt_context *ctx; ++ ++ if (!xprt) { ++ enfs_log_error("invalid xprt pointer.\n"); ++ return -EINVAL; ++ } ++ ++ ctx = kzalloc(sizeof(struct enfs_xprt_context), GFP_KERNEL); ++ if (!ctx) { ++ enfs_log_error("add xprt test failed.\n"); ++ return -ENOMEM; ++ } ++ ++ xprt->multipath_context = (void *)ctx; ++ return 0; ++} ++ ++// free multi_context and iostat memory ++void enfs_free_xprt_ctx(struct rpc_xprt *xprt) ++{ ++ struct enfs_xprt_context *ctx = xprt->multipath_context; ++ ++ if (ctx) { ++ if (ctx->stats) { ++ rpc_free_iostats(ctx->stats); ++ ctx->stats = NULL; ++ } ++ kfree(xprt->multipath_context); ++ xprt->multipath_context = NULL; ++ } ++} +diff --git a/fs/nfs/enfs/enfs_path.h b/fs/nfs/enfs/enfs_path.h +new file mode 100644 +index 000000000..97b1ef373 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_path.h +@@ -0,0 +1,12 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ */ ++ ++#ifndef ENFS_PATH_H ++#define ENFS_PATH_H ++ ++int enfs_alloc_xprt_ctx(struct rpc_xprt *xprt); ++void enfs_free_xprt_ctx(struct rpc_xprt *xprt); ++ ++#endif // ENFS_PATH_H +diff --git a/fs/nfs/enfs/enfs_proc.c b/fs/nfs/enfs/enfs_proc.c +new file mode 100644 +index 000000000..53fa1a076 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_proc.c +@@ -0,0 +1,545 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "../../../net/sunrpc/netns.h" ++ ++#include "enfs.h" ++#include "enfs_log.h" ++#include "enfs_proc.h" ++#include "enfs_multipath.h" ++#include "pm_state.h" ++ ++#define ENFS_PROC_DIR "enfs" ++#define ENFS_PROC_PATH_STATUS_LEN 256 ++ ++static struct proc_dir_entry *enfs_proc_parent; ++ ++void ++enfs_iterate_each_rpc_clnt(int (*fn)(struct rpc_clnt *clnt, void *data), ++ void *data) ++{ ++ struct net *net; ++ struct sunrpc_net *sn; ++ struct rpc_clnt *clnt; ++ ++ rcu_read_lock(); ++ for_each_net_rcu(net) { ++ sn = net_generic(net, sunrpc_net_id); ++ if (sn == NULL) ++ continue; ++ spin_lock(&sn->rpc_client_lock); ++ list_for_each_entry(clnt, &sn->all_clients, cl_clients) { ++ fn(clnt, data); ++ } ++ spin_unlock(&sn->rpc_client_lock); ++ } ++ rcu_read_unlock(); ++} ++ ++struct proc_dir_entry *enfs_get_proc_parent(void) ++{ ++ return enfs_proc_parent; ++} ++ ++static int sockaddr_ip_to_str(struct sockaddr *addr, char *buf, int len) ++{ ++ switch (addr->sa_family) { ++ case AF_INET: { ++ struct sockaddr_in *sin = (struct sockaddr_in *)addr; ++ ++ snprintf(buf, len, "%pI4", &sin->sin_addr); ++ return 0; ++ } ++ case AF_INET6: { ++ struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)addr; ++ ++ snprintf(buf, len, "%pI6", &sin6->sin6_addr); ++ return 0; ++ } ++ default: ++ break; ++ } ++ return 1; ++} ++ ++static bool should_print(const char *name) ++{ ++ int i; ++ static const char * const proc_names[] = { ++ "READ", ++ "WRITE", ++ }; ++ ++ if (name == NULL) ++ return false; ++ ++ for (i = 0; i < ARRAY_SIZE(proc_names); i++) { ++ if (strcmp(name, proc_names[i]) == 0) ++ return true; ++ } ++ return false; ++} ++ ++struct enfs_xprt_iter { ++ unsigned int id; ++ struct seq_file *seq; ++ unsigned int max_addrs_length; ++}; ++ ++static int debug_show_xprt(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, ++ void *data) ++{ ++ struct enfs_xprt_context *ctx = NULL; ++ ++ if (xprt->multipath_context) ++ ctx = xprt->multipath_context; ++ ++ pr_info(" xprt:%p ctx:%p main:%d queue_len:%lu.\n", xprt, ++ xprt->multipath_context, ++ ctx ? ctx->main : false, ++ atomic_long_read(&xprt->queuelen)); ++ return 0; ++} ++ ++static int debug_show_clnt(struct rpc_clnt *clnt, void *data) ++{ ++ pr_info(" clnt %d addr:%p enfs:%d\n", ++ clnt->cl_clid, clnt, ++ clnt->cl_enfs); ++ rpc_clnt_iterate_for_each_xprt(clnt, debug_show_xprt, NULL); ++ return 0; ++} ++ ++static void debug_print_all_xprt(void) ++{ ++ enfs_iterate_each_rpc_clnt(debug_show_clnt, NULL); ++} ++ ++static ++void enfs_proc_format_xprt_addr_display(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, ++ char *local_name_buf, ++ int local_name_buf_len, ++ char *remote_name_buf, ++ int remote_name_buf_len) ++{ ++ int err; ++ struct sockaddr_storage srcaddr; ++ struct enfs_xprt_context *ctx; ++ ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ ++ sockaddr_ip_to_str((struct sockaddr *)&xprt->addr, ++ remote_name_buf, remote_name_buf_len); ++ ++ // get local address depend one main or not ++ if (enfs_is_main_xprt(xprt)) { ++ err = rpc_localaddr(clnt, (struct sockaddr *)&srcaddr, ++ sizeof(srcaddr)); ++ if (err != 0) ++ (void)snprintf(local_name_buf, ++ local_name_buf_len, "Unknown"); ++ else ++ sockaddr_ip_to_str((struct sockaddr *)&srcaddr, ++ local_name_buf, ++ local_name_buf_len); ++ } else { ++ sockaddr_ip_to_str((struct sockaddr *)&ctx->srcaddr, ++ local_name_buf, ++ local_name_buf_len); ++ } ++} ++ ++static int enfs_show_xprt_stats(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, ++ void *data) ++{ ++ unsigned int op; ++ unsigned int maxproc = clnt->cl_maxproc; ++ struct enfs_xprt_iter *iter = (struct enfs_xprt_iter *)data; ++ struct enfs_xprt_context *ctx; ++ char local_name[INET6_ADDRSTRLEN]; ++ char remote_name[INET6_ADDRSTRLEN]; ++ ++ if (!xprt->multipath_context) ++ return 0; ++ ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ ++ enfs_proc_format_xprt_addr_display(clnt, xprt, local_name, ++ sizeof(local_name), ++ remote_name, sizeof(remote_name)); ++ ++ seq_printf(iter->seq, "%-6u%-*s%-*s", iter->id, ++ iter->max_addrs_length + 4, ++ local_name, ++ iter->max_addrs_length + 4, ++ remote_name); ++ ++ iter->id++; ++ ++ for (op = 0; op < maxproc; op++) { ++ if (!should_print(clnt->cl_procinfo[op].p_name)) ++ continue; ++ ++ seq_printf(iter->seq, "%-22lu%-22Lu%-22Lu", ++ ctx->stats[op].om_ops, ++ ctx->stats[op].om_ops == 0 ? 0 : ++ ktime_to_ms(ctx->stats[op].om_rtt) / ++ ctx->stats[op].om_ops, ++ ctx->stats[op].om_ops == 0 ? 0 : ++ ktime_to_ms(ctx->stats[op].om_execute) / ++ ctx->stats[op].om_ops); ++ } ++ seq_puts(iter->seq, "\n"); ++ return 0; ++} ++ ++static int rpc_proc_show_path_status(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, ++ void *data) ++{ ++ struct enfs_xprt_iter *iter = (struct enfs_xprt_iter *)data; ++ struct enfs_xprt_context *ctx = NULL; ++ char local_name[INET6_ADDRSTRLEN] = {0}; ++ char remote_name[INET6_ADDRSTRLEN] = {0}; ++ char multiapth_status[ENFS_PROC_PATH_STATUS_LEN] = {0}; ++ char xprt_status[ENFS_PROC_PATH_STATUS_LEN] = {0}; ++ ++ if (!xprt->multipath_context) { ++ enfs_log_debug("multipath_context is null.\n"); ++ return 0; ++ } ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ ++ enfs_proc_format_xprt_addr_display(clnt, xprt, ++ local_name, ++ sizeof(local_name), ++ remote_name, sizeof(remote_name)); ++ ++ pm_get_path_state_desc(xprt, ++ multiapth_status, ++ ENFS_PROC_PATH_STATUS_LEN); ++ ++ pm_get_xprt_state_desc(xprt, ++ xprt_status, ++ ENFS_PROC_PATH_STATUS_LEN); ++ ++ seq_printf(iter->seq, "%-6u%-*s%-*s%-12s%-12s\n", ++ iter->id, iter->max_addrs_length + 4, ++ local_name, iter->max_addrs_length + 4, ++ remote_name, multiapth_status, ++ xprt_status); ++ iter->id++; ++ return 0; ++} ++ ++static int enfs_get_max_addrs_length(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, ++ void *data) ++{ ++ struct enfs_xprt_iter *iter = (struct enfs_xprt_iter *)data; ++ char local_name[INET6_ADDRSTRLEN]; ++ char remote_name[INET6_ADDRSTRLEN]; ++ ++ enfs_proc_format_xprt_addr_display(clnt, xprt, ++ local_name, sizeof(local_name), ++ remote_name, sizeof(remote_name)); ++ ++ if (iter->max_addrs_length < strlen(local_name)) ++ iter->max_addrs_length = strlen(local_name); ++ ++ if (iter->max_addrs_length < strlen(remote_name)) ++ iter->max_addrs_length = strlen(remote_name); ++ ++ return 0; ++} ++ ++static int rpc_proc_clnt_showpath(struct seq_file *seq, void *v) ++{ ++ struct rpc_clnt *clnt = seq->private; ++ struct enfs_xprt_iter iter; ++ ++ iter.seq = seq; ++ iter.id = 0; ++ iter.max_addrs_length = 0; ++ ++ rpc_clnt_iterate_for_each_xprt(clnt, ++ enfs_get_max_addrs_length, ++ (void *)&iter); ++ ++ seq_printf(seq, "%-6s%-*s%-*s%-12s%-12s\n", "id", ++ iter.max_addrs_length + 4, ++ "local_addr", ++ iter.max_addrs_length + 4, ++ "remote_addr", ++ "path_state", ++ "xprt_state"); ++ ++ rpc_clnt_iterate_for_each_xprt(clnt, ++ rpc_proc_show_path_status, ++ (void *)&iter); ++ return 0; ++} ++ ++static int enfs_rpc_proc_show(struct seq_file *seq, void *v) ++{ ++ struct rpc_clnt *clnt = seq->private; ++ struct enfs_xprt_iter iter; ++ ++ iter.seq = seq; ++ iter.id = 0; ++ iter.max_addrs_length = 0; ++ ++ debug_print_all_xprt(); ++ pr_info("enfs proc clnt:%p\n", clnt); ++ ++ rpc_clnt_iterate_for_each_xprt(clnt, ++ enfs_get_max_addrs_length, ++ (void *)&iter); ++ ++ seq_printf(seq, "%-6s%-*s%-*s%-22s%-22s%-22s%-22s%-22s%-22s\n", "id", ++ iter.max_addrs_length + 4, "local_addr", ++ iter.max_addrs_length + 4, ++ "remote_addr", "r_count", ++ "r_rtt", "r_exec", "w_count", "w_rtt", "w_exec"); ++ ++ // rpc_clnt_show_stats(seq, clnt); ++ rpc_clnt_iterate_for_each_xprt(clnt, ++ enfs_show_xprt_stats, ++ (void *)&iter); ++ return 0; ++} ++ ++static int rpc_proc_open(struct inode *inode, struct file *file) ++{ ++ struct rpc_clnt *clnt = PDE_DATA(inode); ++ ++ pr_info("%s %p\n", __func__, clnt); ++ return single_open(file, enfs_rpc_proc_show, clnt); ++} ++ ++static int enfs_reset_xprt_stats(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, ++ void *data) ++{ ++ unsigned int op; ++ struct enfs_xprt_context *ctx; ++ unsigned int maxproc = clnt->cl_maxproc; ++ struct rpc_iostats stats = {0}; ++ ++ if (!xprt->multipath_context) ++ return 0; ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ ++ for (op = 0; op < maxproc; op++) { ++ spin_lock(&ctx->stats[op].om_lock); ++ ctx->stats[op] = stats; ++ spin_unlock(&ctx->stats[op].om_lock); ++ } ++ return 0; ++} ++ ++static void trim_newline_ch(char *str, int len) ++{ ++ int i; ++ ++ for (i = 0; str[i] != '\0' && i < len; i++) { ++ if (str[i] == '\n') ++ str[i] = '\0'; ++ } ++} ++ ++static ssize_t enfs_proc_write(struct file *file, ++ const char __user *user_buf, ++ size_t len, ++ loff_t *offset) ++{ ++ char buffer[128]; ++ struct rpc_clnt *clnt = ++ ((struct seq_file *)file->private_data)->private; ++ ++ if (len >= sizeof(buffer)) ++ return -E2BIG; ++ ++ if (copy_from_user(buffer, user_buf, len) != 0) ++ return -EFAULT; ++ ++ buffer[len] = '\0'; ++ trim_newline_ch(buffer, len); ++ if (strcmp(buffer, "reset") != 0) ++ return -EINVAL; ++ ++ rpc_clnt_iterate_for_each_xprt(clnt, enfs_reset_xprt_stats, NULL); ++ return len; ++} ++ ++static int rpc_proc_show_path(struct inode *inode, struct file *file) ++{ ++ struct rpc_clnt *clnt = PDE_DATA(inode); ++ ++ return single_open(file, rpc_proc_clnt_showpath, clnt); ++} ++ ++static const struct file_operations rpc_proc_fops = { ++ .owner = THIS_MODULE, ++ .open = rpc_proc_open, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++ .write = enfs_proc_write, ++}; ++ ++static const struct file_operations rpc_show_path_fops = { ++ .owner = THIS_MODULE, ++ .open = rpc_proc_show_path, ++ .read = seq_read, ++ .llseek = seq_lseek, ++ .release = single_release, ++}; ++ ++static int clnt_proc_name(struct rpc_clnt *clnt, char *buf, int len) ++{ ++ int ret; ++ ++ ret = snprintf(buf, len, "%s_%u", ++ rpc_peeraddr2str(clnt, RPC_DISPLAY_ADDR), ++ clnt->cl_clid); ++ if (ret > len) ++ return -E2BIG; ++ return 0; ++} ++ ++static int enfs_proc_create_file(struct rpc_clnt *clnt) ++{ ++ int err; ++ char buf[128]; ++ ++ struct proc_dir_entry *clnt_entry; ++ struct proc_dir_entry *stat_entry; ++ ++ err = clnt_proc_name(clnt, buf, sizeof(buf)); ++ if (err) ++ return err; ++ ++ clnt_entry = proc_mkdir(buf, enfs_proc_parent); ++ if (clnt_entry == NULL) ++ return -EINVAL; ++ ++ stat_entry = proc_create_data("stat", ++ 0, clnt_entry, ++ &rpc_proc_fops, clnt); ++ ++ if (stat_entry == NULL) ++ return -EINVAL; ++ ++ stat_entry = proc_create_data("path", ++ 0, clnt_entry, ++ &rpc_show_path_fops, clnt); ++ ++ if (stat_entry == NULL) ++ return -EINVAL; ++ ++ return 0; ++} ++ ++void enfs_count_iostat(struct rpc_task *task) ++{ ++ struct enfs_xprt_context *ctx = task->tk_xprt->multipath_context; ++ ++ if (!ctx || !ctx->stats) ++ return; ++ rpc_count_iostats(task, ctx->stats); ++} ++ ++static void enfs_proc_delete_file(struct rpc_clnt *clnt) ++{ ++ int err; ++ char buf[128]; ++ ++ err = clnt_proc_name(clnt, buf, sizeof(buf)); ++ if (err) { ++ pr_err("gen clnt name failed.\n"); ++ return; ++ } ++ remove_proc_subtree(buf, enfs_proc_parent); ++} ++ ++// create proc file "/porc/enfs/[mount_ip]_[id]/stat" ++int enfs_proc_create_clnt(struct rpc_clnt *clnt) ++{ ++ int err; ++ ++ err = enfs_proc_create_file(clnt); ++ if (err) { ++ pr_err("create client %d\n", err); ++ return err; ++ } ++ ++ return 0; ++} ++ ++void enfs_proc_delete_clnt(struct rpc_clnt *clnt) ++{ ++ if (clnt->cl_enfs) ++ enfs_proc_delete_file(clnt); ++} ++ ++static int enfs_proc_create_parent(void) ++{ ++ enfs_proc_parent = proc_mkdir(ENFS_PROC_DIR, NULL); ++ ++ if (enfs_proc_parent == NULL) { ++ pr_err("Enfs create proc dir err\n"); ++ return -ENOMEM; ++ } ++ return 0; ++} ++ ++static void enfs_proc_delete_parent(void) ++{ ++ remove_proc_entry(ENFS_PROC_DIR, NULL); ++} ++ ++static int enfs_proc_init_create_clnt(struct rpc_clnt *clnt, void *data) ++{ ++ if (clnt->cl_enfs) ++ enfs_proc_create_file(clnt); ++ return 0; ++} ++ ++static int enfs_proc_destroy_clnt(struct rpc_clnt *clnt, void *data) ++{ ++ if (clnt->cl_enfs) ++ enfs_proc_delete_file(clnt); ++ return 0; ++} ++ ++int enfs_proc_init(void) ++{ ++ int err; ++ ++ err = enfs_proc_create_parent(); ++ if (err) ++ return err; ++ ++ enfs_iterate_each_rpc_clnt(enfs_proc_init_create_clnt, NULL); ++ return 0; ++} ++ ++void enfs_proc_exit(void) ++{ ++ enfs_iterate_each_rpc_clnt(enfs_proc_destroy_clnt, NULL); ++ enfs_proc_delete_parent(); ++} +diff --git a/fs/nfs/enfs/enfs_proc.h b/fs/nfs/enfs/enfs_proc.h +new file mode 100644 +index 000000000..321951031 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_proc.h +@@ -0,0 +1,21 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Client-side ENFS PROC. ++ * ++ * Copyright (c) 2023. Huawei Technologies Co., Ltd. All rights reserved. ++ */ ++#ifndef ENFS_PROC_H ++#define ENFS_PROC_H ++ ++struct rpc_clnt; ++struct rpc_task; ++struct proc_dir_entry; ++ ++int enfs_proc_init(void); ++void enfs_proc_exit(void); ++struct proc_dir_entry *enfs_get_proc_parent(void); ++int enfs_proc_create_clnt(struct rpc_clnt *clnt); ++void enfs_proc_delete_clnt(struct rpc_clnt *clnt); ++void enfs_count_iostat(struct rpc_task *task); ++ ++#endif +diff --git a/fs/nfs/enfs/enfs_remount.c b/fs/nfs/enfs/enfs_remount.c +new file mode 100644 +index 000000000..3b7d6a03d +--- /dev/null ++++ b/fs/nfs/enfs/enfs_remount.c +@@ -0,0 +1,220 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: remount ip source file ++ * Create: 2023-08-12 ++ */ ++#include "enfs_remount.h" ++ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "enfs.h" ++#include "enfs_log.h" ++#include "enfs_multipath.h" ++#include "enfs_multipath_parse.h" ++#include "enfs_path.h" ++#include "enfs_proc.h" ++#include "enfs_multipath_client.h" ++ ++static bool enfs_rpc_xprt_switch_need_delete_addr( ++ struct multipath_mount_options *enfs_option, ++ struct sockaddr *dstaddr, struct sockaddr *srcaddr) ++{ ++ int i; ++ bool find_same_ip = false; ++ int32_t local_total; ++ int32_t remote_total; ++ ++ local_total = enfs_option->local_ip_list->count; ++ remote_total = enfs_option->remote_ip_list->count; ++ if (local_total == 0 || remote_total == 0) { ++ pr_err("no ip list is present.\n"); ++ return false; ++ } ++ ++ for (i = 0; i < local_total; i++) { ++ find_same_ip = ++ rpc_cmp_addr((struct sockaddr *) ++ &enfs_option->local_ip_list->address[i], ++ srcaddr); ++ if (find_same_ip) ++ break; ++ } ++ ++ if (find_same_ip == false) ++ return true; ++ ++ find_same_ip = false; ++ for (i = 0; i < remote_total; i++) { ++ find_same_ip = ++ rpc_cmp_addr((struct sockaddr *) ++ &enfs_option->remote_ip_list->address[i], ++ dstaddr); ++ if (find_same_ip) ++ break; ++ } ++ ++ if (find_same_ip == false) ++ return true; ++ ++ return false; ++} ++ ++// Used in rcu_lock ++static bool enfs_delete_xprt_from_switch(struct rpc_xprt *xprt, ++ void *enfs_option, ++ struct rpc_xprt_switch *xps) ++{ ++ struct enfs_xprt_context *ctx = NULL; ++ struct multipath_mount_options *mopt = ++ (struct multipath_mount_options *)enfs_option; ++ ++ if (enfs_is_main_xprt(xprt)) ++ return true; ++ ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ if (enfs_rpc_xprt_switch_need_delete_addr(mopt, ++ (struct sockaddr *)&xprt->addr, ++ (struct sockaddr *)&ctx->srcaddr)) { ++ ++ print_enfs_multipath_addr((struct sockaddr *)&ctx->srcaddr, ++ (struct sockaddr *)&xprt->addr); ++ rpc_xprt_switch_remove_xprt(xps, xprt); ++ return true; ++ } ++ ++ return false; ++} ++ ++void enfs_clnt_delete_obsolete_xprts(struct nfs_client *nfs_client, ++ void *enfs_option) ++{ ++ int xprt_count = 0; ++ struct rpc_xprt *pos = NULL; ++ struct rpc_xprt_switch *xps = NULL; ++ ++ rcu_read_lock(); ++ xps = xprt_switch_get( ++ rcu_dereference( ++ nfs_client->cl_rpcclient->cl_xpi.xpi_xpswitch)); ++ if (xps == NULL) { ++ rcu_read_unlock(); ++ xprt_switch_put(xps); ++ return; ++ } ++ list_for_each_entry_rcu(pos, &xps->xps_xprt_list, xprt_switch) { ++ if (xprt_count < MAX_XPRT_NUM_PER_CLIENT) { ++ if (enfs_delete_xprt_from_switch( ++ pos, enfs_option, xps) == false) ++ xprt_count++; ++ } else ++ rpc_xprt_switch_remove_xprt(xps, pos); ++ } ++ rcu_read_unlock(); ++ xprt_switch_put(xps); ++} ++ ++int enfs_remount_iplist(struct nfs_client *nfs_client, void *enfs_option) ++{ ++ int errno = 0; ++ char servername[48]; ++ struct multipath_mount_options *remount_lists = ++ (struct multipath_mount_options *)enfs_option; ++ struct multipath_client_info *client_info = ++ (struct multipath_client_info *)nfs_client->cl_multipath_data; ++ struct xprt_create xprtargs; ++ struct rpc_create_args args = { ++ .protocol = nfs_client->cl_proto, ++ .net = nfs_client->cl_net, ++ .addrsize = nfs_client->cl_addrlen, ++ .servername = nfs_client->cl_hostname, ++ }; ++ ++ memset(&xprtargs, 0, sizeof(struct xprt_create)); ++ ++ //mount is not use multipath ++ if (client_info == NULL || enfs_option == NULL) { ++ enfs_log_error( ++ "mount information or remount information is empty.\n"); ++ return -EINVAL; ++ } ++ ++ //remount : localaddrs and remoteaddrs are empty ++ if (remount_lists->local_ip_list->count == 0 && ++ remount_lists->remote_ip_list->count == 0) { ++ enfs_log_info("remount local_ip_list and remote_ip_list are NULL\n"); ++ return 0; ++ } ++ ++ errno = enfs_config_xprt_create_args(&xprtargs, ++ &args, servername, sizeof(servername)); ++ ++ if (errno) { ++ enfs_log_error("config_xprt_create failed! errno:%d\n", errno); ++ return errno; ++ } ++ ++ if (remount_lists->local_ip_list->count == 0) { ++ if (client_info->local_ip_list->count == 0) { ++ errno = rpc_localaddr(nfs_client->cl_rpcclient, ++ (struct sockaddr *) ++ &remount_lists->local_ip_list->address[0], ++ sizeof(struct sockaddr_storage)); ++ if (errno) { ++ enfs_log_error("get clnt srcaddr errno:%d\n", ++ errno); ++ return errno; ++ } ++ remount_lists->local_ip_list->count = 1; ++ } else ++ memcpy(remount_lists->local_ip_list, ++ client_info->local_ip_list, ++ sizeof(struct nfs_ip_list)); ++ } ++ ++ if (remount_lists->remote_ip_list->count == 0) { ++ if (client_info->remote_ip_list->count == 0) { ++ errno = rpc_peeraddr(nfs_client->cl_rpcclient, ++ (struct sockaddr *) ++ &remount_lists->remote_ip_list->address[0], ++ sizeof(struct sockaddr_storage)); ++ if (errno == 0) { ++ enfs_log_error("get clnt dstaddr errno:%d\n", ++ errno); ++ return errno; ++ } ++ remount_lists->remote_ip_list->count = 1; ++ } else ++ memcpy(remount_lists->remote_ip_list, ++ client_info->remote_ip_list, ++ sizeof(struct nfs_ip_list)); ++ } ++ ++ enfs_log_info("Remount creating new links...\n"); ++ enfs_xprt_ippair_create(&xprtargs, ++ nfs_client->cl_rpcclient, ++ remount_lists); ++ ++ enfs_log_info("Remount deleting obsolete links...\n"); ++ enfs_clnt_delete_obsolete_xprts(nfs_client, remount_lists); ++ ++ memcpy(client_info->local_ip_list, ++ remount_lists->local_ip_list, ++ sizeof(struct nfs_ip_list)); ++ memcpy(client_info->remote_ip_list, ++ remount_lists->remote_ip_list, ++ sizeof(struct nfs_ip_list)); ++ ++ return 0; ++} +diff --git a/fs/nfs/enfs/enfs_remount.h b/fs/nfs/enfs/enfs_remount.h +new file mode 100644 +index 000000000..c7a1dcbb6 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_remount.h +@@ -0,0 +1,14 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: remount ip header file ++ * Create: 2023-08-12 ++ */ ++#ifndef _ENFS_REMOUNT_ ++#define _ENFS_REMOUNT_ ++#include ++#include "enfs.h" ++ ++int enfs_remount_iplist(struct nfs_client *nfs_client, void *enfs_option); ++ ++#endif +diff --git a/fs/nfs/enfs/enfs_roundrobin.c b/fs/nfs/enfs/enfs_roundrobin.c +new file mode 100644 +index 000000000..4e4eda784 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_roundrobin.c +@@ -0,0 +1,255 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include "enfs_roundrobin.h" ++ ++#include "enfs.h" ++#include "enfs_config.h" ++#include "pm_state.h" ++ ++typedef struct rpc_xprt *(*enfs_xprt_switch_find_xprt_t)( ++ struct rpc_xprt_switch *xps, const struct rpc_xprt *cur); ++static const struct rpc_xprt_iter_ops enfs_xprt_iter_roundrobin; ++static const struct rpc_xprt_iter_ops enfs_xprt_iter_singular; ++ ++static bool enfs_xprt_is_active(struct rpc_xprt *xprt) ++{ ++ enum pm_path_state state; ++ ++ if (kref_read(&xprt->kref) <= 0) ++ return false; ++ ++ state = pm_get_path_state(xprt); ++ if (state == PM_STATE_NORMAL) ++ return true; ++ ++ return false; ++} ++ ++static struct rpc_xprt *enfs_lb_set_cursor_xprt( ++ struct rpc_xprt_switch *xps, struct rpc_xprt **cursor, ++ enfs_xprt_switch_find_xprt_t find_next) ++{ ++ struct rpc_xprt *pos; ++ struct rpc_xprt *old; ++ ++ old = smp_load_acquire(cursor); /* read latest cursor */ ++ pos = find_next(xps, old); ++ smp_store_release(cursor, pos); /* let cursor point to pos */ ++ return pos; ++} ++ ++static ++struct rpc_xprt *enfs_lb_find_next_entry_roundrobin( ++ struct rpc_xprt_switch *xps, const struct rpc_xprt *cur) ++{ ++ struct rpc_xprt *pos; ++ struct rpc_xprt *prev = NULL; ++ bool found = false; ++ struct rpc_xprt *min_queuelen_xprt = NULL; ++ unsigned long pos_xprt_queuelen; ++ unsigned long min_xprt_queuelen = 0; ++ ++ unsigned long xps_queuelen = atomic_long_read(&xps->xps_queuelen); ++ // delete origin xprt ++ unsigned int multipath_nactive = READ_ONCE(xps->xps_nactive) - 1; ++ ++ list_for_each_entry_rcu(pos, &xps->xps_xprt_list, xprt_switch) { ++ if (enfs_is_main_xprt(pos) || !enfs_xprt_is_active(pos)) { ++ prev = pos; ++ continue; ++ } ++ ++ pos_xprt_queuelen = atomic_long_read(&pos->queuelen); ++ if (min_queuelen_xprt == NULL || ++ pos_xprt_queuelen < min_xprt_queuelen) { ++ ++ min_queuelen_xprt = pos; ++ min_xprt_queuelen = pos_xprt_queuelen; ++ } ++ ++ if (cur == prev) ++ found = true; ++ ++ if (found && pos_xprt_queuelen * ++ multipath_nactive <= xps_queuelen) ++ return pos; ++ prev = pos; ++ }; ++ ++ return min_queuelen_xprt; ++} ++ ++struct rpc_xprt *enfs_lb_switch_find_first_active_xprt( ++ struct rpc_xprt_switch *xps) ++{ ++ struct rpc_xprt *pos; ++ ++ list_for_each_entry_rcu(pos, &xps->xps_xprt_list, xprt_switch) { ++ if (enfs_xprt_is_active(pos)) ++ return pos; ++ }; ++ return NULL; ++} ++ ++struct rpc_xprt *enfs_lb_switch_get_main_xprt(struct rpc_xprt_switch *xps) ++{ ++ return list_first_or_null_rcu(&xps->xps_xprt_list, ++ struct rpc_xprt, xprt_switch); ++} ++ ++static struct rpc_xprt *enfs_lb_switch_get_next_xprt_roundrobin( ++ struct rpc_xprt_switch *xps, const struct rpc_xprt *cur) ++{ ++ struct rpc_xprt *xprt; ++ ++ // disable multipath ++ if (enfs_get_config_multipath_state()) ++ return enfs_lb_switch_get_main_xprt(xps); ++ ++ xprt = enfs_lb_find_next_entry_roundrobin(xps, cur); ++ if (xprt != NULL) ++ return xprt; ++ ++ return enfs_lb_switch_get_main_xprt(xps); ++} ++ ++static ++struct rpc_xprt *enfs_lb_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi) ++{ ++ struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); ++ ++ if (xps == NULL) ++ return NULL; ++ ++ return enfs_lb_set_cursor_xprt(xps, &xpi->xpi_cursor, ++ enfs_lb_switch_get_next_xprt_roundrobin); ++} ++ ++static ++struct rpc_xprt *enfs_lb_switch_find_singular_entry( ++ struct rpc_xprt_switch *xps, const struct rpc_xprt *cur) ++{ ++ struct rpc_xprt *pos; ++ bool found = false; ++ ++ list_for_each_entry_rcu(pos, &xps->xps_xprt_list, xprt_switch) { ++ if (cur == pos) ++ found = true; ++ ++ if (found && enfs_xprt_is_active(pos)) ++ return pos; ++ } ++ return NULL; ++} ++ ++struct rpc_xprt *enfs_lb_get_singular_xprt( ++ struct rpc_xprt_switch *xps, const struct rpc_xprt *cur) ++{ ++ struct rpc_xprt *xprt; ++ ++ if (xps == NULL) ++ return NULL; ++ ++ // disable multipath ++ if (enfs_get_config_multipath_state()) ++ return enfs_lb_switch_get_main_xprt(xps); ++ ++ if (cur == NULL || xps->xps_nxprts < 2) ++ return enfs_lb_switch_find_first_active_xprt(xps); ++ ++ xprt = enfs_lb_switch_find_singular_entry(xps, cur); ++ if (!xprt) ++ return enfs_lb_switch_get_main_xprt(xps); ++ ++ return xprt; ++} ++ ++static ++struct rpc_xprt *enfs_lb_iter_next_entry_sigular(struct rpc_xprt_iter *xpi) ++{ ++ struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); ++ ++ if (xps == NULL) ++ return NULL; ++ ++ return enfs_lb_set_cursor_xprt(xps, &xpi->xpi_cursor, ++ enfs_lb_get_singular_xprt); ++} ++ ++static void enfs_lb_iter_default_rewind(struct rpc_xprt_iter *xpi) ++{ ++ WRITE_ONCE(xpi->xpi_cursor, NULL); ++} ++ ++static void enfs_lb_switch_set_roundrobin(struct rpc_clnt *clnt) ++{ ++ struct rpc_xprt_switch *xps; ++ ++ rcu_read_lock(); ++ xps = rcu_dereference(clnt->cl_xpi.xpi_xpswitch); ++ rcu_read_unlock(); ++ if (clnt->cl_vers == 3) { ++ ++ if (READ_ONCE(xps->xps_iter_ops) != &enfs_xprt_iter_roundrobin) ++ WRITE_ONCE(xps->xps_iter_ops, ++ &enfs_xprt_iter_roundrobin); ++ ++ return; ++ } ++ if (READ_ONCE(xps->xps_iter_ops) != &enfs_xprt_iter_singular) ++ WRITE_ONCE(xps->xps_iter_ops, &enfs_xprt_iter_singular); ++} ++ ++static ++struct rpc_xprt *enfs_lb_switch_find_current(struct list_head *head, ++ const struct rpc_xprt *cur) ++{ ++ struct rpc_xprt *pos; ++ ++ list_for_each_entry_rcu(pos, head, xprt_switch) { ++ if (cur == pos) ++ return pos; ++ } ++ return NULL; ++} ++ ++static struct rpc_xprt *enfs_lb_iter_current_entry(struct rpc_xprt_iter *xpi) ++{ ++ struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch); ++ struct list_head *head; ++ ++ if (xps == NULL) ++ return NULL; ++ head = &xps->xps_xprt_list; ++ if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2) ++ return enfs_lb_switch_get_main_xprt(xps); ++ return enfs_lb_switch_find_current(head, xpi->xpi_cursor); ++} ++ ++void enfs_lb_set_policy(struct rpc_clnt *clnt) ++{ ++ enfs_lb_switch_set_roundrobin(clnt); ++} ++ ++static const struct rpc_xprt_iter_ops enfs_xprt_iter_roundrobin = { ++ .xpi_rewind = enfs_lb_iter_default_rewind, ++ .xpi_xprt = enfs_lb_iter_current_entry, ++ .xpi_next = enfs_lb_iter_next_entry_roundrobin, ++}; ++ ++static const struct rpc_xprt_iter_ops enfs_xprt_iter_singular = { ++ .xpi_rewind = enfs_lb_iter_default_rewind, ++ .xpi_xprt = enfs_lb_iter_current_entry, ++ .xpi_next = enfs_lb_iter_next_entry_sigular, ++}; +diff --git a/fs/nfs/enfs/enfs_roundrobin.h b/fs/nfs/enfs/enfs_roundrobin.h +new file mode 100644 +index 000000000..b72b088a6 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_roundrobin.h +@@ -0,0 +1,9 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ */ ++#ifndef ENFS_ROUNDROBIN_H ++#define ENFS_ROUNDROBIN_H ++ ++void enfs_lb_set_policy(struct rpc_clnt *clnt); ++#endif diff --git a/0005-add_enfs_module_for_sunrpc_failover_and_configure.patch b/0005-add_enfs_module_for_sunrpc_failover_and_configure.patch new file mode 100644 index 0000000000000000000000000000000000000000..9f3beea93dd695ca2cc20e57e25d5a6206124bc8 --- /dev/null +++ b/0005-add_enfs_module_for_sunrpc_failover_and_configure.patch @@ -0,0 +1,1600 @@ +diff --git a/fs/nfs/enfs/enfs_config.c b/fs/nfs/enfs/enfs_config.c +new file mode 100644 +index 000000000..a0ca93114 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_config.c +@@ -0,0 +1,378 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ */ ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "enfs_errcode.h" ++#include "enfs_log.h" ++#include "enfs_config.h" ++ ++#define MAX_FILE_SIZE 8192 ++#define STRING_BUF_SIZE 128 ++#define CONFIG_FILE_PATH "/etc/enfs/config.ini" ++#define ENFS_NOTIFY_FILE_PERIOD 1000UL ++ ++#define MAX_PATH_DETECT_INTERVAL 300 ++#define MIN_PATH_DETECT_INTERVAL 5 ++#define MAX_PATH_DETECT_TIMEOUT 60 ++#define MIN_PATH_DETECT_TIMEOUT 1 ++#define MAX_MULTIPATH_TIMEOUT 60 ++#define MIN_MULTIPATH_TIMEOUT 0 ++#define MAX_MULTIPATH_STATE ENFS_MULTIPATH_DISABLE ++#define MIN_MULTIPATH_STATE ENFS_MULTIPATH_ENABLE ++ ++#define DEFAULT_PATH_DETECT_INTERVAL 10 ++#define DEFAULT_PATH_DETECT_TIMEOUT 5 ++#define DEFAULT_MULTIPATH_TIMEOUT 0 ++#define DEFAULT_MULTIPATH_STATE ENFS_MULTIPATH_ENABLE ++#define DEFAULT_LOADBALANCE_MODE ENFS_LOADBALANCE_RR ++ ++typedef int (*check_and_assign_func)(char *, char *, int, int); ++ ++struct enfs_config_info { ++ int path_detect_interval; ++ int path_detect_timeout; ++ int multipath_timeout; ++ int loadbalance_mode; ++ int multipath_state; ++}; ++ ++struct check_and_assign_value { ++ char *field_name; ++ check_and_assign_func func; ++ int min_value; ++ int max_value; ++}; ++ ++static struct enfs_config_info g_enfs_config_info; ++static struct timespec64 modify_time; ++static struct task_struct *thread; ++ ++static int enfs_check_config_value(char *value, int min_value, int max_value) ++{ ++ unsigned long num_value; ++ int ret; ++ ++ ret = kstrtol(value, 10, &num_value); ++ if (ret != 0) { ++ enfs_log_error("Failed to convert string to int\n"); ++ return -EINVAL; ++ } ++ ++ if (num_value < min_value || num_value > max_value) ++ return -EINVAL; ++ ++ return num_value; ++} ++ ++static int enfs_check_and_assign_int_value(char *field_name, char *value, ++ int min_value, int max_value) ++{ ++ int int_value = enfs_check_config_value(value, min_value, max_value); ++ ++ if (int_value < 0) ++ return -EINVAL; ++ ++ if (strcmp(field_name, "path_detect_interval") == 0) { ++ g_enfs_config_info.path_detect_interval = int_value; ++ return ENFS_RET_OK; ++ } ++ if (strcmp(field_name, "path_detect_timeout") == 0) { ++ g_enfs_config_info.path_detect_timeout = int_value; ++ return ENFS_RET_OK; ++ } ++ if (strcmp(field_name, "multipath_timeout") == 0) { ++ g_enfs_config_info.multipath_timeout = int_value; ++ return ENFS_RET_OK; ++ } ++ if (strcmp(field_name, "multipath_disable") == 0) { ++ g_enfs_config_info.multipath_state = int_value; ++ return ENFS_RET_OK; ++ } ++ return -EINVAL; ++} ++ ++static int enfs_check_and_assign_loadbalance_mode(char *field_name, ++ char *value, ++ int min_value, ++ int max_value) ++{ ++ if (value == NULL) ++ return -EINVAL; ++ ++ if (strcmp(field_name, "multipath_select_policy") == 0) { ++ if (strcmp(value, "roundrobin") == 0) { ++ g_enfs_config_info.loadbalance_mode ++ = ENFS_LOADBALANCE_RR; ++ return ENFS_RET_OK; ++ } ++ } ++ return -EINVAL; ++} ++ ++static const struct check_and_assign_value g_check_and_assign_value[] = { ++ {"path_detect_interval", enfs_check_and_assign_int_value, ++ MIN_PATH_DETECT_INTERVAL, MAX_PATH_DETECT_INTERVAL}, ++ {"path_detect_timeout", enfs_check_and_assign_int_value, ++ MIN_PATH_DETECT_TIMEOUT, MAX_PATH_DETECT_TIMEOUT}, ++ {"multipath_timeout", enfs_check_and_assign_int_value, ++ MIN_MULTIPATH_TIMEOUT, MAX_MULTIPATH_TIMEOUT}, ++ {"multipath_disable", enfs_check_and_assign_int_value, ++ MIN_MULTIPATH_STATE, MAX_MULTIPATH_STATE}, ++ {"multipath_select_policy", enfs_check_and_assign_loadbalance_mode, ++ 0, 0}, ++}; ++ ++static int enfs_read_config_file(char *buffer, char *file_path) ++{ ++ int ret; ++ struct file *filp = NULL; ++ loff_t f_pos = 0; ++ mm_segment_t fs; ++ ++ ++ filp = filp_open(file_path, O_RDONLY, 0); ++ ++ if (IS_ERR(filp)) { ++ enfs_log_error("Failed to open file %s\n", CONFIG_FILE_PATH); ++ ret = -ENOENT; ++ return ret; ++ } ++ ++ fs = get_fs(); ++ set_fs(get_ds()); ++ kernel_read(filp, buffer, MAX_FILE_SIZE, &f_pos); ++ set_fs(fs); ++ ++ ret = filp_close(filp, NULL); ++ if (ret) { ++ enfs_log_error("Close File:%s failed:%d.\n", ++ CONFIG_FILE_PATH, ret); ++ return -EINVAL; ++ } ++ return ENFS_RET_OK; ++} ++ ++static int enfs_deal_with_comment_line(char *buffer) ++{ ++ int ret; ++ char *pos = strchr(buffer, '\n'); ++ ++ if (pos != NULL) ++ ret = strlen(buffer) - strlen(pos); ++ else ++ ret = strlen(buffer); ++ ++ return ret; ++} ++ ++static int enfs_parse_key_value_from_config(char *buffer, char *key, ++ char *value, int keyLen, ++ int valueLen) ++{ ++ char *line; ++ char *tokenPtr; ++ int len; ++ char *tem; ++ char *pos = strchr(buffer, '\n'); ++ ++ if (pos != NULL) ++ len = strlen(buffer) - strlen(pos); ++ else ++ len = strlen(buffer); ++ ++ line = kmalloc(len + 1, GFP_KERNEL); ++ if (!line) { ++ enfs_log_error("Failed to allocate memory.\n"); ++ return -ENOMEM; ++ } ++ line[len] = '\0'; ++ strncpy(line, buffer, len); ++ ++ tem = line; ++ tokenPtr = strsep(&tem, "="); ++ if (tokenPtr == NULL || tem == NULL) { ++ kfree(line); ++ return len; ++ } ++ strncpy(key, strim(tokenPtr), keyLen); ++ strncpy(value, strim(tem), valueLen); ++ ++ kfree(line); ++ return len; ++} ++ ++static int enfs_get_value_from_config_file(char *buffer, char *field_name, ++ char *value, int valueLen) ++{ ++ int ret; ++ char key[STRING_BUF_SIZE + 1] = {0}; ++ char val[STRING_BUF_SIZE + 1] = {0}; ++ ++ while (buffer[0] != '\0') { ++ if (buffer[0] == '\n') { ++ buffer++; ++ } else if (buffer[0] == '#') { ++ ret = enfs_deal_with_comment_line(buffer); ++ if (ret > 0) ++ buffer += ret; ++ } else { ++ ret = enfs_parse_key_value_from_config(buffer, key, val, ++ STRING_BUF_SIZE, ++ STRING_BUF_SIZE); ++ if (ret < 0) { ++ enfs_log_error("failed parse key value, %d\n" ++ , ret); ++ return ret; ++ } ++ key[STRING_BUF_SIZE] = '\0'; ++ val[STRING_BUF_SIZE] = '\0'; ++ ++ buffer += ret; ++ ++ if (strcmp(field_name, key) == 0) { ++ strncpy(value, val, valueLen); ++ return ENFS_RET_OK; ++ } ++ } ++ } ++ enfs_log_error("can not find value which matched field_name: %s.\n", ++ field_name); ++ return -EINVAL; ++} ++ ++int enfs_config_load(void) ++{ ++ char value[STRING_BUF_SIZE + 1]; ++ int ret; ++ int table_len; ++ int min; ++ int max; ++ int i; ++ char *buffer; ++ ++ buffer = kmalloc(MAX_FILE_SIZE, GFP_KERNEL); ++ if (!buffer) { ++ enfs_log_error("Failed to allocate memory.\n"); ++ return -ENOMEM; ++ } ++ memset(buffer, 0, MAX_FILE_SIZE); ++ ++ g_enfs_config_info.path_detect_interval = DEFAULT_PATH_DETECT_INTERVAL; ++ g_enfs_config_info.path_detect_timeout = DEFAULT_PATH_DETECT_TIMEOUT; ++ g_enfs_config_info.multipath_timeout = DEFAULT_MULTIPATH_TIMEOUT; ++ g_enfs_config_info.multipath_state = DEFAULT_MULTIPATH_STATE; ++ g_enfs_config_info.loadbalance_mode = DEFAULT_LOADBALANCE_MODE; ++ ++ table_len = sizeof(g_check_and_assign_value) / ++ sizeof(g_check_and_assign_value[0]); ++ ++ ret = enfs_read_config_file(buffer, CONFIG_FILE_PATH); ++ if (ret != 0) { ++ kfree(buffer); ++ return ret; ++ } ++ ++ for (i = 0; i < table_len; i++) { ++ ret = enfs_get_value_from_config_file(buffer, ++ g_check_and_assign_value[i].field_name, ++ value, STRING_BUF_SIZE); ++ if (ret < 0) ++ continue; ++ ++ value[STRING_BUF_SIZE] = '\0'; ++ min = g_check_and_assign_value[i].min_value; ++ max = g_check_and_assign_value[i].max_value; ++ if (g_check_and_assign_value[i].func != NULL) ++ (*g_check_and_assign_value[i].func)( ++ g_check_and_assign_value[i].field_name, ++ value, min, max); ++ } ++ ++ kfree(buffer); ++ return ENFS_RET_OK; ++} ++ ++int enfs_get_config_path_detect_interval(void) ++{ ++ return g_enfs_config_info.path_detect_interval; ++} ++ ++int enfs_get_config_path_detect_timeout(void) ++{ ++ return g_enfs_config_info.path_detect_timeout; ++} ++ ++int enfs_get_config_multipath_timeout(void) ++{ ++ return g_enfs_config_info.multipath_timeout; ++} ++ ++int enfs_get_config_multipath_state(void) ++{ ++ return g_enfs_config_info.multipath_state; ++} ++ ++int enfs_get_config_loadbalance_mode(void) ++{ ++ return g_enfs_config_info.loadbalance_mode; ++} ++ ++static bool enfs_file_changed(const char *filename) ++{ ++ int err; ++ struct kstat file_stat; ++ ++ err = vfs_stat(filename, &file_stat); ++ if (err) { ++ pr_err("failed to open file:%s err:%d\n", filename, err); ++ return false; ++ } ++ ++ if (timespec64_compare(&modify_time, &file_stat.mtime) == -1) { ++ modify_time = file_stat.mtime; ++ pr_info("file change: %lld %lld\n", modify_time.tv_sec, ++ file_stat.mtime.tv_sec); ++ return true; ++ } ++ ++ return false; ++} ++ ++static int enfs_thread_func(void *data) ++{ ++ while (!kthread_should_stop()) { ++ if (enfs_file_changed(CONFIG_FILE_PATH)) ++ enfs_config_load(); ++ ++ msleep(ENFS_NOTIFY_FILE_PERIOD); ++ } ++ return 0; ++} ++ ++int enfs_config_timer_init(void) ++{ ++ thread = kthread_run(enfs_thread_func, NULL, "enfs_notiy_file_thread"); ++ if (IS_ERR(thread)) { ++ pr_err("Failed to create kernel thread\n"); ++ return PTR_ERR(thread); ++ } ++ return 0; ++} ++ ++void enfs_config_timer_exit(void) ++{ ++ pr_info("enfs_notify_file_exit\n"); ++ if (thread) ++ kthread_stop(thread); ++} +diff --git a/fs/nfs/enfs/enfs_config.h b/fs/nfs/enfs/enfs_config.h +new file mode 100644 +index 000000000..4eff0ccc3 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_config.h +@@ -0,0 +1,31 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: nfs configuration ++ * Create: 2023-07-27 ++ */ ++ ++#ifndef ENFS_CONFIG_H ++#define ENFS_CONFIG_H ++ ++#include ++ ++enum enfs_multipath_state { ++ ENFS_MULTIPATH_ENABLE = 0, ++ ENFS_MULTIPATH_DISABLE = 1, ++}; ++ ++enum enfs_loadbalance_mode { ++ ENFS_LOADBALANCE_RR, ++}; ++ ++ ++int enfs_get_config_path_detect_interval(void); ++int enfs_get_config_path_detect_timeout(void); ++int enfs_get_config_multipath_timeout(void); ++int enfs_get_config_multipath_state(void); ++int enfs_get_config_loadbalance_mode(void); ++int enfs_config_load(void); ++int enfs_config_timer_init(void); ++void enfs_config_timer_exit(void); ++#endif // ENFS_CONFIG_H +diff --git a/fs/nfs/enfs/enfs_errcode.h b/fs/nfs/enfs/enfs_errcode.h +new file mode 100644 +index 000000000..ffa089088 +--- /dev/null ++++ b/fs/nfs/enfs/enfs_errcode.h +@@ -0,0 +1,16 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: nfs errocode ++ * Create: 2023-07-31 ++ */ ++ ++#ifndef ENFS_ERRCODE_H ++#define ENFS_ERRCODE_H ++ ++enum { ++ ENFS_RET_OK = 0, ++ ENFS_RET_FAIL ++}; ++ ++#endif // ENFS_ERRCODE_H +diff --git a/fs/nfs/enfs/enfs_log.h b/fs/nfs/enfs/enfs_log.h +new file mode 100644 +index 000000000..e12f8f3ae +--- /dev/null ++++ b/fs/nfs/enfs/enfs_log.h +@@ -0,0 +1,24 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: enfs log ++ * Create: 2023-07-31 ++ */ ++#ifndef ENFS_LOG_H ++#define ENFS_LOG_H ++ ++#include ++ ++#define enfs_log_info(fmt, ...) \ ++ pr_info("enfs:[%s]" pr_fmt(fmt), \ ++ __func__, ##__VA_ARGS__) ++ ++#define enfs_log_error(fmt, ...) \ ++ pr_err("enfs:[%s]" pr_fmt(fmt), \ ++ __func__, ##__VA_ARGS__) ++ ++#define enfs_log_debug(fmt, ...) \ ++ pr_debug("enfs:[%s]" pr_fmt(fmt), \ ++ __func__, ##__VA_ARGS__) ++ ++#endif // ENFS_ERRCODE_H +diff --git a/fs/nfs/enfs/failover_com.h b/fs/nfs/enfs/failover_com.h +new file mode 100644 +index 000000000..c52940da2 +--- /dev/null ++++ b/fs/nfs/enfs/failover_com.h +@@ -0,0 +1,23 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: failover time commont header file ++ * Create: 2023-08-02 ++ */ ++#ifndef FAILOVER_COMMON_H ++#define FAILOVER_COMMON_H ++ ++static inline bool failover_is_enfs_clnt(struct rpc_clnt *clnt) ++{ ++ struct rpc_clnt *next = clnt->cl_parent; ++ ++ while (next) { ++ if (next == next->cl_parent) ++ break; ++ next = next->cl_parent; ++ } ++ ++ return next != NULL ? next->cl_enfs : clnt->cl_enfs; ++} ++ ++#endif // FAILOVER_COMMON_H +diff --git a/fs/nfs/enfs/failover_path.c b/fs/nfs/enfs/failover_path.c +new file mode 100644 +index 000000000..2f5387216 +--- /dev/null ++++ b/fs/nfs/enfs/failover_path.c +@@ -0,0 +1,212 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: nfs path failover file ++ * Create: 2023-08-02 ++ */ ++ ++#include "failover_path.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include "enfs_config.h" ++#include "enfs_log.h" ++#include "failover_com.h" ++#include "pm_state.h" ++#include "pm_ping.h" ++ ++enum failover_policy_t { ++ FAILOVER_NOACTION = 1, ++ FAILOVER_RETRY, ++ FAILOVER_RETRY_DELAY, ++}; ++ ++static void failover_retry_path(struct rpc_task *task) ++{ ++ int ret; ++ ++ ret = rpc_restart_call(task); ++ ++ if (ret == 1) { ++ xprt_release(task); ++ rpc_init_task_retry_counters(task); ++ rpc_task_release_transport(task); ++ task->tk_xprt = rpc_task_get_next_xprt(task->tk_client); ++ } ++} ++ ++static void failover_retry_path_delay(struct rpc_task *task, int32_t delay) ++{ ++ failover_retry_path(task); ++ rpc_delay(task, delay); ++} ++ ++static void failover_retry_path_by_policy(struct rpc_task *task, ++ enum failover_policy_t policy) ++{ ++ if (policy == FAILOVER_RETRY) ++ failover_retry_path(task); ++ else if (policy == FAILOVER_RETRY_DELAY) ++ failover_retry_path_delay(task, 3 * HZ); // delay 3s ++} ++ ++static ++enum failover_policy_t failover_get_nfs3_retry_policy(struct rpc_task *task) ++{ ++ enum failover_policy_t policy = FAILOVER_NOACTION; ++ const struct rpc_procinfo *procinfo = task->tk_msg.rpc_proc; ++ u32 proc; ++ ++ if (unlikely(procinfo == NULL)) { ++ enfs_log_error("the task contains no valid proc.\n"); ++ return FAILOVER_NOACTION; ++ } ++ ++ proc = procinfo->p_proc; ++ ++ switch (proc) { ++ case NFS3PROC_CREATE: ++ case NFS3PROC_MKDIR: ++ case NFS3PROC_REMOVE: ++ case NFS3PROC_RMDIR: ++ case NFS3PROC_SYMLINK: ++ case NFS3PROC_LINK: ++ case NFS3PROC_SETATTR: ++ case NFS3PROC_WRITE: ++ policy = FAILOVER_RETRY_DELAY; ++ default: ++ policy = FAILOVER_RETRY; ++ } ++ return policy; ++} ++ ++static ++enum failover_policy_t failover_get_nfs4_retry_policy(struct rpc_task *task) ++{ ++ enum failover_policy_t policy = FAILOVER_NOACTION; ++ const struct rpc_procinfo *procinfo = task->tk_msg.rpc_proc; ++ u32 proc_idx; ++ ++ if (unlikely(procinfo == NULL)) { ++ enfs_log_error("the task contains no valid proc.\n"); ++ return FAILOVER_NOACTION; ++ } ++ ++ proc_idx = procinfo->p_statidx; ++ ++ switch (proc_idx) { ++ case NFSPROC4_CLNT_CREATE: ++ case NFSPROC4_CLNT_REMOVE: ++ case NFSPROC4_CLNT_LINK: ++ case NFSPROC4_CLNT_SYMLINK: ++ case NFSPROC4_CLNT_SETATTR: ++ case NFSPROC4_CLNT_WRITE: ++ case NFSPROC4_CLNT_RENAME: ++ case NFSPROC4_CLNT_SETACL: ++ policy = FAILOVER_RETRY_DELAY; ++ default: ++ policy = FAILOVER_RETRY; ++ } ++ return policy; ++} ++ ++static enum failover_policy_t failover_get_retry_policy(struct rpc_task *task) ++{ ++ struct rpc_clnt *clnt = task->tk_client; ++ u32 version = clnt->cl_vers; ++ enum failover_policy_t policy = FAILOVER_NOACTION; ++ ++ // 1. if the task meant to send to certain xprt, take no action ++ if (task->tk_flags & RPC_TASK_FIXED) ++ return FAILOVER_NOACTION; ++ ++ // 2. get policy by different version of nfs protocal ++ if (version == 3) // nfs v3 ++ policy = failover_get_nfs3_retry_policy(task); ++ else if (version == 4) // nfs v4 ++ policy = failover_get_nfs4_retry_policy(task); ++ else ++ return FAILOVER_NOACTION; ++ ++ // 3. if the task is not send to target, retry immediately ++ if (!RPC_WAS_SENT(task)) ++ policy = FAILOVER_RETRY; ++ ++ return policy; ++} ++ ++static int failover_check_task(struct rpc_task *task) ++{ ++ struct rpc_clnt *clnt = NULL; ++ int disable_mpath = enfs_get_config_multipath_state(); ++ ++ if (disable_mpath != ENFS_MULTIPATH_ENABLE) { ++ enfs_log_debug("Multipath is not enabled.\n"); ++ return -EINVAL; ++ } ++ ++ if (unlikely((task == NULL) || (task->tk_client == NULL))) { ++ enfs_log_error("The task is not valid.\n"); ++ return -EINVAL; ++ } ++ ++ clnt = task->tk_client; ++ ++ if (clnt->cl_prog != NFS_PROGRAM) { ++ enfs_log_debug("The clnt is not prog{%u} type.\n", ++ clnt->cl_prog); ++ return -EINVAL; ++ } ++ ++ if (!failover_is_enfs_clnt(clnt)) { ++ enfs_log_debug("The clnt is not a enfs-managed type.\n"); ++ return -EINVAL; ++ } ++ return 0; ++} ++ ++void failover_handle(struct rpc_task *task) ++{ ++ enum failover_policy_t policy; ++ int ret; ++ ++ ret = failover_check_task(task); ++ if (ret != 0) ++ return; ++ ++ pm_set_path_state(task->tk_xprt, PM_STATE_FAULT); ++ ++ policy = failover_get_retry_policy(task); ++ ++ failover_retry_path_by_policy(task, policy); ++} ++ ++bool failover_task_need_call_start_again(struct rpc_task *task) ++{ ++ int ret; ++ ++ ret = failover_check_task(task); ++ if (ret != 0) ++ return false; ++ ++ return true; ++} ++ ++bool failover_prepare_transmit(struct rpc_task *task) ++{ ++ if (task->tk_flags & RPC_TASK_FIXED) ++ return true; ++ ++ if (pm_ping_is_test_xprt_task(task)) ++ return true; ++ ++ if (pm_get_path_state(task->tk_xprt) == PM_STATE_FAULT) { ++ task->tk_status = -ETIMEDOUT; ++ return false; ++ } ++ ++ return true; ++} +diff --git a/fs/nfs/enfs/failover_path.h b/fs/nfs/enfs/failover_path.h +new file mode 100644 +index 000000000..9159ada07 +--- /dev/null ++++ b/fs/nfs/enfs/failover_path.h +@@ -0,0 +1,16 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: nfs path failover header file ++ * Create: 2023-08-02 ++ */ ++ ++#ifndef FAILOVER_PATH_H ++#define FAILOVER_PATH_H ++ ++#include ++ ++void failover_handle(struct rpc_task *task); ++bool failover_prepare_transmit(struct rpc_task *task); ++bool failover_task_need_call_start_again(struct rpc_task *task); ++#endif // FAILOVER_PATH_H +diff --git a/fs/nfs/enfs/failover_time.c b/fs/nfs/enfs/failover_time.c +new file mode 100644 +index 000000000..866ea82d1 +--- /dev/null ++++ b/fs/nfs/enfs/failover_time.c +@@ -0,0 +1,99 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: failover time file ++ * Create: 2023-08-02 ++ */ ++ ++#include "failover_time.h" ++#include ++#include ++#include "enfs_config.h" ++#include "enfs_log.h" ++#include "failover_com.h" ++#include "pm_ping.h" ++ ++static unsigned long failover_get_mulitipath_timeout(struct rpc_clnt *clnt) ++{ ++ unsigned long config_tmo = enfs_get_config_multipath_timeout() * HZ; ++ unsigned long clnt_tmo = clnt->cl_timeout->to_initval; ++ ++ if (config_tmo == 0) ++ return clnt_tmo; ++ ++ return config_tmo > clnt_tmo ? clnt_tmo : config_tmo; ++} ++ ++void failover_adjust_task_timeout(struct rpc_task *task, void *condition) ++{ ++ struct rpc_clnt *clnt = NULL; ++ unsigned long tmo; ++ int disable_mpath = enfs_get_config_multipath_state(); ++ ++ if (disable_mpath != ENFS_MULTIPATH_ENABLE) { ++ enfs_log_debug("Multipath is not enabled.\n"); ++ return; ++ } ++ ++ clnt = task->tk_client; ++ if (unlikely(clnt == NULL)) { ++ enfs_log_error("task associate client is NULL.\n"); ++ return; ++ } ++ ++ if (!failover_is_enfs_clnt(clnt)) { ++ enfs_log_debug("The clnt is not a enfs-managed type.\n"); ++ return; ++ } ++ ++ tmo = failover_get_mulitipath_timeout(clnt); ++ if (tmo == 0) { ++ enfs_log_debug("Multipath is not enabled.\n"); ++ return; ++ } ++ ++ if (task->tk_timeout != 0) ++ task->tk_timeout = ++ task->tk_timeout < tmo ? task->tk_timeout : tmo; ++ else ++ task->tk_timeout = tmo; ++} ++ ++void failover_init_task_req(struct rpc_task *task, struct rpc_rqst *req) ++{ ++ struct rpc_clnt *clnt = NULL; ++ int disable_mpath = enfs_get_config_multipath_state(); ++ ++ if (disable_mpath != ENFS_MULTIPATH_ENABLE) { ++ enfs_log_debug("Multipath is not enabled.\n"); ++ return; ++ } ++ ++ clnt = task->tk_client; ++ if (unlikely(clnt == NULL)) { ++ enfs_log_error("task associate client is NULL.\n"); ++ return; ++ } ++ ++ if (!failover_is_enfs_clnt(clnt)) { ++ enfs_log_debug("The clnt is not a enfs-managed type.\n"); ++ return; ++ } ++ ++ if (!pm_ping_is_test_xprt_task(task)) ++ req->rq_timeout = failover_get_mulitipath_timeout(clnt); ++ else { ++ req->rq_timeout = enfs_get_config_path_detect_timeout() * HZ; ++ req->rq_majortimeo = req->rq_timeout + jiffies; ++ } ++ ++ /* ++ * when task is retried, the req is new, we lost major-timeout times, ++ * so we have to restore req major ++ * timeouts from the task, if it is stored. ++ */ ++ if (task->tk_major_timeo != 0) ++ req->rq_majortimeo = task->tk_major_timeo; ++ else ++ task->tk_major_timeo = req->rq_majortimeo; ++} +diff --git a/fs/nfs/enfs/failover_time.h b/fs/nfs/enfs/failover_time.h +new file mode 100644 +index 000000000..ede25b577 +--- /dev/null ++++ b/fs/nfs/enfs/failover_time.h +@@ -0,0 +1,16 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: failover time header file ++ * Create: 2023-08-02 ++ */ ++ ++#ifndef FAILOVER_TIME_H ++#define FAILOVER_TIME_H ++ ++#include ++ ++void failover_adjust_task_timeout(struct rpc_task *task, void *condition); ++void failover_init_task_req(struct rpc_task *task, struct rpc_rqst *req); ++ ++#endif // FAILOVER_TIME_H +diff --git a/fs/nfs/enfs/init.h b/fs/nfs/enfs/init.h +new file mode 100644 +index 000000000..d81af9b02 +--- /dev/null ++++ b/fs/nfs/enfs/init.h +@@ -0,0 +1,16 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: nfs client init ++ * Create: 2023-07-31 ++ */ ++ ++#ifndef ENFS_INIT_H ++#define ENFS_INIT_H ++ ++#include ++ ++int32_t enfs_init(void); ++void enfs_fini(void); ++ ++#endif +diff --git a/fs/nfs/enfs/mgmt_init.c b/fs/nfs/enfs/mgmt_init.c +new file mode 100644 +index 000000000..122790775 +--- /dev/null ++++ b/fs/nfs/enfs/mgmt_init.c +@@ -0,0 +1,21 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: mgmt component init ++ * Create: 2023-07-31 ++ */ ++ ++#include "mgmt_init.h" ++#include ++#include "enfs_errcode.h" ++#include "enfs_config.h" ++ ++int32_t mgmt_init(void) ++{ ++ return enfs_config_timer_init(); ++} ++ ++void mgmt_fini(void) ++{ ++ enfs_config_timer_exit(); ++} +diff --git a/fs/nfs/enfs/mgmt_init.h b/fs/nfs/enfs/mgmt_init.h +new file mode 100644 +index 000000000..779732740 +--- /dev/null ++++ b/fs/nfs/enfs/mgmt_init.h +@@ -0,0 +1,17 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: mgmt component init ++ * Create: 2023-07-31 ++ */ ++ ++#ifndef MGMT_INIT_H ++#define MGMT_INIT_H ++ ++#include ++ ++int32_t mgmt_init(void); ++void mgmt_fini(void); ++ ++ ++#endif // MGMT_INIT_H +diff --git a/fs/nfs/enfs/pm_ping.c b/fs/nfs/enfs/pm_ping.c +new file mode 100644 +index 000000000..9ecbe9a77 +--- /dev/null ++++ b/fs/nfs/enfs/pm_ping.c +@@ -0,0 +1,420 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: path state header file ++ * Create: 2023-08-21 ++ */ ++ ++#include "pm_ping.h" ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++#include ++ ++#include "../../../net/sunrpc/netns.h" ++#include "pm_state.h" ++#include "enfs.h" ++#include "enfs_log.h" ++#include "enfs_config.h" ++ ++#define SLEEP_INTERVAL 2 ++extern unsigned int sunrpc_net_id; ++ ++static struct task_struct *pm_ping_timer_thread; ++//protect pint_execute_workq ++static spinlock_t ping_execute_workq_lock; ++// timer for test xprt workqueue ++static struct workqueue_struct *ping_execute_workq; ++// count the ping xprt work on flight ++static atomic_t check_xprt_count; ++ ++struct ping_xprt_work { ++ struct rpc_xprt *xprt; // use this specific xprt ++ struct rpc_clnt *clnt; // use this specific rpc_client ++ struct work_struct ping_work; ++}; ++ ++struct pm_ping_async_callback { ++ void *data; ++ void (*func)(void *data); ++}; ++ ++// set xprt's enum pm_check_state ++void pm_ping_set_path_check_state(struct rpc_xprt *xprt, ++ enum pm_check_state state) ++{ ++ struct enfs_xprt_context *ctx = NULL; ++ ++ if (IS_ERR(xprt)) { ++ enfs_log_error("The xprt ptr is not exist.\n"); ++ return; ++ } ++ ++ if (xprt == NULL) { ++ enfs_log_error("The xprt is not valid.\n"); ++ return; ++ } ++ ++ xprt_get(xprt); ++ ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ if (ctx == NULL) { ++ enfs_log_error("The xprt multipath ctx is not valid.\n"); ++ xprt_put(xprt); ++ return; ++ } ++ ++ atomic_set(&ctx->path_check_state, state); ++ xprt_put(xprt); ++} ++ ++// get xprt's enum pm_check_state ++static enum pm_check_state pm_ping_get_path_check_state(struct rpc_xprt *xprt) ++{ ++ struct enfs_xprt_context *ctx = NULL; ++ enum pm_check_state state; ++ ++ if (xprt == NULL) { ++ enfs_log_error("The xprt is not valid.\n"); ++ return PM_CHECK_UNDEFINE; ++ } ++ ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ if (ctx == NULL) { ++ enfs_log_error("The xprt multipath ctx is not valid.\n"); ++ return PM_CHECK_UNDEFINE; ++ } ++ ++ state = atomic_read(&ctx->path_check_state); ++ ++ return state; ++} ++ ++static void pm_ping_call_done_callback(void *data) ++{ ++ struct pm_ping_async_callback *callback_data = ++ (struct pm_ping_async_callback *)data; ++ ++ if (callback_data == NULL) ++ return; ++ ++ callback_data->func(callback_data->data); ++ ++ kfree(callback_data); ++} ++ ++// Default callback for async RPC calls ++static void pm_ping_call_done(struct rpc_task *task, void *data) ++{ ++ struct rpc_xprt *xprt = task->tk_xprt; ++ ++ atomic_dec(&check_xprt_count); ++ if (task->tk_status >= 0) ++ pm_set_path_state(xprt, PM_STATE_NORMAL); ++ else ++ pm_set_path_state(xprt, PM_STATE_FAULT); ++ ++ pm_ping_set_path_check_state(xprt, PM_CHECK_FINISH); ++ ++ pm_ping_call_done_callback(data); ++} ++ ++// register func to rpc_call_done ++static const struct rpc_call_ops pm_ping_set_status_ops = { ++ .rpc_call_done = pm_ping_call_done, ++}; ++ ++// execute work which in work_queue ++static void pm_ping_execute_work(struct work_struct *work) ++{ ++ int ret = 0; ++ ++ // get the work information ++ struct ping_xprt_work *work_info = ++ container_of(work, struct ping_xprt_work, ping_work); ++ ++ // if check state is pending ++ if (pm_ping_get_path_check_state(work_info->xprt) == PM_CHECK_WAITING) { ++ ++ pm_ping_set_path_check_state(work_info->xprt, ++ PM_CHECK_CHECKING); ++ ++ ret = rpc_clnt_test_xprt(work_info->clnt, ++ work_info->xprt, ++ &pm_ping_set_status_ops, ++ NULL, ++ RPC_TASK_ASYNC | RPC_TASK_FIXED); ++ ++ if (ret < 0) { ++ enfs_log_debug("ping xprt execute failed ,ret %d", ret); ++ ++ pm_ping_set_path_check_state(work_info->xprt, ++ PM_CHECK_FINISH); ++ ++ } else ++ atomic_inc(&check_xprt_count); ++ ++ } ++ ++ atomic_dec(&work_info->clnt->cl_count); ++ xprt_put(work_info->xprt); ++ kfree(work_info); ++ work_info = NULL; ++} ++ ++static bool pm_ping_workqueue_queue_work(struct work_struct *work) ++{ ++ bool ret = false; ++ ++ spin_lock(&ping_execute_workq_lock); ++ ++ if (ping_execute_workq != NULL) ++ ret = queue_work(ping_execute_workq, work); ++ ++ spin_unlock(&ping_execute_workq_lock); ++ return ret; ++} ++ ++// init test work and add this work to workqueue ++static int pm_ping_add_work(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, void *data) ++{ ++ struct ping_xprt_work *work_info; ++ bool ret = false; ++ ++ if (IS_ERR(xprt) || xprt == NULL) { ++ enfs_log_error("The xprt ptr is not exist.\n"); ++ return -EINVAL; ++ } ++ ++ if (IS_ERR(clnt) || clnt == NULL) { ++ enfs_log_error("The clnt ptr is not exist.\n"); ++ return -EINVAL; ++ } ++ ++ if (!xprt->multipath_context) { ++ enfs_log_error("multipath_context is null.\n"); ++ return -EINVAL; ++ } ++ ++ // check xprt pending status, if pending status equals Finish ++ // means this xprt can inster to work queue ++ if (pm_ping_get_path_check_state(xprt) == ++ PM_CHECK_FINISH || ++ pm_ping_get_path_check_state(xprt) == ++ PM_CHECK_INIT) { ++ ++ enfs_log_debug("find xprt pointer. %p\n", xprt); ++ work_info = kzalloc(sizeof(struct ping_xprt_work), GFP_ATOMIC); ++ if (work_info == NULL) ++ return -ENOMEM; ++ work_info->clnt = clnt; ++ atomic_inc(&clnt->cl_count); ++ work_info->xprt = xprt; ++ xprt_get(xprt); ++ INIT_WORK(&work_info->ping_work, pm_ping_execute_work); ++ pm_ping_set_path_check_state(xprt, PM_CHECK_WAITING); ++ ++ ret = pm_ping_workqueue_queue_work(&work_info->ping_work); ++ if (!ret) { ++ atomic_dec(&work_info->clnt->cl_count); ++ xprt_put(work_info->xprt); ++ kfree(work_info); ++ return -EINVAL; ++ } ++ } ++ return 0; ++} ++ ++// encapsulate pm_ping_add_work() ++static int pm_ping_execute_xprt_test(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, void *data) ++{ ++ pm_ping_add_work(clnt, xprt, NULL); ++ // return 0 for rpc_clnt_iterate_for_each_xprt(); ++ // because negative value will stop iterate all xprt ++ // and we need return negative value for debug ++ // Therefore, we need this function to iterate all xprt ++ return 0; ++} ++ ++// export to other module add ping work to workqueue ++int pm_ping_rpc_test_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt) ++{ ++ int ret; ++ ++ ret = pm_ping_add_work(clnt, xprt, NULL); ++ return ret; ++} ++ ++// iterate xprt in the client ++static void pm_ping_loop_rpclnt(struct sunrpc_net *sn) ++{ ++ struct rpc_clnt *clnt; ++ ++ spin_lock(&sn->rpc_client_lock); ++ list_for_each_entry_rcu(clnt, &sn->all_clients, cl_clients) { ++ if (clnt->cl_enfs) { ++ enfs_log_debug("find rpc_clnt. %p\n", clnt); ++ rpc_clnt_iterate_for_each_xprt(clnt, ++ pm_ping_execute_xprt_test, NULL); ++ } ++ } ++ spin_unlock(&sn->rpc_client_lock); ++} ++ ++// iterate each clnt in the sunrpc_net ++static void pm_ping_loop_sunrpc_net(void) ++{ ++ struct net *net; ++ struct sunrpc_net *sn; ++ ++ rcu_read_lock(); ++ for_each_net_rcu(net) { ++ sn = net_generic(net, sunrpc_net_id); ++ if (sn == NULL) ++ continue; ++ pm_ping_loop_rpclnt(sn); ++ } ++ rcu_read_unlock(); ++} ++ ++static int pm_ping_routine(void *data) ++{ ++ while (!kthread_should_stop()) { ++ // equale 0 means open multipath ++ if (enfs_get_config_multipath_state() == ++ ENFS_MULTIPATH_ENABLE) ++ pm_ping_loop_sunrpc_net(); ++ ++ msleep( ++ enfs_get_config_path_detect_interval() * 1000); ++ } ++ return 0; ++} ++ ++// start thread to cycly ping ++static int pm_ping_start(void) ++{ ++ pm_ping_timer_thread = ++ kthread_run(pm_ping_routine, NULL, "pm_ping_routine"); ++ if (IS_ERR(pm_ping_timer_thread)) { ++ enfs_log_error("Failed to create kernel thread\n"); ++ return PTR_ERR(pm_ping_timer_thread); ++ } ++ return 0; ++} ++ ++// initialize workqueue ++static int pm_ping_workqueue_init(void) ++{ ++ struct workqueue_struct *queue = NULL; ++ ++ queue = create_workqueue("pm_ping_workqueue"); ++ ++ if (queue == NULL) { ++ enfs_log_error("create workqueue failed.\n"); ++ return -ENOMEM; ++ } ++ ++ spin_lock(&ping_execute_workq_lock); ++ ping_execute_workq = queue; ++ spin_unlock(&ping_execute_workq_lock); ++ enfs_log_info("create workqueue succeeeded.\n"); ++ return 0; ++} ++ ++static void pm_ping_workqueue_fini(void) ++{ ++ struct workqueue_struct *queue = NULL; ++ ++ spin_lock(&ping_execute_workq_lock); ++ queue = ping_execute_workq; ++ ping_execute_workq = NULL; ++ spin_unlock(&ping_execute_workq_lock); ++ ++ enfs_log_info("delete work queue\n"); ++ ++ if (queue != NULL) { ++ flush_workqueue(queue); ++ destroy_workqueue(queue); ++ } ++} ++ ++// module exit func ++void pm_ping_fini(void) ++{ ++ if (pm_ping_timer_thread) ++ kthread_stop(pm_ping_timer_thread); ++ ++ pm_ping_workqueue_fini(); ++ ++ while (atomic_read(&check_xprt_count) != 0) ++ msleep(SLEEP_INTERVAL); ++} ++ ++// module init func ++int pm_ping_init(void) ++{ ++ int ret; ++ ++ atomic_set(&check_xprt_count, 0); ++ ret = pm_ping_workqueue_init(); ++ if (ret != 0) { ++ enfs_log_error("PM_PING Module loading failed.\n"); ++ return ret; ++ } ++ ret = pm_ping_start(); ++ if (ret != 0) { ++ enfs_log_error("PM_PING Module loading failed.\n"); ++ pm_ping_workqueue_fini(); ++ return ret; ++ } ++ ++ return ret; ++} ++ ++bool pm_ping_is_test_xprt_task(struct rpc_task *task) ++{ ++ return task->tk_ops == &pm_ping_set_status_ops ? true : false; ++} ++ ++int pm_ping_rpc_test_xprt_with_callback(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, ++ void (*func)(void *data), ++ void *data) ++{ ++ int ret; ++ ++ struct pm_ping_async_callback *callback_data = ++ kzalloc(sizeof(struct pm_ping_async_callback), GFP_KERNEL); ++ ++ if (callback_data == NULL) { ++ enfs_log_error("failed to mzalloc mem\n"); ++ return -ENOMEM; ++ } ++ ++ callback_data->data = data; ++ callback_data->func = func; ++ atomic_inc(&check_xprt_count); ++ ret = rpc_clnt_test_xprt(clnt, xprt, ++ &pm_ping_set_status_ops, ++ callback_data, ++ RPC_TASK_ASYNC | RPC_TASK_FIXED); ++ ++ if (ret < 0) { ++ enfs_log_debug("ping xprt execute failed ,ret %d", ret); ++ atomic_dec(&check_xprt_count); ++ } ++ ++ return ret; ++} +diff --git a/fs/nfs/enfs/pm_ping.h b/fs/nfs/enfs/pm_ping.h +new file mode 100644 +index 000000000..8b159b286 +--- /dev/null ++++ b/fs/nfs/enfs/pm_ping.h +@@ -0,0 +1,32 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: nfs configuration ++ * Create: 2023-07-27 ++ */ ++ ++#ifndef PM_PING_H ++#define PM_PING_H ++ ++#include ++ ++enum pm_check_state { ++ PM_CHECK_INIT, // this xprt never been queued ++ PM_CHECK_WAITING, // this xprt waiting in the queue ++ PM_CHECK_CHECKING, // this xprt is testing ++ PM_CHECK_FINISH, // this xprt has been finished ++ PM_CHECK_UNDEFINE, // undefine multipath struct ++}; ++ ++int pm_ping_init(void); ++void pm_ping_fini(void); ++int pm_ping_rpc_test_xprt(struct rpc_clnt *clnt, struct rpc_xprt *xprt); ++void pm_ping_set_path_check_state(struct rpc_xprt *xprt, ++ enum pm_check_state state); ++bool pm_ping_is_test_xprt_task(struct rpc_task *task); ++int pm_ping_rpc_test_xprt_with_callback(struct rpc_clnt *clnt, ++ struct rpc_xprt *xprt, ++ void (*func)(void *data), ++ void *data); ++ ++#endif // PM_PING_H +diff --git a/fs/nfs/enfs/pm_state.c b/fs/nfs/enfs/pm_state.c +new file mode 100644 +index 000000000..279028dc0 +--- /dev/null ++++ b/fs/nfs/enfs/pm_state.c +@@ -0,0 +1,156 @@ ++// SPDX-License-Identifier: GPL-2.0 ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: path state file ++ * Create: 2023-08-12 ++ */ ++#include "pm_state.h" ++#include ++ ++#include "enfs.h" ++#include "enfs_log.h" ++ ++enum pm_path_state pm_get_path_state(struct rpc_xprt *xprt) ++{ ++ struct enfs_xprt_context *ctx = NULL; ++ enum pm_path_state state; ++ ++ if (xprt == NULL) { ++ enfs_log_error("The xprt is not valid.\n"); ++ return PM_STATE_UNDEFINED; ++ } ++ ++ xprt_get(xprt); ++ ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ if (ctx == NULL) { ++ enfs_log_error("The xprt multipath ctx is not valid.\n"); ++ xprt_put(xprt); ++ return PM_STATE_UNDEFINED; ++ } ++ ++ state = atomic_read(&ctx->path_state); ++ ++ xprt_put(xprt); ++ ++ return state; ++} ++ ++void pm_set_path_state(struct rpc_xprt *xprt, enum pm_path_state state) ++{ ++ struct enfs_xprt_context *ctx = NULL; ++ enum pm_path_state cur_state; ++ ++ if (xprt == NULL) { ++ enfs_log_error("The xprt is not valid.\n"); ++ return; ++ } ++ ++ xprt_get(xprt); ++ ++ ctx = (struct enfs_xprt_context *)xprt->multipath_context; ++ if (ctx == NULL) { ++ enfs_log_error("The xprt multipath ctx is not valid.\n"); ++ xprt_put(xprt); ++ return; ++ } ++ ++ cur_state = atomic_read(&ctx->path_state); ++ if (cur_state == state) { ++ xprt_put(xprt); ++ return; ++ } ++ ++ atomic_set(&ctx->path_state, state); ++ enfs_log_info("The xprt {%p} path state change from {%d} to {%d}.\n", ++ xprt, cur_state, state); ++ ++ xprt_put(xprt); ++} ++ ++void pm_get_path_state_desc(struct rpc_xprt *xprt, char *buf, int len) ++{ ++ enum pm_path_state state; ++ ++ if (xprt == NULL) { ++ enfs_log_error("The xprt is not valid.\n"); ++ return; ++ } ++ ++ if ((buf == NULL) || (len <= 0)) { ++ enfs_log_error("Buffer is not valid, len=%d.\n", len); ++ return; ++ } ++ ++ state = pm_get_path_state(xprt); ++ ++ switch (state) { ++ case PM_STATE_INIT: ++ (void)snprintf(buf, len, "Init"); ++ break; ++ case PM_STATE_NORMAL: ++ (void)snprintf(buf, len, "Normal"); ++ break; ++ case PM_STATE_FAULT: ++ (void)snprintf(buf, len, "Fault"); ++ break; ++ default: ++ (void)snprintf(buf, len, "Unknown"); ++ break; ++ } ++} ++ ++void pm_get_xprt_state_desc(struct rpc_xprt *xprt, char *buf, int len) ++{ ++ int i; ++ unsigned long state; ++ static unsigned long xprt_mask[] = { ++ XPRT_LOCKED, XPRT_CONNECTED, ++ XPRT_CONNECTING, XPRT_CLOSE_WAIT, ++ XPRT_BOUND, XPRT_BINDING, XPRT_CLOSING, ++ XPRT_CONGESTED}; ++ ++ static const char *const xprt_state_desc[] = { ++ "LOCKED", "CONNECTED", "CONNECTING", ++ "CLOSE_WAIT", "BOUND", "BINDING", ++ "CLOSING", "CONGESTED"}; ++ int pos = 0; ++ int ret = 0; ++ ++ if (xprt == NULL) { ++ enfs_log_error("The xprt is not valid.\n"); ++ return; ++ } ++ ++ if ((buf == NULL) || (len <= 0)) { ++ enfs_log_error( ++ "Xprt state buffer is not valid, len=%d.\n", ++ len); ++ return; ++ } ++ ++ xprt_get(xprt); ++ state = READ_ONCE(xprt->state); ++ xprt_put(xprt); ++ ++ for (i = 0; i < ARRAY_SIZE(xprt_mask); ++i) { ++ if (pos >= len) ++ break; ++ ++ if (!test_bit(xprt_mask[i], &state)) ++ continue; ++ ++ if (pos == 0) ++ ret = snprintf(buf, len, "%s", xprt_state_desc[i]); ++ else ++ ret = snprintf(buf + pos, len - pos, "|%s", ++ xprt_state_desc[i]); ++ ++ if (ret < 0) { ++ enfs_log_error("format state failed, ret %d.\n", ret); ++ break; ++ } ++ ++ pos += ret; ++ } ++} +diff --git a/fs/nfs/enfs/pm_state.h b/fs/nfs/enfs/pm_state.h +new file mode 100644 +index 000000000..469af998d +--- /dev/null ++++ b/fs/nfs/enfs/pm_state.h +@@ -0,0 +1,27 @@ ++/* SPDX-License-Identifier: GPL-2.0 */ ++/* ++ * Copyright (c) Huawei Technologies Co., Ltd. 2023-2023. All rights reserved. ++ * Description: path state header file ++ * Create: 2023-08-12 ++ */ ++ ++#ifndef PM_STATE_H ++#define PM_STATE_H ++ ++#include ++#include ++ ++enum pm_path_state { ++ PM_STATE_INIT, ++ PM_STATE_NORMAL, ++ PM_STATE_FAULT, ++ PM_STATE_UNDEFINED // xprt is not multipath xprt ++}; ++ ++void pm_set_path_state(struct rpc_xprt *xprt, enum pm_path_state state); ++enum pm_path_state pm_get_path_state(struct rpc_xprt *xprt); ++ ++void pm_get_path_state_desc(struct rpc_xprt *xprt, char *buf, int len); ++void pm_get_xprt_state_desc(struct rpc_xprt *xprt, char *buf, int len); ++ ++#endif // PM_STATE_H diff --git a/0006-add_enfs_compile_option.patch b/0006-add_enfs_compile_option.patch new file mode 100644 index 0000000000000000000000000000000000000000..71140c1a2583a9d77ca7380305c2a6892729ed59 --- /dev/null +++ b/0006-add_enfs_compile_option.patch @@ -0,0 +1,70 @@ +diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig +index b04256636d4b..ae53510c0627 100644 +--- a/arch/arm64/configs/openeuler_defconfig ++++ b/arch/arm64/configs/openeuler_defconfig +@@ -5344,6 +5344,7 @@ CONFIG_LOCKD=m + CONFIG_LOCKD_V4=y + CONFIG_NFS_ACL_SUPPORT=m + CONFIG_NFS_COMMON=y ++# CONFIG_ENFS is not set + CONFIG_SUNRPC=m + CONFIG_SUNRPC_GSS=m + CONFIG_SUNRPC_BACKCHANNEL=y +diff --git a/arch/x86/configs/openeuler_defconfig b/arch/x86/configs/openeuler_defconfig +index 59baeb2973af..ccc317f7fdb2 100644 +--- a/arch/x86/configs/openeuler_defconfig ++++ b/arch/x86/configs/openeuler_defconfig +@@ -6825,6 +6825,7 @@ CONFIG_LOCKD=m + CONFIG_LOCKD_V4=y + CONFIG_NFS_ACL_SUPPORT=m + CONFIG_NFS_COMMON=y ++# CONFIG_ENFS is not set + CONFIG_SUNRPC=m + CONFIG_SUNRPC_GSS=m + CONFIG_SUNRPC_BACKCHANNEL=y +diff --git a/fs/nfs/Kconfig b/fs/nfs/Kconfig +index e55f86713948..872c9b7671b1 100644 +--- a/fs/nfs/Kconfig ++++ b/fs/nfs/Kconfig +@@ -196,3 +196,14 @@ config NFS_DEBUG + depends on NFS_FS && SUNRPC_DEBUG + select CRC32 + default y ++ ++config ENFS ++ tristate "NFS client support for ENFS" ++ depends on NFS_FS ++ default n ++ help ++ This option enables support multipath of the NFS protocol ++ in the kernel's NFS client. ++ This feature will improve performance and reliability. ++ ++ If sure, say Y. +diff --git a/fs/nfs/Makefile b/fs/nfs/Makefile +index c587e3c4c6a6..19d0ac2ba3b8 100644 +--- a/fs/nfs/Makefile ++++ b/fs/nfs/Makefile +@@ -12,6 +12,7 @@ nfs-y := client.o dir.o file.o getroot.o inode.o super.o \ + nfs-$(CONFIG_ROOT_NFS) += nfsroot.o + nfs-$(CONFIG_SYSCTL) += sysctl.o + nfs-$(CONFIG_NFS_FSCACHE) += fscache.o fscache-index.o ++nfs-$(CONFIG_ENFS) += enfs_adapter.o + + obj-$(CONFIG_NFS_V2) += nfsv2.o + nfsv2-y := nfs2super.o proc.o nfs2xdr.o +@@ -34,3 +35,5 @@ nfsv4-$(CONFIG_NFS_V4_2) += nfs42proc.o + obj-$(CONFIG_PNFS_FILE_LAYOUT) += filelayout/ + obj-$(CONFIG_PNFS_BLOCK) += blocklayout/ + obj-$(CONFIG_PNFS_FLEXFILE_LAYOUT) += flexfilelayout/ ++ ++obj-$(CONFIG_ENFS) += enfs/ +diff --git a/net/sunrpc/Makefile b/net/sunrpc/Makefile +index 090658c3da12..fe4e3b28c5d1 100644 +--- a/net/sunrpc/Makefile ++++ b/net/sunrpc/Makefile +@@ -19,3 +19,4 @@ sunrpc-$(CONFIG_SUNRPC_DEBUG) += debugfs.o + sunrpc-$(CONFIG_SUNRPC_BACKCHANNEL) += backchannel_rqst.o + sunrpc-$(CONFIG_PROC_FS) += stats.o + sunrpc-$(CONFIG_SYSCTL) += sysctl.o ++sunrpc-$(CONFIG_ENFS) += sunrpc_enfs_adapter.o