diff --git a/fs/Makefile b/fs/Makefile index f1b15345e6a17a98b3cd783dbcad3ba90a7bb2ec..45e54c34c309ca6809d25feaba59388b46dddf19 100644 --- a/fs/Makefile +++ b/fs/Makefile @@ -16,6 +16,7 @@ obj-y := open.o read_write.o file_table.o super.o \ stack.o fs_struct.o statfs.o fs_pin.o nsfs.o \ fs_types.o fs_context.o fs_parser.o fsopen.o init.o \ kernel_read_file.o mnt_idmapping.o remap_range.o +obj-y += fs_ctl.o obj-$(CONFIG_BUFFER_HEAD) += buffer.o mpage.o obj-$(CONFIG_PROC_FS) += proc_namespace.o diff --git a/fs/cachefiles/cache.c b/fs/cachefiles/cache.c index 9fb06dc165202cbc86d690b51a9e7354fe8fc56e..1bbe4393d28932bcf5c9441ad98e3f806b737303 100644 --- a/fs/cachefiles/cache.c +++ b/fs/cachefiles/cache.c @@ -367,6 +367,7 @@ static void cachefiles_withdraw_volumes(struct cachefiles_cache *cache) continue; } list_del_init(&volume->cache_link); + cachefiles_get_volume(volume); } spin_unlock(&cache->object_list_lock); if (!volume) @@ -374,6 +375,7 @@ static void cachefiles_withdraw_volumes(struct cachefiles_cache *cache) cachefiles_withdraw_volume(volume); fscache_put_volume(vcookie, fscache_volume_put_withdraw); + cachefiles_put_volume(volume); } _leave(""); diff --git a/fs/cachefiles/daemon.c b/fs/cachefiles/daemon.c index 89b11336a83697298225e0dc61a0dfa6a43910cd..81d12106ce7aecb843c1a49bede55a0a1965b091 100644 --- a/fs/cachefiles/daemon.c +++ b/fs/cachefiles/daemon.c @@ -587,7 +587,7 @@ static int cachefiles_daemon_secctx(struct cachefiles_cache *cache, char *args) if (cache->secctx) { pr_err("Second security context specified\n"); - return -EINVAL; + return -EEXIST; } secctx = kstrdup(args, GFP_KERNEL); @@ -654,6 +654,12 @@ static int cachefiles_daemon_cull(struct cachefiles_cache *cache, char *args) if (!d_can_lookup(path.dentry)) goto notdir; + /* limit the scope of cull */ + if (cache->mnt != path.mnt) { + path_put(&path); + return -EOPNOTSUPP; + } + cachefiles_begin_secure(cache, &saved_cred); ret = cachefiles_cull(cache, path.dentry, args); cachefiles_end_secure(cache, saved_cred); @@ -780,6 +786,10 @@ static int cachefiles_daemon_bind(struct cachefiles_cache *cache, char *args) if (IS_ENABLED(CONFIG_CACHEFILES_ONDEMAND)) { if (!strcmp(args, "ondemand")) { + if (!cachefiles_ondemand_is_enabled()) { + pr_err("ondemand mode is disabled\n"); + return -EINVAL; + } set_bit(CACHEFILES_ONDEMAND_MODE, &cache->flags); } else if (*args) { pr_err("Invalid argument to the 'bind' command\n"); diff --git a/fs/cachefiles/interface.c b/fs/cachefiles/interface.c index 35ba2117a6f652c8903e32db08b10824bf537660..4d44e74c22fd345ae8ffa2d8d7952ac17ee29fcd 100644 --- a/fs/cachefiles/interface.c +++ b/fs/cachefiles/interface.c @@ -22,7 +22,7 @@ static struct cachefiles_object *cachefiles_alloc_object(struct fscache_cookie *cookie) { struct fscache_volume *vcookie = cookie->volume; - struct cachefiles_volume *volume = vcookie->cache_priv; + struct cachefiles_volume *volume = READ_ONCE(vcookie->cache_priv); struct cachefiles_object *object; _enter("{%s},%x,", vcookie->key, cookie->debug_id); @@ -38,6 +38,15 @@ struct cachefiles_object *cachefiles_alloc_object(struct fscache_cookie *cookie) refcount_set(&object->ref, 1); + /* + * After enabling sync_volume_unhash, fscache_relinquish_volume() may + * release cachefiles_volume while fscache_volume->ref is non-zero. + * However, cachefiles_object may still access cachefiles_cache through + * object->volume->cache (e.g., in cachefiles_ondemand_fd_release). + * Therefore, we need to pin cachefiles_volume through the object. + */ + cachefiles_get_volume(volume); + spin_lock_init(&object->lock); INIT_LIST_HEAD(&object->cache_link); object->volume = volume; @@ -96,6 +105,7 @@ void cachefiles_put_object(struct cachefiles_object *object, cachefiles_ondemand_deinit_obj_info(object); cache = object->volume->cache->cache; fscache_put_cookie(object->cookie, fscache_cookie_put_object); + cachefiles_put_volume(object->volume); object->cookie = NULL; kmem_cache_free(cachefiles_object_jar, object); fscache_uncount_object(cache); @@ -327,6 +337,8 @@ static void cachefiles_commit_object(struct cachefiles_object *object, static void cachefiles_clean_up_object(struct cachefiles_object *object, struct cachefiles_cache *cache) { + struct file *file; + if (test_bit(FSCACHE_COOKIE_RETIRED, &object->cookie->flags)) { if (!test_bit(CACHEFILES_OBJECT_USING_TMPFILE, &object->flags)) { cachefiles_see_object(object, cachefiles_obj_see_clean_delete); @@ -342,10 +354,14 @@ static void cachefiles_clean_up_object(struct cachefiles_object *object, } cachefiles_unmark_inode_in_use(object, object->file); - if (object->file) { - fput(object->file); - object->file = NULL; - } + + spin_lock(&object->lock); + file = object->file; + object->file = NULL; + spin_unlock(&object->lock); + + if (file) + fput(file); } /* diff --git a/fs/cachefiles/internal.h b/fs/cachefiles/internal.h index 111ad6ecd4baf34cee5b17b9c45dd54bcb472bc6..7215e96fc2cf5f18115e864f0cec02eb1cc4e7a6 100644 --- a/fs/cachefiles/internal.h +++ b/fs/cachefiles/internal.h @@ -37,9 +37,12 @@ enum cachefiles_content { * Cached volume representation. */ struct cachefiles_volume { + refcount_t ref; struct cachefiles_cache *cache; struct list_head cache_link; /* Link in cache->volumes */ struct fscache_volume *vcookie; /* The netfs's representation */ + struct mutex lock; /* Lock for cachefiles_volume */ + bool dir_has_put; /* Indicates cache dir has been put */ struct dentry *dentry; /* The volume dentry */ struct dentry *fanout[256]; /* Fanout subdirs */ }; @@ -405,6 +408,8 @@ static inline void cachefiles_end_secure(struct cachefiles_cache *cache, /* * volume.c */ +void cachefiles_get_volume(struct cachefiles_volume *volume); +void cachefiles_put_volume(struct cachefiles_volume *volume); void cachefiles_acquire_volume(struct fscache_volume *volume); void cachefiles_free_volume(struct fscache_volume *volume); void cachefiles_withdraw_volume(struct cachefiles_volume *volume); diff --git a/fs/cachefiles/io.c b/fs/cachefiles/io.c index 009d23cd435b544d2b7176c86a0f42eb4d2e2a4a..e55c7cc6a997649ab3bcdd7b60115ac266df094c 100644 --- a/fs/cachefiles/io.c +++ b/fs/cachefiles/io.c @@ -12,8 +12,12 @@ #include #include #include +#include #include "internal.h" +static bool cachefiles_buffered_ondemand = true; +module_param_named(buffered_ondemand, cachefiles_buffered_ondemand, bool, 0644); + struct cachefiles_kiocb { struct kiocb iocb; refcount_t ki_refcnt; @@ -78,6 +82,7 @@ static int cachefiles_read(struct netfs_cache_resources *cres, void *term_func_priv) { struct cachefiles_object *object; + struct cachefiles_cache *cache; struct cachefiles_kiocb *ki; struct file *file; unsigned int old_nofs; @@ -89,6 +94,7 @@ static int cachefiles_read(struct netfs_cache_resources *cres, fscache_count_read(); object = cachefiles_cres_object(cres); + cache = object->volume->cache; file = cachefiles_cres_file(cres); _enter("%pD,%li,%llx,%zx/%llx", @@ -146,6 +152,9 @@ static int cachefiles_read(struct netfs_cache_resources *cres, ki->term_func_priv = term_func_priv; ki->was_async = true; + if (cachefiles_in_ondemand_mode(cache) && cachefiles_buffered_ondemand) + ki->iocb.ki_flags &= ~IOCB_DIRECT; + if (ki->term_func) ki->iocb.ki_complete = cachefiles_read_complete; @@ -315,6 +324,9 @@ int __cachefiles_write(struct cachefiles_object *object, ki->was_async = true; ki->b_writing = (len + (1 << cache->bshift) - 1) >> cache->bshift; + if (cachefiles_in_ondemand_mode(cache) && cachefiles_buffered_ondemand) + ki->iocb.ki_flags &= ~IOCB_DIRECT; + if (ki->term_func) ki->iocb.ki_complete = cachefiles_write_complete; atomic_long_add(ki->b_writing, &cache->b_writing); diff --git a/fs/cachefiles/namei.c b/fs/cachefiles/namei.c index 594e41582ae9ca3498f9212268d2703d59b47b5f..6de170514831248ecbd5efa7c54c6239d9219ee0 100644 --- a/fs/cachefiles/namei.c +++ b/fs/cachefiles/namei.c @@ -690,11 +690,6 @@ bool cachefiles_commit_tmpfile(struct cachefiles_cache *cache, } if (!d_is_negative(dentry)) { - if (d_backing_inode(dentry) == file_inode(object->file)) { - success = true; - goto out_dput; - } - ret = cachefiles_unlink(volume->cache, object, fan, dentry, FSCACHE_OBJECT_IS_STALE); if (ret < 0) diff --git a/fs/cachefiles/ondemand.c b/fs/cachefiles/ondemand.c index 2185e2908dba891917b6b0091865fe14ef946663..6a0921f31a8a36113c9e8db45b3d84e48d6ab43a 100644 --- a/fs/cachefiles/ondemand.c +++ b/fs/cachefiles/ondemand.c @@ -61,26 +61,35 @@ static ssize_t cachefiles_ondemand_fd_write_iter(struct kiocb *kiocb, { struct cachefiles_object *object = kiocb->ki_filp->private_data; struct cachefiles_cache *cache = object->volume->cache; - struct file *file = object->file; - size_t len = iter->count; + struct file *file; + size_t len = iter->count, aligned_len = len; loff_t pos = kiocb->ki_pos; const struct cred *saved_cred; int ret; - if (!file) + spin_lock(&object->lock); + file = object->file; + if (!file) { + spin_unlock(&object->lock); return -ENOBUFS; + } + get_file(file); + spin_unlock(&object->lock); cachefiles_begin_secure(cache, &saved_cred); - ret = __cachefiles_prepare_write(object, file, &pos, &len, true); + ret = __cachefiles_prepare_write(object, file, &pos, &aligned_len, true); cachefiles_end_secure(cache, saved_cred); if (ret < 0) - return ret; + goto out; trace_cachefiles_ondemand_fd_write(object, file_inode(file), pos, len); ret = __cachefiles_write(object, file, pos, iter, NULL, NULL); - if (!ret) + if (!ret) { ret = len; - + kiocb->ki_pos += ret; + } +out: + fput(file); return ret; } @@ -88,12 +97,22 @@ static loff_t cachefiles_ondemand_fd_llseek(struct file *filp, loff_t pos, int whence) { struct cachefiles_object *object = filp->private_data; - struct file *file = object->file; + struct file *file; + loff_t ret; - if (!file) + spin_lock(&object->lock); + file = object->file; + if (!file) { + spin_unlock(&object->lock); return -ENOBUFS; + } + get_file(file); + spin_unlock(&object->lock); + + ret = vfs_llseek(file, pos, whence); + fput(file); - return vfs_llseek(file, pos, whence); + return ret; } static long cachefiles_ondemand_fd_ioctl(struct file *filp, unsigned int ioctl, diff --git a/fs/cachefiles/volume.c b/fs/cachefiles/volume.c index 781aac4ef274bdf5459e073de762c045f13f9112..582dd42192b196bc33375d0d04046ddca69d11bf 100644 --- a/fs/cachefiles/volume.c +++ b/fs/cachefiles/volume.c @@ -10,6 +10,19 @@ #include "internal.h" #include +void cachefiles_get_volume(struct cachefiles_volume *volume) +{ + refcount_inc(&volume->ref); +} + +void cachefiles_put_volume(struct cachefiles_volume *volume) +{ + if (refcount_dec_and_test(&volume->ref)) { + mutex_destroy(&volume->lock); + kfree(volume); + } +} + /* * Allocate and set up a volume representation. We make sure all the fanout * directories are created and pinned. @@ -33,6 +46,7 @@ void cachefiles_acquire_volume(struct fscache_volume *vcookie) volume->vcookie = vcookie; volume->cache = cache; INIT_LIST_HEAD(&volume->cache_link); + mutex_init(&volume->lock); cachefiles_begin_secure(cache, &saved_cred); @@ -77,7 +91,17 @@ void cachefiles_acquire_volume(struct fscache_volume *vcookie) cachefiles_end_secure(cache, saved_cred); - vcookie->cache_priv = volume; + /* + * The purpose of introducing volume->ref is twofold: + * 1) To allow cachefiles_object to pin cachefiles_volume. + * 2) To handle the concurrency between cachefiles_free_volume() and + * cachefiles_withdraw_volume() introduced by enabling sync_unhash + * volume, preventing the former from releasing cachefiles_volume and + * causing a use-after-free in the latter. + */ + refcount_set(&volume->ref, 1); + /* Prevent writing vcookie->cache_priv before writing volume->ref. */ + smp_store_release(&vcookie->cache_priv, volume); n_accesses = atomic_inc_return(&vcookie->n_accesses); /* Stop wakeups on dec-to-0 */ trace_fscache_access_volume(vcookie->debug_id, 0, refcount_read(&vcookie->ref), @@ -98,6 +122,7 @@ void cachefiles_acquire_volume(struct fscache_volume *vcookie) error_name: kfree(name); error_vol: + mutex_destroy(&volume->lock); kfree(volume); cachefiles_end_secure(cache, saved_cred); } @@ -111,24 +136,40 @@ static void __cachefiles_free_volume(struct cachefiles_volume *volume) _enter(""); - volume->vcookie->cache_priv = NULL; + mutex_lock(&volume->lock); + if (volume->dir_has_put) { + mutex_unlock(&volume->lock); + return; + } + + volume->dir_has_put = true; for (i = 0; i < 256; i++) cachefiles_put_directory(volume->fanout[i]); cachefiles_put_directory(volume->dentry); - kfree(volume); + mutex_unlock(&volume->lock); } void cachefiles_free_volume(struct fscache_volume *vcookie) { struct cachefiles_volume *volume = vcookie->cache_priv; - if (volume) { + /* + * Prevents access to the cachefiles_cache that has been freed caused + * by the concurrency between cachefiles_free_volume() and + * cachefiles_daemon_release(), the later may kree(cache). + */ + mutex_lock(&volume->lock); + if (!volume->dir_has_put) { spin_lock(&volume->cache->object_list_lock); list_del_init(&volume->cache_link); spin_unlock(&volume->cache->object_list_lock); - __cachefiles_free_volume(volume); } + mutex_unlock(&volume->lock); + + vcookie->cache_priv = NULL; + __cachefiles_free_volume(volume); + cachefiles_put_volume(volume); } void cachefiles_withdraw_volume(struct cachefiles_volume *volume) diff --git a/fs/erofs/fscache.c b/fs/erofs/fscache.c index afc37c9029ce78b9fc3d5e5663d80090968731de..7bcd1d261e7df45a1d23284d058e672f91f61941 100644 --- a/fs/erofs/fscache.c +++ b/fs/erofs/fscache.c @@ -5,6 +5,7 @@ */ #include #include +#include #include "internal.h" static DEFINE_MUTEX(erofs_domain_list_lock); @@ -366,6 +367,9 @@ static int erofs_fscache_register_volume(struct super_block *sb) erofs_err(sb, "failed to register volume for %s", name); ret = volume ? PTR_ERR(volume) : -EOPNOTSUPP; volume = NULL; + } else { + /* enable synchronous unhashing for the associated volumes */ + fscache_set_sync_volume_unhash(volume->cache); } sbi->volume = volume; diff --git a/fs/erofs/internal.h b/fs/erofs/internal.h index 787cc9ff902944d17ccd967b855fb5164d8efe7f..a53152693ae249827f528533d7bf75989dd70f9b 100644 --- a/fs/erofs/internal.h +++ b/fs/erofs/internal.h @@ -176,6 +176,7 @@ struct erofs_sb_info { struct erofs_domain *domain; char *fsid; char *domain_id; + bool ondemand_enabled; }; #define EROFS_SB(sb) ((struct erofs_sb_info *)(sb)->s_fs_info) diff --git a/fs/erofs/super.c b/fs/erofs/super.c index 113414e6f35b964ffdb66e1002bfca98f609440f..43e3a8322a6c324cbcf4ae15730312916d9b9a26 100644 --- a/fs/erofs/super.c +++ b/fs/erofs/super.c @@ -359,9 +359,6 @@ static int erofs_read_superblock(struct super_block *sb) /* handle multiple devices */ ret = erofs_scan_devices(sb, dsb); - - if (erofs_is_fscache_mode(sb)) - erofs_info(sb, "EXPERIMENTAL fscache-based on-demand read feature in use. Use at your own risk!"); out: erofs_put_metabuf(&buf); return ret; @@ -514,12 +511,20 @@ static int erofs_fc_parse_param(struct fs_context *fc, break; #ifdef CONFIG_EROFS_FS_ONDEMAND case Opt_fsid: + if (!sbi->ondemand_enabled) { + errorfc(fc, "fsid option not supported"); + break; + } kfree(sbi->fsid); sbi->fsid = kstrdup(param->string, GFP_KERNEL); if (!sbi->fsid) return -ENOMEM; break; case Opt_domain_id: + if (!sbi->ondemand_enabled) { + errorfc(fc, "domain_id option not supported"); + break; + } kfree(sbi->domain_id); sbi->domain_id = kstrdup(param->string, GFP_KERNEL); if (!sbi->domain_id) @@ -762,10 +767,18 @@ static const struct fs_context_operations erofs_context_ops = { .free = erofs_fc_free, }; +static inline bool erofs_can_init(void) +{ + return READ_ONCE(erofs_enabled) || cachefiles_ondemand_is_enabled(); +} + static int erofs_init_fs_context(struct fs_context *fc) { struct erofs_sb_info *sbi; + if (!erofs_can_init()) + return -EOPNOTSUPP; + sbi = kzalloc(sizeof(*sbi), GFP_KERNEL); if (!sbi) return -ENOMEM; @@ -775,6 +788,7 @@ static int erofs_init_fs_context(struct fs_context *fc) kfree(sbi); return -ENOMEM; } + sbi->ondemand_enabled = cachefiles_ondemand_is_enabled(); fc->s_fs_info = sbi; idr_init(&sbi->devs->tree); diff --git a/fs/fs_ctl.c b/fs/fs_ctl.c new file mode 100644 index 0000000000000000000000000000000000000000..37f8a7567b96708555c3ddff4d7c25c614b8c7bb --- /dev/null +++ b/fs/fs_ctl.c @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2024. Huawei Technologies Co., Ltd */ + +#include +#include +#include + +#if IS_ENABLED(CONFIG_EROFS_FS) || IS_ENABLED(CONFIG_CACHEFILES_ONDEMAND) +static int param_set_bool_on_only_once(const char *s, const struct kernel_param *kp) +{ + int ret; + bool value, *res = kp->arg; + + if (!s) + s = "1"; + + ret = strtobool(s, &value); + if (ret) + return ret; + + if (!value && *res) + return -EBUSY; + + if (value && !*res) + WRITE_ONCE(*res, true); + + return 0; +} +#endif + +#if IS_ENABLED(CONFIG_EROFS_FS) +bool erofs_enabled = true; +EXPORT_SYMBOL(erofs_enabled); +module_param_call(erofs_enabled, param_set_bool_on_only_once, param_get_bool, + &erofs_enabled, 0644); +#endif + +#if IS_ENABLED(CONFIG_CACHEFILES_ONDEMAND) +bool cachefiles_ondemand_enabled; +EXPORT_SYMBOL(cachefiles_ondemand_enabled); +module_param_call(cachefiles_ondemand_enabled, param_set_bool_on_only_once, param_get_bool, + &cachefiles_ondemand_enabled, 0644); +#endif diff --git a/fs/fscache/cache.c b/fs/fscache/cache.c index 9397ed39b0b4ecbdd9c9b5860887162990c2f66d..2ff7ea90f13fe56cc04cbd7b5a0259eaf931663f 100644 --- a/fs/fscache/cache.c +++ b/fs/fscache/cache.c @@ -213,7 +213,6 @@ void fscache_relinquish_cache(struct fscache_cache *cache) fscache_cache_put_prep_failed : fscache_cache_put_relinquish; - cache->ops = NULL; cache->cache_priv = NULL; fscache_set_cache_state(cache, FSCACHE_CACHE_IS_NOT_PRESENT); fscache_put_cache(cache, where); diff --git a/fs/fscache/cookie.c b/fs/fscache/cookie.c index d4d4b3a8b10603794e57333adfbf66562b491639..a42985d5033c21cc989b66ab563ea5f77a791e32 100644 --- a/fs/fscache/cookie.c +++ b/fs/fscache/cookie.c @@ -319,6 +319,26 @@ static bool fscache_cookie_same(const struct fscache_cookie *a, static atomic_t fscache_cookie_debug_id = ATOMIC_INIT(1); +/* + * Increase the volume->n_hash_cookies count. Must be called when a cookie + * associated with the volume is added on the hash link. + */ +static void fscache_increase_volume_hash_cookies(struct fscache_volume *volume) +{ + atomic_inc(&volume->n_hash_cookies); +} + +/* + * Decrease the volume->n_hash_cookies count. Must be called when a cookie + * associated with the volume is unhashed. When the count reaches 0, the volume + * waiting to be unhash needs to be awakened. + */ +static void fscache_decrease_volume_hash_cookies(struct fscache_volume *volume) +{ + if (atomic_dec_and_test(&volume->n_hash_cookies)) + wake_up_var(&volume->n_hash_cookies); +} + /* * Allocate a cookie. */ @@ -425,6 +445,9 @@ static bool fscache_hash_cookie(struct fscache_cookie *candidate) set_bit(FSCACHE_COOKIE_IS_HASHED, &candidate->flags); hlist_bl_unlock(h); + if (fscache_test_sync_volume_unhash(candidate->volume->cache)) + fscache_increase_volume_hash_cookies(candidate->volume); + if (wait_for) { fscache_wait_on_collision(candidate, wait_for); fscache_put_cookie(wait_for, fscache_cookie_put_hash_collision); @@ -933,6 +956,7 @@ static void fscache_cookie_drop_from_lru(struct fscache_cookie *cookie) static void fscache_unhash_cookie(struct fscache_cookie *cookie) { struct hlist_bl_head *h; + struct fscache_volume *volume = cookie->volume; unsigned int bucket; bucket = cookie->key_hash & (ARRAY_SIZE(fscache_cookie_hash) - 1); @@ -942,6 +966,10 @@ static void fscache_unhash_cookie(struct fscache_cookie *cookie) hlist_bl_del(&cookie->hash_link); clear_bit(FSCACHE_COOKIE_IS_HASHED, &cookie->flags); hlist_bl_unlock(h); + + if (fscache_test_sync_volume_unhash(volume->cache)) + fscache_decrease_volume_hash_cookies(volume); + fscache_stat(&fscache_n_relinquishes_dropped); } diff --git a/fs/fscache/volume.c b/fs/fscache/volume.c index cb75c07b5281a5479c731d89ce01dd87e9aa12da..2f8e7bd8ab1c43c0d85571b4700e75732fd3b0b5 100644 --- a/fs/fscache/volume.c +++ b/fs/fscache/volume.c @@ -151,18 +151,23 @@ static bool fscache_is_acquire_pending(struct fscache_volume *volume) return test_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &volume->flags); } -static void fscache_wait_on_volume_collision(struct fscache_volume *candidate, +static int fscache_wait_on_volume_collision(struct fscache_volume *candidate, unsigned int collidee_debug_id) { - wait_on_bit_timeout(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, - TASK_UNINTERRUPTIBLE, 20 * HZ); + int ret; + + ret = wait_on_bit_timeout(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, + TASK_INTERRUPTIBLE, 20 * HZ); + if (ret == -EINTR) + return ret; if (fscache_is_acquire_pending(candidate)) { pr_notice("Potential volume collision new=%08x old=%08x", candidate->debug_id, collidee_debug_id); fscache_stat(&fscache_n_volumes_collision); - wait_on_bit(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, - TASK_UNINTERRUPTIBLE); + return wait_on_bit(&candidate->flags, FSCACHE_VOLUME_ACQUIRE_PENDING, + TASK_INTERRUPTIBLE); } + return 0; } /* @@ -183,8 +188,10 @@ static bool fscache_hash_volume(struct fscache_volume *candidate) hlist_bl_lock(h); hlist_bl_for_each_entry(cursor, p, h, hash_link) { if (fscache_volume_same(candidate, cursor)) { - if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags)) + if (!test_bit(FSCACHE_VOLUME_RELINQUISHED, &cursor->flags)) { + fscache_see_volume(cursor, fscache_volume_collision); goto collision; + } fscache_see_volume(cursor, fscache_volume_get_hash_collision); set_bit(FSCACHE_VOLUME_COLLIDED_WITH, &cursor->flags); set_bit(FSCACHE_VOLUME_ACQUIRE_PENDING, &candidate->flags); @@ -196,12 +203,16 @@ static bool fscache_hash_volume(struct fscache_volume *candidate) hlist_bl_add_head(&candidate->hash_link, h); hlist_bl_unlock(h); - if (fscache_is_acquire_pending(candidate)) - fscache_wait_on_volume_collision(candidate, collidee_debug_id); + if (fscache_is_acquire_pending(candidate) && + fscache_wait_on_volume_collision(candidate, collidee_debug_id)) { + hlist_bl_lock(h); + hlist_bl_del_init(&candidate->hash_link); + pr_err("Wait duplicate volume unhashed interrupted\n"); + goto collision; + } return true; collision: - fscache_see_volume(cursor, fscache_volume_collision); hlist_bl_unlock(h); return false; } @@ -322,8 +333,7 @@ void fscache_create_volume(struct fscache_volume *volume, bool wait) } return; no_wait: - clear_bit_unlock(FSCACHE_VOLUME_CREATING, &volume->flags); - wake_up_bit(&volume->flags, FSCACHE_VOLUME_CREATING); + clear_and_wake_up_bit(FSCACHE_VOLUME_CREATING, &volume->flags); } /* @@ -379,12 +389,21 @@ static void fscache_unhash_volume(struct fscache_volume *volume) h = &fscache_volume_hash[bucket]; hlist_bl_lock(h); - hlist_bl_del(&volume->hash_link); + hlist_bl_del_init(&volume->hash_link); if (test_bit(FSCACHE_VOLUME_COLLIDED_WITH, &volume->flags)) fscache_wake_pending_volume(volume, h); hlist_bl_unlock(h); } +/* + * Clear the volume->cache_priv. + */ +static void fscache_clear_volume_priv(struct fscache_volume *volume) +{ + if (volume->cache_priv) + volume->cache->ops->free_volume(volume); +} + /* * Drop a cache's volume attachments. */ @@ -392,14 +411,7 @@ static void fscache_free_volume(struct fscache_volume *volume) { struct fscache_cache *cache = volume->cache; - if (volume->cache_priv) { - __fscache_begin_volume_access(volume, NULL, - fscache_access_relinquish_volume); - if (volume->cache_priv) - cache->ops->free_volume(volume); - fscache_end_volume_access(volume, NULL, - fscache_access_relinquish_volume_end); - } + fscache_clear_volume_priv(volume); down_write(&fscache_addremove_sem); list_del_init(&volume->proc_link); @@ -451,6 +463,24 @@ void __fscache_relinquish_volume(struct fscache_volume *volume, memcpy(volume->coherency, coherency_data, volume->coherency_len); } + if (fscache_test_sync_volume_unhash(volume->cache)) { + int ret = wait_var_event_killable(&volume->n_hash_cookies, + (atomic_read_acquire(&volume->n_hash_cookies) == 0)); + if (ret == -ERESTARTSYS) { + pr_info("Waiting for unhashing has been interrupted!" \ + "(n_hash_cookies %d)\n", + atomic_read(&volume->n_hash_cookies)); + } else if (!hlist_bl_unhashed(&volume->hash_link)) { + /* + * To ensure that the corresponding cache entries can + * be created on the next mount, thereby completing + * the mount successfully. + */ + fscache_clear_volume_priv(volume); + fscache_unhash_volume(volume); + } + } + fscache_put_volume(volume, fscache_volume_put_relinquish); } EXPORT_SYMBOL(__fscache_relinquish_volume); diff --git a/include/linux/fs.h b/include/linux/fs.h index db0ad52d3b489127cfe185ba2f10aea2757e1607..0efc8bea182dc4456d52b0393d8743bc3771e473 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -3582,4 +3582,21 @@ static inline void fs_file_read_update_args_by_trace(struct kiocb *iocb) {} static inline void fs_file_read_do_trace(struct kiocb *iocb) {} #endif +#if IS_ENABLED(CONFIG_EROFS_FS) +extern bool erofs_enabled; +#endif + +#if IS_ENABLED(CONFIG_CACHEFILES_ONDEMAND) +extern bool cachefiles_ondemand_enabled; +static inline bool cachefiles_ondemand_is_enabled(void) +{ + return READ_ONCE(cachefiles_ondemand_enabled); +} +#else +static inline bool cachefiles_ondemand_is_enabled(void) +{ + return false; +} +#endif + #endif /* _LINUX_FS_H */ diff --git a/include/linux/fscache-cache.h b/include/linux/fscache-cache.h index 151043af41a1c821f5cfdee46e673966ed03c726..642fc1276c632954a3e7013c7f4c96777cf119cd 100644 --- a/include/linux/fscache-cache.h +++ b/include/linux/fscache-cache.h @@ -44,8 +44,9 @@ struct fscache_cache { unsigned int debug_id; enum fscache_cache_state state; char *name; + KABI_USE(1, unsigned long flags) +#define FSCACHE_CACHE_SYNC_VOLUME_UNHASH 0 /* Mark to enable volume sync unhash */ - KABI_RESERVE(1) KABI_RESERVE(2) KABI_RESERVE(3) KABI_RESERVE(4) @@ -199,6 +200,31 @@ static inline void fscache_wait_for_objects(struct fscache_cache *cache) atomic_read(&cache->object_count) == 0); } +/** + * fscache_set_sync_volume_unhash - Enable the synchronization unhash mechanism + * @cache: The cache to set + * + * Enable the volume synchronization wait-unhash mechanism. The volume will be + * unhashed in advance, without relying on the reference count reaching zero. + */ +static inline void fscache_set_sync_volume_unhash(struct fscache_cache *cache) +{ + set_bit(FSCACHE_CACHE_SYNC_VOLUME_UNHASH, &cache->flags); +} + +/** + * fscache_test_sync_volume_unhash - Check whether the synchronization + * unhash mechanism is enabled + * @cache: The cache to query + * + * Indicate whether the volume synchronization wait-unhash mechanism is + * currently enabled. + */ +static inline bool fscache_test_sync_volume_unhash(struct fscache_cache *cache) +{ + return test_bit(FSCACHE_CACHE_SYNC_VOLUME_UNHASH, &cache->flags); +} + #ifdef CONFIG_FSCACHE_STATS extern atomic_t fscache_n_read; extern atomic_t fscache_n_write; diff --git a/include/linux/fscache.h b/include/linux/fscache.h index 3afe007834d18e99ad345751d1efbcc0219f9943..99cb06ee3f6548bcafe68864d270c1af219d1d8e 100644 --- a/include/linux/fscache.h +++ b/include/linux/fscache.h @@ -90,7 +90,7 @@ struct fscache_volume { #define FSCACHE_VOLUME_CREATING 4 /* Volume is being created on disk */ u8 coherency_len; /* Length of the coherency data */ - KABI_RESERVE(1) + KABI_USE(1, atomic_t n_hash_cookies) /* Number of hashed data cookies in volume */ KABI_RESERVE(2) KABI_RESERVE(3) KABI_RESERVE(4)