diff --git a/arch/arm64/include/asm/xcall.h b/arch/arm64/include/asm/xcall.h index c9143b7d2096ea035f94e61f902da7cb97eb5ed7..b6bcc120575a5cf34bbfde32b95efbefa814de73 100644 --- a/arch/arm64/include/asm/xcall.h +++ b/arch/arm64/include/asm/xcall.h @@ -4,6 +4,7 @@ #include #include +#include #include #include #include @@ -48,6 +49,7 @@ struct xcall_area { refcount_t ref; struct xcall *xcall; void *sys_call_data[NR_syscalls]; + struct mmu_notifier xcall_mmu_notifier; }; extern const syscall_fn_t *default_sys_call_table(void); diff --git a/arch/arm64/kernel/xcall/core.c b/arch/arm64/kernel/xcall/core.c index a88c4ed6e575c9ed6cb53e3800dda8481ec4ce9d..8a3e984f2f6b6650fe0409778850f5f1e2689d09 100644 --- a/arch/arm64/kernel/xcall/core.c +++ b/arch/arm64/kernel/xcall/core.c @@ -277,6 +277,9 @@ void clear_xcall_area(struct mm_struct *mm) if (!refcount_dec_and_test(&area->ref)) return; + if (area->xcall_mmu_notifier.mm == mm && atomic_read(&mm->mm_count) > 1) + mmu_notifier_unregister(&area->xcall_mmu_notifier, mm); + if (area->xcall) put_xcall(area->xcall); diff --git a/arch/arm64/kernel/xcall/proc.c b/arch/arm64/kernel/xcall/proc.c index 12032120c7d6d3d0eca792dc9df08b6436efab85..0e3b714ac73023e06d3c3d6cb2d5568b6228c39f 100644 --- a/arch/arm64/kernel/xcall/proc.c +++ b/arch/arm64/kernel/xcall/proc.c @@ -16,6 +16,8 @@ static LIST_HEAD(comm_list); static DECLARE_RWSEM(comm_rwsem); +struct proc_dir_entry *root_xcall_dir; + static void free_xcall_comm(struct xcall_comm *info) { if (!info) @@ -213,6 +215,19 @@ static ssize_t xcall_comm_write(struct file *file, return ret ? ret : nbytes; } +struct proc_dir_entry *xcall_subdir_create(const char *name) +{ + return proc_mkdir(name, root_xcall_dir); +} +EXPORT_SYMBOL(xcall_subdir_create); + +struct proc_dir_entry *xcall_proc_create(const char *name, umode_t mode, + struct proc_dir_entry *parent, const struct proc_ops *proc_ops) +{ + return proc_create(name, mode, parent, proc_ops); +} +EXPORT_SYMBOL(xcall_proc_create); + static const struct proc_ops xcall_comm_ops = { .proc_open = xcall_comm_open, .proc_read = seq_read, @@ -223,8 +238,6 @@ static const struct proc_ops xcall_comm_ops = { static int __init xcall_proc_init(void) { - struct proc_dir_entry *root_xcall_dir; - if (!static_key_enabled(&xcall_enable)) return 0; diff --git a/drivers/staging/xcall/prefetch.c b/drivers/staging/xcall/prefetch.c index 923ca2a23c98d6e7f4c1a070200ec644da872e73..572bc4c90690f60e3209b198a0806722cc27c6bd 100644 --- a/drivers/staging/xcall/prefetch.c +++ b/drivers/staging/xcall/prefetch.c @@ -17,11 +17,15 @@ #include -#define MAX_FD 100 +#define MAX_FD 1024 #define XCALL_CACHE_PAGE_ORDER 2 #define XCALL_CACHE_BUF_SIZE ((1 << XCALL_CACHE_PAGE_ORDER) * PAGE_SIZE) +#define current_epoll_events() \ + ((struct epoll_event *) \ + ((((struct xcall_area *)(current->mm->xcall))->sys_call_data)[__NR_epoll_ctl])) + #define current_prefetch_items() \ ((struct prefetch_item *) \ ((((struct xcall_area *)(current->mm->xcall))->sys_call_data)[__NR_epoll_pwait])) @@ -31,7 +35,7 @@ static DEFINE_PER_CPU_ALIGNED(unsigned long, xcall_cache_miss); static struct workqueue_struct *rc_work; static struct cpumask xcall_mask; -struct proc_dir_entry *xcall_proc_dir, *prefetch_dir, *xcall_mask_dir; +struct proc_dir_entry *prefetch_proc_dir, *prefetch_dir, *xcall_mask_dir; enum cache_state { XCALL_CACHE_NONE = 0, @@ -200,21 +204,32 @@ static int get_async_prefetch_cpu(struct prefetch_item *pfi) return pfi->cpu; } -static void xcall_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) +static void prefetch_pfi_release(struct mmu_notifier *mn, struct mm_struct *mm) { struct xcall_area *area = mm_xcall_area(mm); - void *area_private_data = NULL; + struct prefetch_item *prefetch_items = NULL; + struct epoll_event *events = NULL; + int i; + + events = xchg(&area->sys_call_data[__NR_epoll_ctl], NULL); + if (events) + kfree(events); - area_private_data = xchg(&area->sys_call_data[__NR_epoll_pwait], NULL); - kfree(area_private_data); + prefetch_items = xchg(&area->sys_call_data[__NR_epoll_pwait], NULL); + if (!prefetch_items) + return; + + for (i = 0; i < MAX_FD; i++) { + cancel_work_sync(&prefetch_items[i].work); + if (prefetch_items[i].cache_pages) + __free_pages(prefetch_items[i].cache_pages, XCALL_CACHE_PAGE_ORDER); + prefetch_items[i].cache = NULL; + } + kfree(prefetch_items); } static struct mmu_notifier_ops xcall_mmu_notifier_ops = { - .release = xcall_mm_release, -}; - -static struct mmu_notifier xcall_mmu_notifier = { - .ops = &xcall_mmu_notifier_ops, + .release = prefetch_pfi_release, }; static void xcall_cancel_work(unsigned int fd) @@ -321,22 +336,25 @@ static long __do_sys_epoll_create(struct pt_regs *regs) int i; struct xcall_area *area = mm_xcall_area(current->mm); struct prefetch_item *items = NULL; + struct epoll_events *events = NULL; ret = default_sys_call_table()[__NR_epoll_create1](regs); if (ret < 0) return ret; - if (current_prefetch_items()) return ret; + events = kcalloc(MAX_FD, sizeof(struct epoll_event), GFP_KERNEL); + if (!events) + return ret; + if (cmpxchg(&area->sys_call_data[__NR_epoll_ctl], NULL, events)) + goto free_events; + items = kcalloc(MAX_FD, sizeof(struct prefetch_item), GFP_KERNEL); if (!items) - return -ENOMEM; - - if (cmpxchg(&area->sys_call_data[__NR_epoll_pwait], NULL, items)) { - kfree(items); - return ret; - } + goto free_events; + if (cmpxchg(&area->sys_call_data[__NR_epoll_pwait], NULL, items)) + goto free_items; for (i = 0; i < MAX_FD; i++) { items[i].cache_pages = alloc_pages(GFP_KERNEL_ACCOUNT | __GFP_ZERO, @@ -354,7 +372,15 @@ static long __do_sys_epoll_create(struct pt_regs *regs) items[i].file = NULL; set_prefetch_numa_cpu(&items[i]); } - mmu_notifier_register(&xcall_mmu_notifier, current->mm); + + area->xcall_mmu_notifier.ops = &xcall_mmu_notifier_ops; + mmu_notifier_register(&area->xcall_mmu_notifier, current->mm); + return ret; + +free_items: + kfree(items); +free_events: + kfree(events); return ret; } @@ -380,12 +406,10 @@ static long __do_sys_epoll_ctl(struct pt_regs *regs) if (!file) return ret; - if (!sock_from_file(file)) { - fput(file); - return ret; - } - if (cmpxchg(&pfi->file, NULL, file)) - fput(file); + if (sock_from_file(file)) + cmpxchg(&pfi->file, NULL, file); + + fput(file); break; case EPOLL_CTL_DEL: xcall_cancel_work(fd); @@ -399,8 +423,8 @@ static long __do_sys_epoll_pwait(struct pt_regs *regs) { void __user *buf = (void *)regs->regs[1]; struct prefetch_item *pfi = NULL; - struct epoll_event events[MAX_FD] = {0}; - int i, fd, cpu, prefech_task_num; + struct epoll_event *events = current_epoll_events(); + int i, fd, cpu, prefetch_task_num; long ret; ret = default_sys_call_table()[__NR_epoll_pwait](regs); @@ -410,11 +434,11 @@ static long __do_sys_epoll_pwait(struct pt_regs *regs) if (!current_prefetch_items()) return ret; - prefech_task_num = ret > MAX_FD ? MAX_FD : ret; - if (copy_from_user(events, buf, prefech_task_num * sizeof(struct epoll_event))) + prefetch_task_num = ret > MAX_FD ? MAX_FD : ret; + if (copy_from_user(events, buf, prefetch_task_num * sizeof(struct epoll_event))) return ret; - for (i = 0; i < prefech_task_num; i++) { + for (i = 0; i < prefetch_task_num; i++) { fd = events[i].data; if (!(events[i].events & EPOLLIN) || fd >= MAX_FD) continue; @@ -446,7 +470,6 @@ static long __do_sys_close(struct pt_regs *regs) pfi_old_file = pfi->file; pfi_new_file = cmpxchg(&pfi->file, pfi_old_file, NULL); if (pfi_new_file == pfi_old_file) { - fput(pfi_old_file); atomic_set(&pfi->state, XCALL_CACHE_NONE); pfi->len = 0; pfi->pos = 0; @@ -474,7 +497,7 @@ static long __do_sys_read(struct pt_regs *regs) /* MANDATORY */ struct xcall_prog xcall_prefetch_prog = { - .name = "xcall_prefetch", + .name = "prefetch", .owner = THIS_MODULE, .objs = { { @@ -503,14 +526,14 @@ struct xcall_prog xcall_prefetch_prog = { static int __init init_xcall_prefetch_procfs(void) { - xcall_proc_dir = proc_mkdir("xcall_feature", NULL); - if (!xcall_proc_dir) + prefetch_proc_dir = xcall_subdir_create("prefetch"); + if (!prefetch_proc_dir) return -ENOMEM; - prefetch_dir = proc_create("prefetch", 0640, xcall_proc_dir, - &xcall_prefetch_fops); + prefetch_dir = xcall_proc_create("prefetch", 0640, prefetch_proc_dir, + &xcall_prefetch_fops); if (!prefetch_dir) - goto rm_xcall_proc_dir; - xcall_mask_dir = proc_create("cpu_list", 0640, xcall_proc_dir, + goto rm_prefetch_proc_dir; + xcall_mask_dir = proc_create("cpu_list", 0640, prefetch_proc_dir, &xcall_mask_fops); if (!xcall_mask_dir) goto rm_prefetch_dir; @@ -520,8 +543,8 @@ static int __init init_xcall_prefetch_procfs(void) rm_prefetch_dir: proc_remove(prefetch_dir); -rm_xcall_proc_dir: - proc_remove(xcall_proc_dir); +rm_prefetch_proc_dir: + proc_remove(prefetch_proc_dir); return -ENOMEM; } @@ -549,7 +572,7 @@ static int __init xcall_prefetch_init(void) remove_dir: proc_remove(prefetch_dir); proc_remove(xcall_mask_dir); - proc_remove(xcall_proc_dir); + proc_remove(prefetch_proc_dir); destroy_queue: destroy_workqueue(rc_work); return ret; @@ -564,8 +587,8 @@ static void __exit xcall_prefetch_exit(void) proc_remove(prefetch_dir); if (xcall_mask_dir) proc_remove(xcall_mask_dir); - if (xcall_proc_dir) - proc_remove(xcall_proc_dir); + if (prefetch_proc_dir) + proc_remove(prefetch_proc_dir); xcall_prog_unregister(&xcall_prefetch_prog); } diff --git a/include/linux/xcall.h b/include/linux/xcall.h index 510aebe4e7c0614daf828da0aa41e92511b9669d..215542097dfa0ff144f3cbda6a08999f5068990d 100644 --- a/include/linux/xcall.h +++ b/include/linux/xcall.h @@ -7,6 +7,7 @@ #define _LINUX_XCALL_H #include +#include struct vm_area_struct; struct mm_struct; @@ -32,6 +33,9 @@ struct xcall_prog { extern int xcall_prog_register(struct xcall_prog *prog); extern void xcall_prog_unregister(struct xcall_prog *prog); extern void mm_init_xcall_area(struct mm_struct *mm, struct task_struct *p); +struct proc_dir_entry *xcall_subdir_create(const char *name); +struct proc_dir_entry *xcall_proc_create(const char *name, umode_t mode, + struct proc_dir_entry *parent, const struct proc_ops *proc_ops); extern void clear_xcall_area(struct mm_struct *mm); extern int xcall_mmap(struct vm_area_struct *vma, struct mm_struct *mm); #else /* !CONFIG_DYNAMIC_XCALL */