From a6ee5d3100d0699b583193e9a394f9d4793b86c7 Mon Sep 17 00:00:00 2001 From: "minjie.yu" Date: Wed, 12 Jun 2024 14:10:26 +0800 Subject: [PATCH] staging: add ashmem and ion driver Signed-off-by: minjie.yu --- drivers/staging/Kconfig | 2 + drivers/staging/Makefile | 1 + drivers/staging/android/Kconfig | 21 + drivers/staging/android/Makefile | 6 + drivers/staging/android/TODO | 13 + drivers/staging/android/ashmem.c | 1192 +++++++++++++++++ drivers/staging/android/ashmem.h | 47 + drivers/staging/android/ion/Kconfig | 14 + drivers/staging/android/ion/Makefile | 4 + drivers/staging/android/ion/heaps/Kconfig | 15 + drivers/staging/android/ion/heaps/Makefile | 5 + .../staging/android/ion/heaps/ion_cma_heap.c | 151 +++ .../staging/android/ion/heaps/ion_page_pool.c | 172 +++ .../staging/android/ion/heaps/ion_page_pool.h | 67 + .../android/ion/heaps/ion_system_heap.c | 344 +++++ drivers/staging/android/ion/ion.c | 518 +++++++ drivers/staging/android/ion/ion_buffer.c | 278 ++++ drivers/staging/android/ion/ion_dma_buf.c | 359 +++++ drivers/staging/android/ion/ion_heap.c | 291 ++++ drivers/staging/android/ion/ion_private.h | 58 + .../staging/android/ion/ion_protected_heap.c | 526 ++++++++ drivers/staging/android/ion/ion_trace.h | 55 + drivers/staging/android/uapi/ashmem.h | 54 + drivers/staging/android/uapi/ion.h | 127 ++ 24 files changed, 4320 insertions(+) create mode 100644 drivers/staging/android/Kconfig create mode 100644 drivers/staging/android/Makefile create mode 100644 drivers/staging/android/TODO create mode 100644 drivers/staging/android/ashmem.c create mode 100644 drivers/staging/android/ashmem.h create mode 100644 drivers/staging/android/ion/Kconfig create mode 100644 drivers/staging/android/ion/Makefile create mode 100644 drivers/staging/android/ion/heaps/Kconfig create mode 100644 drivers/staging/android/ion/heaps/Makefile create mode 100644 drivers/staging/android/ion/heaps/ion_cma_heap.c create mode 100644 drivers/staging/android/ion/heaps/ion_page_pool.c create mode 100644 drivers/staging/android/ion/heaps/ion_page_pool.h create mode 100644 drivers/staging/android/ion/heaps/ion_system_heap.c create mode 100644 drivers/staging/android/ion/ion.c create mode 100644 drivers/staging/android/ion/ion_buffer.c create mode 100644 drivers/staging/android/ion/ion_dma_buf.c create mode 100644 drivers/staging/android/ion/ion_heap.c create mode 100644 drivers/staging/android/ion/ion_private.h create mode 100644 drivers/staging/android/ion/ion_protected_heap.c create mode 100644 drivers/staging/android/ion/ion_trace.h create mode 100644 drivers/staging/android/uapi/ashmem.h create mode 100644 drivers/staging/android/uapi/ion.h diff --git a/drivers/staging/Kconfig b/drivers/staging/Kconfig index 5dc634897b48..7dc75c5400d2 100644 --- a/drivers/staging/Kconfig +++ b/drivers/staging/Kconfig @@ -54,6 +54,8 @@ source "drivers/staging/nvec/Kconfig" source "drivers/staging/media/Kconfig" +source "drivers/staging/android/Kconfig" + source "drivers/staging/board/Kconfig" source "drivers/staging/gdm724x/Kconfig" diff --git a/drivers/staging/Makefile b/drivers/staging/Makefile index aec6e94a3c69..a842711b1a6b 100644 --- a/drivers/staging/Makefile +++ b/drivers/staging/Makefile @@ -17,6 +17,7 @@ obj-$(CONFIG_IIO) += iio/ obj-$(CONFIG_FB_SM750) += sm750fb/ obj-$(CONFIG_USB_EMXX) += emxx_udc/ obj-$(CONFIG_MFD_NVEC) += nvec/ +obj-$(CONFIG_ANDROID) += android/ obj-$(CONFIG_STAGING_BOARD) += board/ obj-$(CONFIG_LTE_GDM724X) += gdm724x/ obj-$(CONFIG_FB_TFT) += fbtft/ diff --git a/drivers/staging/android/Kconfig b/drivers/staging/android/Kconfig new file mode 100644 index 000000000000..8d8fd5c29349 --- /dev/null +++ b/drivers/staging/android/Kconfig @@ -0,0 +1,21 @@ +# SPDX-License-Identifier: GPL-2.0 +menu "Android" + +if ANDROID + +config ASHMEM + bool "Enable the Anonymous Shared Memory Subsystem" + depends on SHMEM + help + The ashmem subsystem is a new shared memory allocator, similar to + POSIX SHM but with different behavior and sporting a simpler + file-based API. + + It is, in theory, a good memory allocator for low-memory devices, + because it can discard shared memory units when under memory pressure. + +source "drivers/staging/android/ion/Kconfig" + +endif # if ANDROID + +endmenu diff --git a/drivers/staging/android/Makefile b/drivers/staging/android/Makefile new file mode 100644 index 000000000000..3b66cd0b0ec5 --- /dev/null +++ b/drivers/staging/android/Makefile @@ -0,0 +1,6 @@ +# SPDX-License-Identifier: GPL-2.0 +ccflags-y += -I$(src) # needed for trace events + +obj-y += ion/ + +obj-$(CONFIG_ASHMEM) += ashmem.o diff --git a/drivers/staging/android/TODO b/drivers/staging/android/TODO new file mode 100644 index 000000000000..80eccfaf6db5 --- /dev/null +++ b/drivers/staging/android/TODO @@ -0,0 +1,13 @@ +TODO: + - sparse fixes + - rename files to be not so "generic" + - add proper arch dependencies as needed + - audit userspace interfaces to make sure they are sane + + +ion/ + - Split /dev/ion up into multiple nodes (e.g. /dev/ion/heap0) + - Better test framework (integration with VGEM was suggested) + +Please send patches to Greg Kroah-Hartman and Cc: +Arve Hjønnevåg and Riley Andrews diff --git a/drivers/staging/android/ashmem.c b/drivers/staging/android/ashmem.c new file mode 100644 index 000000000000..a1d11cc23ff5 --- /dev/null +++ b/drivers/staging/android/ashmem.c @@ -0,0 +1,1192 @@ +// SPDX-License-Identifier: GPL-2.0 +/* mm/ashmem.c + * + * Anonymous Shared Memory Subsystem, ashmem + * + * Copyright (C) 2008 Google, Inc. + * + * Robert Love + */ + +#define pr_fmt(fmt) "ashmem: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ashmem.h" + +#define ASHMEM_NAME_PREFIX "dev/ashmem/" +#define ASHMEM_NAME_PREFIX_LEN (sizeof(ASHMEM_NAME_PREFIX) - 1) +#define ASHMEM_FULL_NAME_LEN (ASHMEM_NAME_LEN + ASHMEM_NAME_PREFIX_LEN) + +#ifdef CONFIG_PURGEABLE_ASHMEM +#define PURGEABLE_ASHMEM_INIT_REFCOUNT 1 +#define PURGEABLE_ASHMEM_UNPIN_REFCOUNT 0 +#define PURGEABLE_ASHMEM_PIN_OFFSET 0 +#define PURGEABLE_ASHMEM_PIN_LEN 0 +#endif + +/** + * struct ashmem_area - The anonymous shared memory area + * @name: The optional name in /proc/pid/maps + * @unpinned_list: The list of all ashmem areas + * @file: The shmem-based backing file + * @size: The size of the mapping, in bytes + * @prot_mask: The allowed protection bits, as vm_flags + * + * The lifecycle of this structure is from our parent file's open() until + * its release(). It is also protected by 'ashmem_mutex' + * + * Warning: Mappings do NOT pin this structure; It dies on close() + */ +struct ashmem_area { + char name[ASHMEM_FULL_NAME_LEN]; + struct list_head unpinned_list; + struct file *file; + size_t size; + unsigned long prot_mask; +#ifdef CONFIG_PURGEABLE_ASHMEM + bool is_purgeable; + bool purged; + unsigned int id; + unsigned int create_time; + int ref_count; +#endif +}; + +/** + * struct ashmem_range - A range of unpinned/evictable pages + * @lru: The entry in the LRU list + * @unpinned: The entry in its area's unpinned list + * @asma: The associated anonymous shared memory area. + * @pgstart: The starting page (inclusive) + * @pgend: The ending page (inclusive) + * @purged: The purge status (ASHMEM_NOT or ASHMEM_WAS_PURGED) + * + * The lifecycle of this structure is from unpin to pin. + * It is protected by 'ashmem_mutex' + */ +struct ashmem_range { + struct list_head lru; + struct list_head unpinned; + struct ashmem_area *asma; + size_t pgstart; + size_t pgend; + unsigned int purged; +}; + +/* LRU list of unpinned pages, protected by ashmem_mutex */ +static LIST_HEAD(ashmem_lru_list); + +static atomic_t ashmem_shrink_inflight = ATOMIC_INIT(0); +static DECLARE_WAIT_QUEUE_HEAD(ashmem_shrink_wait); + +/* + * long lru_count - The count of pages on our LRU list. + * + * This is protected by ashmem_mutex. + */ +static unsigned long lru_count; + +/* + * ashmem_mutex - protects the list of and each individual ashmem_area + * + * Lock Ordering: ashmex_mutex -> i_mutex -> i_alloc_sem + */ +static DEFINE_MUTEX(ashmem_mutex); + +static struct kmem_cache *ashmem_area_cachep __read_mostly; +static struct kmem_cache *ashmem_range_cachep __read_mostly; + +/* + * A separate lockdep class for the backing shmem inodes to resolve the lockdep + * warning about the race between kswapd taking fs_reclaim before inode_lock + * and write syscall taking inode_lock and then fs_reclaim. + * Note that such race is impossible because ashmem does not support write + * syscalls operating on the backing shmem. + */ +static struct lock_class_key backing_shmem_inode_class; + +void ashmem_mutex_lock(void) +{ + mutex_lock(&ashmem_mutex); +} + +void ashmem_mutex_unlock(void) +{ + mutex_unlock(&ashmem_mutex); +} + +static inline unsigned long range_size(struct ashmem_range *range) +{ + return range->pgend - range->pgstart + 1; +} + +static inline bool range_on_lru(struct ashmem_range *range) +{ + return range->purged == ASHMEM_NOT_PURGED; +} + +static inline bool page_range_subsumes_range(struct ashmem_range *range, + size_t start, size_t end) +{ + return (range->pgstart >= start) && (range->pgend <= end); +} + +static inline bool page_range_subsumed_by_range(struct ashmem_range *range, + size_t start, size_t end) +{ + return (range->pgstart <= start) && (range->pgend >= end); +} + +static inline bool page_in_range(struct ashmem_range *range, size_t page) +{ + return (range->pgstart <= page) && (range->pgend >= page); +} + +static inline bool page_range_in_range(struct ashmem_range *range, + size_t start, size_t end) +{ + return page_in_range(range, start) || page_in_range(range, end) || + page_range_subsumes_range(range, start, end); +} + +static inline bool range_before_page(struct ashmem_range *range, + size_t page) +{ + return range->pgend < page; +} + +#ifdef CONFIG_PURGEABLE_ASHMEM +static inline bool is_purgeable_ashmem(const struct ashmem_area *asma) +{ + return (asma && asma->is_purgeable); +} +#endif +#define PROT_MASK (PROT_EXEC | PROT_READ | PROT_WRITE) + +/** + * lru_add() - Adds a range of memory to the LRU list + * @range: The memory range being added. + * + * The range is first added to the end (tail) of the LRU list. + * After this, the size of the range is added to @lru_count + */ +static inline void lru_add(struct ashmem_range *range) +{ + list_add_tail(&range->lru, &ashmem_lru_list); + lru_count += range_size(range); +} + +/** + * lru_del() - Removes a range of memory from the LRU list + * @range: The memory range being removed + * + * The range is first deleted from the LRU list. + * After this, the size of the range is removed from @lru_count + */ +static inline void lru_del(struct ashmem_range *range) +{ + list_del(&range->lru); + lru_count -= range_size(range); +} + +/** + * range_alloc() - Allocates and initializes a new ashmem_range structure + * @asma: The associated ashmem_area + * @prev_range: The previous ashmem_range in the sorted asma->unpinned list + * @purged: Initial purge status (ASMEM_NOT_PURGED or ASHMEM_WAS_PURGED) + * @start: The starting page (inclusive) + * @end: The ending page (inclusive) + * + * This function is protected by ashmem_mutex. + */ +static void range_alloc(struct ashmem_area *asma, + struct ashmem_range *prev_range, unsigned int purged, + size_t start, size_t end, + struct ashmem_range **new_range) +{ + struct ashmem_range *range = *new_range; + + *new_range = NULL; + range->asma = asma; + range->pgstart = start; + range->pgend = end; + range->purged = purged; + + list_add_tail(&range->unpinned, &prev_range->unpinned); + + if (range_on_lru(range)) + lru_add(range); +} + +/** + * range_del() - Deletes and deallocates an ashmem_range structure + * @range: The associated ashmem_range that has previously been allocated + */ +static void range_del(struct ashmem_range *range) +{ + list_del(&range->unpinned); + if (range_on_lru(range)) + lru_del(range); + kmem_cache_free(ashmem_range_cachep, range); +} + +/** + * range_shrink() - Shrinks an ashmem_range + * @range: The associated ashmem_range being shrunk + * @start: The starting byte of the new range + * @end: The ending byte of the new range + * + * This does not modify the data inside the existing range in any way - It + * simply shrinks the boundaries of the range. + * + * Theoretically, with a little tweaking, this could eventually be changed + * to range_resize, and expand the lru_count if the new range is larger. + */ +static inline void range_shrink(struct ashmem_range *range, + size_t start, size_t end) +{ + size_t pre = range_size(range); + + range->pgstart = start; + range->pgend = end; + + if (range_on_lru(range)) + lru_count -= pre - range_size(range); +} + +/** + * ashmem_open() - Opens an Anonymous Shared Memory structure + * @inode: The backing file's index node(?) + * @file: The backing file + * + * Please note that the ashmem_area is not returned by this function - It is + * instead written to "file->private_data". + * + * Return: 0 if successful, or another code if unsuccessful. + */ +static int ashmem_open(struct inode *inode, struct file *file) +{ + struct ashmem_area *asma; + int ret; + + ret = generic_file_open(inode, file); + if (ret) + return ret; + + asma = kmem_cache_zalloc(ashmem_area_cachep, GFP_KERNEL); + if (!asma) + return -ENOMEM; + + INIT_LIST_HEAD(&asma->unpinned_list); + memcpy(asma->name, ASHMEM_NAME_PREFIX, ASHMEM_NAME_PREFIX_LEN); + asma->prot_mask = PROT_MASK; + file->private_data = asma; +#ifdef CONFIG_PURGEABLE_ASHMEM + asma->ref_count = PURGEABLE_ASHMEM_INIT_REFCOUNT; + asma->is_purgeable = false; + asma->purged = false; + asma->id = current->pid; + asma->create_time = ktime_get(); +#endif + return 0; +} + +/** + * ashmem_release() - Releases an Anonymous Shared Memory structure + * @ignored: The backing file's Index Node(?) - It is ignored here. + * @file: The backing file + * + * Return: 0 if successful. If it is anything else, go have a coffee and + * try again. + */ +static int ashmem_release(struct inode *ignored, struct file *file) +{ + struct ashmem_area *asma = file->private_data; + struct ashmem_range *range, *next; + + mutex_lock(&ashmem_mutex); + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) + range_del(range); + mutex_unlock(&ashmem_mutex); + + if (asma->file) + fput(asma->file); + kmem_cache_free(ashmem_area_cachep, asma); + + return 0; +} + +static ssize_t ashmem_read_iter(struct kiocb *iocb, struct iov_iter *iter) +{ + struct ashmem_area *asma = iocb->ki_filp->private_data; + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* If size is not set, or set to 0, always return EOF. */ + if (asma->size == 0) + goto out_unlock; + + if (!asma->file) { + ret = -EBADF; + goto out_unlock; + } + + /* + * asma and asma->file are used outside the lock here. We assume + * once asma->file is set it will never be changed, and will not + * be destroyed until all references to the file are dropped and + * ashmem_release is called. + */ + mutex_unlock(&ashmem_mutex); + ret = vfs_iter_read(asma->file, iter, &iocb->ki_pos, 0); + mutex_lock(&ashmem_mutex); + if (ret > 0) + asma->file->f_pos = iocb->ki_pos; +out_unlock: + mutex_unlock(&ashmem_mutex); + return ret; +} + +static loff_t ashmem_llseek(struct file *file, loff_t offset, int origin) +{ + struct ashmem_area *asma = file->private_data; + loff_t ret; + + mutex_lock(&ashmem_mutex); + + if (asma->size == 0) { + mutex_unlock(&ashmem_mutex); + return -EINVAL; + } + + if (!asma->file) { + mutex_unlock(&ashmem_mutex); + return -EBADF; + } + + mutex_unlock(&ashmem_mutex); + + ret = vfs_llseek(asma->file, offset, origin); + if (ret < 0) + return ret; + + /** Copy f_pos from backing file, since f_ops->llseek() sets it */ + file->f_pos = asma->file->f_pos; + return ret; +} + +static inline vm_flags_t calc_vm_may_flags(unsigned long prot) +{ + return _calc_vm_trans(prot, PROT_READ, VM_MAYREAD) | + _calc_vm_trans(prot, PROT_WRITE, VM_MAYWRITE) | + _calc_vm_trans(prot, PROT_EXEC, VM_MAYEXEC); +} + +static int ashmem_vmfile_mmap(struct file *file, struct vm_area_struct *vma) +{ + /* do not allow to mmap ashmem backing shmem file directly */ + return -EPERM; +} + +static unsigned long +ashmem_vmfile_get_unmapped_area(struct file *file, unsigned long addr, + unsigned long len, unsigned long pgoff, + unsigned long flags) +{ + return current->mm->get_unmapped_area(file, addr, len, pgoff, flags); +} + +static int ashmem_mmap(struct file *file, struct vm_area_struct *vma) +{ + static struct file_operations vmfile_fops; + struct ashmem_area *asma = file->private_data; + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* user needs to SET_SIZE before mapping */ + if (!asma->size) { + ret = -EINVAL; + goto out; + } + + /* requested mapping size larger than object size */ + if (vma->vm_end - vma->vm_start > PAGE_ALIGN(asma->size)) { + ret = -EINVAL; + goto out; + } + + /* requested protection bits must match our allowed protection mask */ + if ((vma->vm_flags & ~calc_vm_prot_bits(asma->prot_mask, 0)) & + calc_vm_prot_bits(PROT_MASK, 0)) { + ret = -EPERM; + goto out; + } + vm_flags_clear(vma, calc_vm_may_flags(~asma->prot_mask)); + + if (!asma->file) { + char *name = ASHMEM_NAME_DEF; + struct file *vmfile; + struct inode *inode; + + if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0') + name = asma->name; + + /* ... and allocate the backing shmem file */ + vmfile = shmem_file_setup(name, asma->size, vma->vm_flags); + if (IS_ERR(vmfile)) { + ret = PTR_ERR(vmfile); + goto out; + } + vmfile->f_mode |= FMODE_LSEEK; + inode = file_inode(vmfile); + lockdep_set_class(&inode->i_rwsem, &backing_shmem_inode_class); + asma->file = vmfile; + /* + * override mmap operation of the vmfile so that it can't be + * remapped which would lead to creation of a new vma with no + * asma permission checks. Have to override get_unmapped_area + * as well to prevent VM_BUG_ON check for f_ops modification. + */ + if (!vmfile_fops.mmap) { + vmfile_fops = *vmfile->f_op; + vmfile_fops.mmap = ashmem_vmfile_mmap; + vmfile_fops.get_unmapped_area = + ashmem_vmfile_get_unmapped_area; + } + vmfile->f_op = &vmfile_fops; + } + get_file(asma->file); + + /* + * XXX - Reworked to use shmem_zero_setup() instead of + * shmem_set_file while we're in staging. -jstultz + */ + if (vma->vm_flags & VM_SHARED) { + ret = shmem_zero_setup(vma); + if (ret) { + fput(asma->file); + goto out; + } + } else { + vma_set_anonymous(vma); + } + + if (vma->vm_file) + fput(vma->vm_file); + vma->vm_file = asma->file; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +/* + * ashmem_shrink - our cache shrinker, called from mm/vmscan.c + * + * 'nr_to_scan' is the number of objects to scan for freeing. + * + * 'gfp_mask' is the mask of the allocation that got us into this mess. + * + * Return value is the number of objects freed or -1 if we cannot + * proceed without risk of deadlock (due to gfp_mask). + * + * We approximate LRU via least-recently-unpinned, jettisoning unpinned partial + * chunks of ashmem regions LRU-wise one-at-a-time until we hit 'nr_to_scan' + * pages freed. + */ +static unsigned long +ashmem_shrink_scan(struct shrinker *shrink, struct shrink_control *sc) +{ + unsigned long freed = 0; + + /* We might recurse into filesystem code, so bail out if necessary */ + if (!(sc->gfp_mask & __GFP_FS)) + return SHRINK_STOP; + + if (!mutex_trylock(&ashmem_mutex)) + return -1; + + while (!list_empty(&ashmem_lru_list)) { + struct ashmem_range *range = + list_first_entry(&ashmem_lru_list, typeof(*range), lru); + loff_t start = range->pgstart * PAGE_SIZE; + loff_t end = (range->pgend + 1) * PAGE_SIZE; + struct file *f = range->asma->file; + + get_file(f); + atomic_inc(&ashmem_shrink_inflight); + range->purged = ASHMEM_WAS_PURGED; +#ifdef CONFIG_PURGEABLE_ASHMEM + if (is_purgeable_ashmem(range->asma)) + range->asma->purged = true; +#endif + lru_del(range); + + freed += range_size(range); + mutex_unlock(&ashmem_mutex); + f->f_op->fallocate(f, + FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + start, end - start); + fput(f); + if (atomic_dec_and_test(&ashmem_shrink_inflight)) + wake_up_all(&ashmem_shrink_wait); + if (!mutex_trylock(&ashmem_mutex)) + goto out; + if (--sc->nr_to_scan <= 0) + break; + } + mutex_unlock(&ashmem_mutex); +out: + return freed; +} + +static unsigned long +ashmem_shrink_count(struct shrinker *shrink, struct shrink_control *sc) +{ + /* + * note that lru_count is count of pages on the lru, not a count of + * objects on the list. This means the scan function needs to return the + * number of pages freed, not the number of objects scanned. + */ + return lru_count; +} + +static struct shrinker ashmem_shrinker = { + .count_objects = ashmem_shrink_count, + .scan_objects = ashmem_shrink_scan, + /* + * XXX (dchinner): I wish people would comment on why they need on + * significant changes to the default value here + */ + .seeks = DEFAULT_SEEKS * 4, +}; + +static int set_prot_mask(struct ashmem_area *asma, unsigned long prot) +{ + int ret = 0; + + mutex_lock(&ashmem_mutex); + + /* the user can only remove, not add, protection bits */ + if ((asma->prot_mask & prot) != prot) { + ret = -EINVAL; + goto out; + } + + /* does the application expect PROT_READ to imply PROT_EXEC? */ + if ((prot & PROT_READ) && (current->personality & READ_IMPLIES_EXEC)) + prot |= PROT_EXEC; + + asma->prot_mask = prot; + +out: + mutex_unlock(&ashmem_mutex); + return ret; +} + +static int set_name(struct ashmem_area *asma, void __user *name) +{ + int len; + int ret = 0; + char local_name[ASHMEM_NAME_LEN]; + + /* + * Holding the ashmem_mutex while doing a copy_from_user might cause + * an data abort which would try to access mmap_lock. If another + * thread has invoked ashmem_mmap then it will be holding the + * semaphore and will be waiting for ashmem_mutex, there by leading to + * deadlock. We'll release the mutex and take the name to a local + * variable that does not need protection and later copy the local + * variable to the structure member with lock held. + */ + len = strncpy_from_user(local_name, name, ASHMEM_NAME_LEN); + if (len < 0) + return len; + + mutex_lock(&ashmem_mutex); + /* cannot change an existing mapping's name */ + if (asma->file) + ret = -EINVAL; + else + strscpy(asma->name + ASHMEM_NAME_PREFIX_LEN, local_name, + ASHMEM_NAME_LEN); + + mutex_unlock(&ashmem_mutex); + return ret; +} + +static int get_name(struct ashmem_area *asma, void __user *name) +{ + int ret = 0; + size_t len; + /* + * Have a local variable to which we'll copy the content + * from asma with the lock held. Later we can copy this to the user + * space safely without holding any locks. So even if we proceed to + * wait for mmap_lock, it won't lead to deadlock. + */ + char local_name[ASHMEM_NAME_LEN]; + + mutex_lock(&ashmem_mutex); + if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0') { + /* + * Copying only `len', instead of ASHMEM_NAME_LEN, bytes + * prevents us from revealing one user's stack to another. + */ + len = strlen(asma->name + ASHMEM_NAME_PREFIX_LEN) + 1; + memcpy(local_name, asma->name + ASHMEM_NAME_PREFIX_LEN, len); + } else { + len = sizeof(ASHMEM_NAME_DEF); + memcpy(local_name, ASHMEM_NAME_DEF, len); + } + mutex_unlock(&ashmem_mutex); + + /* + * Now we are just copying from the stack variable to userland + * No lock held + */ + if (copy_to_user(name, local_name, len)) + ret = -EFAULT; + return ret; +} + +/* + * ashmem_pin - pin the given ashmem region, returning whether it was + * previously purged (ASHMEM_WAS_PURGED) or not (ASHMEM_NOT_PURGED). + * + * Caller must hold ashmem_mutex. + */ +static int ashmem_pin(struct ashmem_area *asma, size_t pgstart, size_t pgend, + struct ashmem_range **new_range) +{ + struct ashmem_range *range, *next; + int ret = ASHMEM_NOT_PURGED; +#ifdef CONFIG_PURGEABLE_ASHMEM + if (is_purgeable_ashmem(asma)) { + asma->ref_count++; + if (asma->ref_count > 1) + return PM_SUCCESS; + } +#endif + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) { + /* moved past last applicable page; we can short circuit */ + if (range_before_page(range, pgstart)) + break; + + /* + * The user can ask us to pin pages that span multiple ranges, + * or to pin pages that aren't even unpinned, so this is messy. + * + * Four cases: + * 1. The requested range subsumes an existing range, so we + * just remove the entire matching range. + * 2. The requested range overlaps the start of an existing + * range, so we just update that range. + * 3. The requested range overlaps the end of an existing + * range, so we just update that range. + * 4. The requested range punches a hole in an existing range, + * so we have to update one side of the range and then + * create a new range for the other side. + */ + if (page_range_in_range(range, pgstart, pgend)) { + ret |= range->purged; + + /* Case #1: Easy. Just nuke the whole thing. */ + if (page_range_subsumes_range(range, pgstart, pgend)) { + range_del(range); + continue; + } + + /* Case #2: We overlap from the start, so adjust it */ + if (range->pgstart >= pgstart) { + range_shrink(range, pgend + 1, range->pgend); + continue; + } + + /* Case #3: We overlap from the rear, so adjust it */ + if (range->pgend <= pgend) { + range_shrink(range, range->pgstart, + pgstart - 1); + continue; + } + + /* + * Case #4: We eat a chunk out of the middle. A bit + * more complicated, we allocate a new range for the + * second half and adjust the first chunk's endpoint. + */ + range_alloc(asma, range, range->purged, + pgend + 1, range->pgend, new_range); + range_shrink(range, range->pgstart, pgstart - 1); + break; + } + } + + return ret; +} + +/* + * ashmem_unpin - unpin the given range of pages. Returns zero on success. + * + * Caller must hold ashmem_mutex. + */ +static int ashmem_unpin(struct ashmem_area *asma, size_t pgstart, size_t pgend, + struct ashmem_range **new_range) +{ + struct ashmem_range *range, *next; + unsigned int purged = ASHMEM_NOT_PURGED; +#ifdef CONFIG_PURGEABLE_ASHMEM + if (is_purgeable_ashmem(asma)) { + if (asma->ref_count > PURGEABLE_ASHMEM_UNPIN_REFCOUNT && + !(--asma->ref_count == PURGEABLE_ASHMEM_UNPIN_REFCOUNT)) + return PM_SUCCESS; + if (asma->ref_count < PURGEABLE_ASHMEM_UNPIN_REFCOUNT) { + asma->ref_count = PURGEABLE_ASHMEM_UNPIN_REFCOUNT; + return PM_FAIL; + } + } +#endif +restart: + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) { + /* short circuit: this is our insertion point */ + if (range_before_page(range, pgstart)) + break; + + /* + * The user can ask us to unpin pages that are already entirely + * or partially pinned. We handle those two cases here. + */ + if (page_range_subsumed_by_range(range, pgstart, pgend)) + return 0; + if (page_range_in_range(range, pgstart, pgend)) { + pgstart = min(range->pgstart, pgstart); + pgend = max(range->pgend, pgend); + purged |= range->purged; + range_del(range); + goto restart; + } + } + + range_alloc(asma, range, purged, pgstart, pgend, new_range); + return 0; +} + +/* + * ashmem_get_pin_status - Returns ASHMEM_IS_UNPINNED if _any_ pages in the + * given interval are unpinned and ASHMEM_IS_PINNED otherwise. + * + * Caller must hold ashmem_mutex. + */ +static int ashmem_get_pin_status(struct ashmem_area *asma, size_t pgstart, + size_t pgend) +{ + struct ashmem_range *range; + int ret = ASHMEM_IS_PINNED; +#ifdef CONFIG_PURGEABLE_ASHMEM + if (is_purgeable_ashmem(asma)) + return asma->ref_count; +#endif + list_for_each_entry(range, &asma->unpinned_list, unpinned) { + if (range_before_page(range, pgstart)) + break; + if (page_range_in_range(range, pgstart, pgend)) { + ret = ASHMEM_IS_UNPINNED; + break; + } + } + + return ret; +} + +static int ashmem_pin_unpin(struct ashmem_area *asma, unsigned long cmd, + void __user *p) +{ + struct ashmem_pin pin; + size_t pgstart, pgend; + int ret = -EINVAL; + struct ashmem_range *range = NULL; + + if (copy_from_user(&pin, p, sizeof(pin))) + return -EFAULT; + + if (cmd == ASHMEM_PIN || cmd == ASHMEM_UNPIN) { + range = kmem_cache_zalloc(ashmem_range_cachep, GFP_KERNEL); + if (!range) + return -ENOMEM; + } + + mutex_lock(&ashmem_mutex); + wait_event(ashmem_shrink_wait, !atomic_read(&ashmem_shrink_inflight)); +#ifdef CONFIG_PURGEABLE_ASHMEM + if (is_purgeable_ashmem(asma)) { + if (pin.offset != PURGEABLE_ASHMEM_PIN_OFFSET || + pin.len != PURGEABLE_ASHMEM_PIN_LEN) { + ret = -EINVAL; + goto out_unlock; + } + } +#endif + if (!asma->file) + goto out_unlock; + + /* per custom, you can pass zero for len to mean "everything onward" */ + if (!pin.len) + pin.len = PAGE_ALIGN(asma->size) - pin.offset; + + if ((pin.offset | pin.len) & ~PAGE_MASK) + goto out_unlock; + + if (((__u32)-1) - pin.offset < pin.len) + goto out_unlock; + + if (PAGE_ALIGN(asma->size) < pin.offset + pin.len) + goto out_unlock; + + pgstart = pin.offset / PAGE_SIZE; + pgend = pgstart + (pin.len / PAGE_SIZE) - 1; + + switch (cmd) { + case ASHMEM_PIN: + ret = ashmem_pin(asma, pgstart, pgend, &range); + break; + case ASHMEM_UNPIN: + ret = ashmem_unpin(asma, pgstart, pgend, &range); + break; + case ASHMEM_GET_PIN_STATUS: + ret = ashmem_get_pin_status(asma, pgstart, pgend); + break; + } + +out_unlock: + mutex_unlock(&ashmem_mutex); + if (range) + kmem_cache_free(ashmem_range_cachep, range); + + return ret; +} + +#ifdef CONFIG_PURGEABLE_ASHMEM +void ashmem_shrinkall(void) +{ + struct shrink_control sc = { + .gfp_mask = GFP_KERNEL, + .nr_to_scan = LONG_MAX, + }; + + ashmem_shrink_scan(&ashmem_shrinker, &sc); +} + +void ashmem_shrink_by_id(const unsigned int ashmem_id, const unsigned int create_time) +{ + struct ashmem_range *range, *next; + bool found = false; + + if (!mutex_trylock(&ashmem_mutex)) + return; + + list_for_each_entry_safe(range, next, &ashmem_lru_list, lru) { + if (!is_purgeable_ashmem(range->asma)) + continue; + if (range->asma->id != ashmem_id || + range->asma->create_time != create_time) + continue; + found = true; + range->asma->purged = true; + break; + } + if (!found) + goto out_unlock; + + loff_t start = range->pgstart * PAGE_SIZE; + loff_t end = (range->pgend + 1) * PAGE_SIZE; + struct file *f = range->asma->file; + + if (!f) + goto out_unlock; + get_file(f); + atomic_inc(&ashmem_shrink_inflight); + range->purged = ASHMEM_WAS_PURGED; + + lru_del(range); + mutex_unlock(&ashmem_mutex); + f->f_op->fallocate(f, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, + start, end - start); + fput(f); + if (atomic_dec_and_test(&ashmem_shrink_inflight)) + wake_up_all(&ashmem_shrink_wait); + return; + +out_unlock: + mutex_unlock(&ashmem_mutex); +} + +static bool is_ashmem_unpin(struct ashmem_area *asma) +{ + struct ashmem_range *range, *next; + int count = 0; + + mutex_lock(&ashmem_mutex); + if (!asma) { + mutex_unlock(&ashmem_mutex); + return false; + } + list_for_each_entry_safe(range, next, &asma->unpinned_list, unpinned) + count++; + mutex_unlock(&ashmem_mutex); + return count > 0 ? true : false; +} + +static long purgeable_ashmem_cmd(struct ashmem_area *asma, unsigned int cmd) +{ + int ret = -EINVAL; + + if (!is_purgeable_ashmem(asma)) + return ret; + mutex_lock(&ashmem_mutex); + switch (cmd) { + case ASHMEM_GET_PURGEABLE: + ret = asma->is_purgeable; + break; + case PURGEABLE_ASHMEM_IS_PURGED: + ret = asma->purged; + break; + case PURGEABLE_ASHMEM_REBUILD_SUCCESS: + asma->purged = false; + ret = PM_SUCCESS; + break; + } + mutex_unlock(&ashmem_mutex); + return ret; +} + +bool get_purgeable_ashmem_metadata(struct file *f, struct purgeable_ashmem_metadata *pmdata) +{ + struct ashmem_area *asma = f->private_data; + + if (!asma) { + return false; + } + pmdata->name = asma->name; + pmdata->size = asma->size; + pmdata->refc = asma->ref_count; + pmdata->purged = asma->purged; + pmdata->is_purgeable = asma->is_purgeable; + pmdata->id = asma->id; + pmdata->create_time = asma->create_time; + return true; +} +#endif +static long ashmem_ioctl(struct file *file, unsigned int cmd, unsigned long arg) +{ + struct ashmem_area *asma = file->private_data; + long ret = -ENOTTY; + + switch (cmd) { + case ASHMEM_SET_NAME: + ret = set_name(asma, (void __user *)arg); + break; + case ASHMEM_GET_NAME: + ret = get_name(asma, (void __user *)arg); + break; + case ASHMEM_SET_SIZE: + ret = -EINVAL; + mutex_lock(&ashmem_mutex); + if (!asma->file) { + ret = 0; + asma->size = (size_t)arg; + } + mutex_unlock(&ashmem_mutex); + break; + case ASHMEM_GET_SIZE: + ret = asma->size; + break; + case ASHMEM_SET_PROT_MASK: + ret = set_prot_mask(asma, arg); + break; + case ASHMEM_GET_PROT_MASK: + ret = asma->prot_mask; + break; + case ASHMEM_PIN: + case ASHMEM_UNPIN: + case ASHMEM_GET_PIN_STATUS: + ret = ashmem_pin_unpin(asma, cmd, (void __user *)arg); + break; + case ASHMEM_PURGE_ALL_CACHES: + ret = -EPERM; + if (capable(CAP_SYS_ADMIN)) { + struct shrink_control sc = { + .gfp_mask = GFP_KERNEL, + .nr_to_scan = LONG_MAX, + }; + ret = ashmem_shrink_count(&ashmem_shrinker, &sc); + ashmem_shrink_scan(&ashmem_shrinker, &sc); + } + break; +#ifdef CONFIG_PURGEABLE_ASHMEM + case ASHMEM_SET_PURGEABLE: + if (is_ashmem_unpin(asma)) { + ret = PM_FAIL; + break; + } + mutex_lock(&ashmem_mutex); + if (asma) { + asma->is_purgeable = true; + ret = PM_SUCCESS; + } + mutex_unlock(&ashmem_mutex); + break; + case ASHMEM_GET_PURGEABLE: + fallthrough; + case PURGEABLE_ASHMEM_IS_PURGED: + fallthrough; + case PURGEABLE_ASHMEM_REBUILD_SUCCESS: + ret = purgeable_ashmem_cmd(asma, cmd); + break; +#endif + } + return ret; +} + +/* support of 32bit userspace on 64bit platforms */ +#ifdef CONFIG_COMPAT +static long compat_ashmem_ioctl(struct file *file, unsigned int cmd, + unsigned long arg) +{ + switch (cmd) { + case COMPAT_ASHMEM_SET_SIZE: + cmd = ASHMEM_SET_SIZE; + break; + case COMPAT_ASHMEM_SET_PROT_MASK: + cmd = ASHMEM_SET_PROT_MASK; + break; + } + return ashmem_ioctl(file, cmd, arg); +} +#endif +#ifdef CONFIG_PROC_FS +static void ashmem_show_fdinfo(struct seq_file *m, struct file *file) +{ + struct ashmem_area *asma = file->private_data; + + mutex_lock(&ashmem_mutex); + + if (asma->file) + seq_printf(m, "inode:\t%ld\n", file_inode(asma->file)->i_ino); + + if (asma->name[ASHMEM_NAME_PREFIX_LEN] != '\0') + seq_printf(m, "name:\t%s\n", + asma->name + ASHMEM_NAME_PREFIX_LEN); + + mutex_unlock(&ashmem_mutex); +} +#endif +static const struct file_operations ashmem_fops = { + .owner = THIS_MODULE, + .open = ashmem_open, + .release = ashmem_release, + .read_iter = ashmem_read_iter, + .llseek = ashmem_llseek, + .mmap = ashmem_mmap, + .unlocked_ioctl = ashmem_ioctl, +#ifdef CONFIG_COMPAT + .compat_ioctl = compat_ashmem_ioctl, +#endif +#ifdef CONFIG_PROC_FS + .show_fdinfo = ashmem_show_fdinfo, +#endif +}; + +int is_ashmem_file(struct file *file) +{ + return file->f_op == &ashmem_fops; +} + +static struct miscdevice ashmem_misc = { + .minor = MISC_DYNAMIC_MINOR, + .name = "ashmem", + .fops = &ashmem_fops, +}; + +size_t get_ashmem_size_by_file(struct file *f) +{ + struct ashmem_area *asma = f->private_data; + + if (asma) + return asma->size; + return 0; +} + +char *get_ashmem_name_by_file(struct file *f) +{ + struct ashmem_area *asma = f->private_data; + + if (asma) + return asma->name; + return NULL; +} + +static int __init ashmem_init(void) +{ + int ret = -ENOMEM; + + ashmem_area_cachep = kmem_cache_create("ashmem_area_cache", + sizeof(struct ashmem_area), + 0, 0, NULL); + if (!ashmem_area_cachep) { + pr_err("failed to create slab cache\n"); + goto out; + } + + ashmem_range_cachep = kmem_cache_create("ashmem_range_cache", + sizeof(struct ashmem_range), + 0, 0, NULL); + if (!ashmem_range_cachep) { + pr_err("failed to create slab cache\n"); + goto out_free1; + } + + ret = misc_register(&ashmem_misc); + if (ret) { + pr_err("failed to register misc device!\n"); + goto out_free2; + } + + ret = register_shrinker(&ashmem_shrinker, "ashmem-shrinker"); + if (ret) { + pr_err("failed to register shrinker!\n"); + goto out_demisc; + } + init_ashmem_process_info(); +#ifdef CONFIG_PURGEABLE_ASHMEM + init_purgeable_ashmem_trigger(); +#endif + pr_info("initialized\n"); + + return 0; + +out_demisc: + misc_deregister(&ashmem_misc); +out_free2: + kmem_cache_destroy(ashmem_range_cachep); +out_free1: + kmem_cache_destroy(ashmem_area_cachep); +out: + return ret; +} +device_initcall(ashmem_init); diff --git a/drivers/staging/android/ashmem.h b/drivers/staging/android/ashmem.h new file mode 100644 index 000000000000..2f04163c67ed --- /dev/null +++ b/drivers/staging/android/ashmem.h @@ -0,0 +1,47 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 */ +/* + * include/linux/ashmem.h + * + * Copyright 2008 Google Inc. + * Author: Robert Love + */ + +#ifndef _LINUX_ASHMEM_H +#define _LINUX_ASHMEM_H + +#include +#include +#include + +#include "uapi/ashmem.h" + +/* support of 32bit userspace on 64bit platforms */ +#ifdef CONFIG_COMPAT +#define COMPAT_ASHMEM_SET_SIZE _IOW(__ASHMEMIOC, 3, compat_size_t) +#define COMPAT_ASHMEM_SET_PROT_MASK _IOW(__ASHMEMIOC, 5, unsigned int) +#endif + +int is_ashmem_file(struct file *file); +size_t get_ashmem_size_by_file(struct file *f); +char *get_ashmem_name_by_file(struct file *f); +void ashmem_mutex_lock(void); +void ashmem_mutex_unlock(void); + +#ifdef CONFIG_PURGEABLE_ASHMEM +struct purgeable_ashmem_metadata { + char *name; + size_t size; + int refc; + bool purged; + bool is_purgeable; + unsigned int id; + unsigned int create_time; +}; + +void ashmem_shrinkall(void); +void ashmem_shrink_by_id(const unsigned int ashmem_id, + const unsigned int create_time); +bool get_purgeable_ashmem_metadata(struct file *f, + struct purgeable_ashmem_metadata *pmdata); +#endif +#endif /* _LINUX_ASHMEM_H */ diff --git a/drivers/staging/android/ion/Kconfig b/drivers/staging/android/ion/Kconfig new file mode 100644 index 000000000000..7b7da979991e --- /dev/null +++ b/drivers/staging/android/ion/Kconfig @@ -0,0 +1,14 @@ +# SPDX-License-Identifier: GPL-2.0 +menuconfig ION + bool "Ion Memory Manager" + depends on HAS_DMA && MMU + select GENERIC_ALLOCATOR + select DMA_SHARED_BUFFER + help + Choose this option to enable the ION Memory Manager, + used by Android to efficiently allocate buffers + from userspace that can be shared between drivers. + If you're not using Android its probably safe to + say N here. + +source "drivers/staging/android/ion/heaps/Kconfig" diff --git a/drivers/staging/android/ion/Makefile b/drivers/staging/android/ion/Makefile new file mode 100644 index 000000000000..7f8fd0f537b4 --- /dev/null +++ b/drivers/staging/android/ion/Makefile @@ -0,0 +1,4 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ION) += ion.o ion_buffer.o ion_dma_buf.o ion_heap.o +CFLAGS_ion_buffer.o = -I$(src) +obj-y += heaps/ diff --git a/drivers/staging/android/ion/heaps/Kconfig b/drivers/staging/android/ion/heaps/Kconfig new file mode 100644 index 000000000000..5034c45a397d --- /dev/null +++ b/drivers/staging/android/ion/heaps/Kconfig @@ -0,0 +1,15 @@ +# SPDX-License-Identifier: GPL-2.0 +config ION_SYSTEM_HEAP + tristate "Ion system heap" + depends on ION + help + Choose this option to enable the Ion system heap. The system heap + is backed by pages from the buddy allocator. If in doubt, say Y. + +config ION_CMA_HEAP + tristate "Ion CMA heap support" + depends on ION && DMA_CMA + help + Choose this option to enable CMA heaps with Ion. This heap is backed + by the Contiguous Memory Allocator (CMA). If your system has these + regions, you should say Y here. diff --git a/drivers/staging/android/ion/heaps/Makefile b/drivers/staging/android/ion/heaps/Makefile new file mode 100644 index 000000000000..82e36e89e978 --- /dev/null +++ b/drivers/staging/android/ion/heaps/Makefile @@ -0,0 +1,5 @@ +# SPDX-License-Identifier: GPL-2.0 +obj-$(CONFIG_ION_SYSTEM_HEAP) += ion_sys_heap.o +ion_sys_heap-y := ion_system_heap.o ion_page_pool.o + +obj-$(CONFIG_ION_CMA_HEAP) += ion_cma_heap.o diff --git a/drivers/staging/android/ion/heaps/ion_cma_heap.c b/drivers/staging/android/ion/heaps/ion_cma_heap.c new file mode 100644 index 000000000000..6ba7fd84c9ee --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_cma_heap.c @@ -0,0 +1,151 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator CMA heap exporter + * + * Copyright (C) Linaro 2012 + * Author: for ST-Ericsson. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +struct ion_cma_heap { + struct ion_heap heap; + struct cma *cma; +} cma_heaps[MAX_CMA_AREAS]; + +#define to_cma_heap(x) container_of(x, struct ion_cma_heap, heap) + +/* ION CMA heap operations functions */ +static int ion_cma_allocate(struct ion_heap *heap, struct ion_buffer *buffer, + unsigned long len, + unsigned long flags) +{ + struct ion_cma_heap *cma_heap = to_cma_heap(heap); + struct sg_table *table; + struct page *pages; + unsigned long size = PAGE_ALIGN(len); + unsigned long nr_pages = size >> PAGE_SHIFT; + unsigned long align = get_order(size); + int ret; + + if (align > CONFIG_CMA_ALIGNMENT) + align = CONFIG_CMA_ALIGNMENT; + + pages = cma_alloc(cma_heap->cma, nr_pages, align, false); + if (!pages) + return -ENOMEM; + + if (PageHighMem(pages)) { + unsigned long nr_clear_pages = nr_pages; + struct page *page = pages; + + while (nr_clear_pages > 0) { + void *vaddr = kmap_atomic(page); + + memset(vaddr, 0, PAGE_SIZE); + kunmap_atomic(vaddr); + page++; + nr_clear_pages--; + } + } else { + memset(page_address(pages), 0, size); + } + + table = kmalloc(sizeof(*table), GFP_KERNEL); + if (!table) + goto err; + + ret = sg_alloc_table(table, 1, GFP_KERNEL); + if (ret) + goto free_mem; + + sg_set_page(table->sgl, pages, size, 0); + + buffer->priv_virt = pages; + buffer->sg_table = table; + + ion_buffer_prep_noncached(buffer); + + return 0; + +free_mem: + kfree(table); +err: + cma_release(cma_heap->cma, pages, nr_pages); + return -ENOMEM; +} + +static void ion_cma_free(struct ion_buffer *buffer) +{ + struct ion_cma_heap *cma_heap = to_cma_heap(buffer->heap); + struct page *pages = buffer->priv_virt; + unsigned long nr_pages = PAGE_ALIGN(buffer->size) >> PAGE_SHIFT; + + /* release memory */ + cma_release(cma_heap->cma, pages, nr_pages); + /* release sg table */ + sg_free_table(buffer->sg_table); + kfree(buffer->sg_table); +} + +static struct ion_heap_ops ion_cma_ops = { + .allocate = ion_cma_allocate, + .free = ion_cma_free, +}; + +static int __ion_add_cma_heap(struct cma *cma, void *data) +{ + int *cma_nr = data; + struct ion_cma_heap *cma_heap; + int ret; + + if (*cma_nr >= MAX_CMA_AREAS) + return -EINVAL; + + cma_heap = &cma_heaps[*cma_nr]; + cma_heap->heap.ops = &ion_cma_ops; + cma_heap->heap.type = ION_HEAP_TYPE_DMA; + cma_heap->heap.name = cma_get_name(cma); + + ret = ion_device_add_heap(&cma_heap->heap); + if (ret) + goto out; + + cma_heap->cma = cma; + *cma_nr += 1; +out: + return 0; +} + +static int __init ion_cma_heap_init(void) +{ + int ret; + int nr = 0; + + ret = cma_for_each_area(__ion_add_cma_heap, &nr); + if (ret) { + for (nr = 0; nr < MAX_CMA_AREAS && cma_heaps[nr].cma; nr++) + ion_device_remove_heap(&cma_heaps[nr].heap); + } + + return ret; +} + +static void __exit ion_cma_heap_exit(void) +{ + int nr; + + for (nr = 0; nr < MAX_CMA_AREAS && cma_heaps[nr].cma; nr++) + ion_device_remove_heap(&cma_heaps[nr].heap); +} + +module_init(ion_cma_heap_init); +module_exit(ion_cma_heap_exit); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.c b/drivers/staging/android/ion/heaps/ion_page_pool.c new file mode 100644 index 000000000000..efd6ce4cc0ad --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_page_pool.c @@ -0,0 +1,172 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator page pool helpers + * + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include + +#include "ion_page_pool.h" + +static inline struct page *ion_page_pool_alloc_pages(struct ion_page_pool *pool) +{ + if (fatal_signal_pending(current)) + return NULL; + return alloc_pages(pool->gfp_mask, pool->order); +} + +static void ion_page_pool_free_pages(struct ion_page_pool *pool, + struct page *page) +{ + __free_pages(page, pool->order); +} + +static void ion_page_pool_add(struct ion_page_pool *pool, struct page *page) +{ + mutex_lock(&pool->mutex); + if (PageHighMem(page)) { + list_add_tail(&page->lru, &pool->high_items); + pool->high_count++; + } else { + list_add_tail(&page->lru, &pool->low_items); + pool->low_count++; + } + + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + 1 << pool->order); + mutex_unlock(&pool->mutex); +} + +static struct page *ion_page_pool_remove(struct ion_page_pool *pool, bool high) +{ + struct page *page; + + if (high) { + BUG_ON(!pool->high_count); + page = list_first_entry(&pool->high_items, struct page, lru); + pool->high_count--; + } else { + BUG_ON(!pool->low_count); + page = list_first_entry(&pool->low_items, struct page, lru); + pool->low_count--; + } + + list_del(&page->lru); + mod_node_page_state(page_pgdat(page), NR_KERNEL_MISC_RECLAIMABLE, + -(1 << pool->order)); + return page; +} + +struct page *ion_page_pool_alloc(struct ion_page_pool *pool) +{ + struct page *page = NULL; + + BUG_ON(!pool); + + mutex_lock(&pool->mutex); + if (pool->high_count) + page = ion_page_pool_remove(pool, true); + else if (pool->low_count) + page = ion_page_pool_remove(pool, false); + mutex_unlock(&pool->mutex); + + if (!page) + page = ion_page_pool_alloc_pages(pool); + + return page; +} +EXPORT_SYMBOL_GPL(ion_page_pool_alloc); + +void ion_page_pool_free(struct ion_page_pool *pool, struct page *page) +{ + BUG_ON(pool->order != compound_order(page)); + + ion_page_pool_add(pool, page); +} +EXPORT_SYMBOL_GPL(ion_page_pool_free); + +static int ion_page_pool_total(struct ion_page_pool *pool, bool high) +{ + int count = pool->low_count; + + if (high) + count += pool->high_count; + + return count << pool->order; +} + +int ion_page_pool_nr_pages(struct ion_page_pool *pool) +{ + int nr_total_pages; + + mutex_lock(&pool->mutex); + nr_total_pages = ion_page_pool_total(pool, true); + mutex_unlock(&pool->mutex); + + return nr_total_pages; +} +EXPORT_SYMBOL_GPL(ion_page_pool_nr_pages); + +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan) +{ + int freed = 0; + bool high; + + if (current_is_kswapd()) + high = true; + else + high = !!(gfp_mask & __GFP_HIGHMEM); + + if (nr_to_scan == 0) + return ion_page_pool_total(pool, high); + + while (freed < nr_to_scan) { + struct page *page; + + mutex_lock(&pool->mutex); + if (pool->low_count) { + page = ion_page_pool_remove(pool, false); + } else if (high && pool->high_count) { + page = ion_page_pool_remove(pool, true); + } else { + mutex_unlock(&pool->mutex); + break; + } + mutex_unlock(&pool->mutex); + ion_page_pool_free_pages(pool, page); + freed += (1 << pool->order); + } + + return freed; +} +EXPORT_SYMBOL_GPL(ion_page_pool_shrink); + +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order) +{ + struct ion_page_pool *pool = kmalloc(sizeof(*pool), GFP_KERNEL); + + if (!pool) + return NULL; + pool->high_count = 0; + pool->low_count = 0; + INIT_LIST_HEAD(&pool->low_items); + INIT_LIST_HEAD(&pool->high_items); + pool->gfp_mask = gfp_mask | __GFP_COMP; + pool->order = order; + mutex_init(&pool->mutex); + plist_node_init(&pool->list, order); + + return pool; +} +EXPORT_SYMBOL_GPL(ion_page_pool_create); + +void ion_page_pool_destroy(struct ion_page_pool *pool) +{ + kfree(pool); +} +EXPORT_SYMBOL_GPL(ion_page_pool_destroy); diff --git a/drivers/staging/android/ion/heaps/ion_page_pool.h b/drivers/staging/android/ion/heaps/ion_page_pool.h new file mode 100644 index 000000000000..10c79090c7a0 --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_page_pool.h @@ -0,0 +1,67 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ION Page Pool kernel interface header + * + * Copyright (C) 2011 Google, Inc. + */ + +#ifndef _ION_PAGE_POOL_H +#define _ION_PAGE_POOL_H + +#include +#include +#include +#include + +/** + * functions for creating and destroying a heap pool -- allows you + * to keep a pool of pre allocated memory to use from your heap. Keeping + * a pool of memory that is ready for dma, ie any cached mapping have been + * invalidated from the cache, provides a significant performance benefit on + * many systems + */ + +/** + * struct ion_page_pool - pagepool struct + * @high_count: number of highmem items in the pool + * @low_count: number of lowmem items in the pool + * @high_items: list of highmem items + * @low_items: list of lowmem items + * @mutex: lock protecting this struct and especially the count + * item list + * @gfp_mask: gfp_mask to use from alloc + * @order: order of pages in the pool + * @list: plist node for list of pools + * + * Allows you to keep a pool of pre allocated pages to use from your heap. + * Keeping a pool of pages that is ready for dma, ie any cached mapping have + * been invalidated from the cache, provides a significant performance benefit + * on many systems + */ +struct ion_page_pool { + int high_count; + int low_count; + struct list_head high_items; + struct list_head low_items; + struct mutex mutex; + gfp_t gfp_mask; + unsigned int order; + struct plist_node list; +}; + +struct ion_page_pool *ion_page_pool_create(gfp_t gfp_mask, unsigned int order); +void ion_page_pool_destroy(struct ion_page_pool *pool); +struct page *ion_page_pool_alloc(struct ion_page_pool *pool); +void ion_page_pool_free(struct ion_page_pool *pool, struct page *page); +int ion_page_pool_nr_pages(struct ion_page_pool *pool); + +/** ion_page_pool_shrink - shrinks the size of the memory cached in the pool + * @pool: the pool + * @gfp_mask: the memory type to reclaim + * @nr_to_scan: number of items to shrink in pages + * + * returns the number of items freed in pages + */ +int ion_page_pool_shrink(struct ion_page_pool *pool, gfp_t gfp_mask, + int nr_to_scan); +#endif /* _ION_PAGE_POOL_H */ diff --git a/drivers/staging/android/ion/heaps/ion_system_heap.c b/drivers/staging/android/ion/heaps/ion_system_heap.c new file mode 100644 index 000000000000..45d23bea3822 --- /dev/null +++ b/drivers/staging/android/ion/heaps/ion_system_heap.c @@ -0,0 +1,344 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator system heap exporter + * + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ion_page_pool.h" + +#define NUM_ORDERS ARRAY_SIZE(orders) + +static gfp_t high_order_gfp_flags = (GFP_HIGHUSER | __GFP_ZERO | __GFP_NOWARN | + __GFP_NORETRY) & ~__GFP_RECLAIM; +static gfp_t low_order_gfp_flags = GFP_HIGHUSER | __GFP_ZERO; +static const unsigned int orders[] = {8, 4, 0}; + +static int order_to_index(unsigned int order) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (order == orders[i]) + return i; + BUG(); + return -1; +} + +static inline unsigned int order_to_size(int order) +{ + return PAGE_SIZE << order; +} + +struct ion_system_heap { + struct ion_heap heap; + struct ion_page_pool *pools[NUM_ORDERS]; +}; + +static struct page *alloc_buffer_page(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long order) +{ + struct ion_page_pool *pool = heap->pools[order_to_index(order)]; + + return ion_page_pool_alloc(pool); +} + +static void free_buffer_page(struct ion_system_heap *heap, + struct ion_buffer *buffer, struct page *page) +{ + struct ion_page_pool *pool; + unsigned int order = compound_order(page); + + /* go to system */ + if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) { + __free_pages(page, order); + return; + } + + pool = heap->pools[order_to_index(order)]; + + ion_page_pool_free(pool, page); +} + +static struct page *alloc_largest_available(struct ion_system_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) +{ + struct page *page; + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < order_to_size(orders[i])) + continue; + if (max_order < orders[i]) + continue; + + page = alloc_buffer_page(heap, buffer, orders[i]); + if (!page) + continue; + + return page; + } + + return NULL; +} + +static int ion_system_heap_allocate(struct ion_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned long flags) +{ + struct ion_system_heap *sys_heap = container_of(heap, + struct ion_system_heap, + heap); + struct sg_table *table; + struct scatterlist *sg; + struct list_head pages; + struct page *page, *tmp_page; + int i = 0; + unsigned long size_remaining = PAGE_ALIGN(size); + unsigned int max_order = orders[0]; + struct list_head lists[8]; + unsigned int block_index[8] = {0}; + unsigned int block_1M = 0; + unsigned int block_64K = 0; + unsigned int maximum; + int j; + + if (size / PAGE_SIZE > totalram_pages() / 2) + return -ENOMEM; + + INIT_LIST_HEAD(&pages); + for (i = 0; i < 8; i++) + INIT_LIST_HEAD(&lists[i]); + + i = 0; + while (size_remaining > 0) { + page = alloc_largest_available(sys_heap, buffer, size_remaining, + max_order); + if (!page) + goto free_pages; + + size_remaining -= page_size(page); + max_order = compound_order(page); + if (max_order) { + if (max_order == 8) + block_1M++; + if (max_order == 4) + block_64K++; + list_add_tail(&page->lru, &pages); + } else { + dma_addr_t phys = page_to_phys(page); + unsigned int bit12_14 = (phys >> 12) & 0x7; + + list_add_tail(&page->lru, &lists[bit12_14]); + block_index[bit12_14]++; + } + + i++; + } + + pr_debug("%s, %d, i = %d, size = %ld\n", __func__, __LINE__, i, size); + + table = kmalloc(sizeof(*table), GFP_KERNEL); + if (!table) + goto free_pages; + + if (sg_alloc_table(table, i, GFP_KERNEL)) + goto free_table; + + maximum = block_index[0]; + for (i = 1; i < 8; i++) + maximum = max(maximum, block_index[i]); + + pr_debug("%s, %d, maximum = %d, block_1M = %d, block_64K = %d\n", + __func__, __LINE__, maximum, block_1M, block_64K); + + for (i = 0; i < 8; i++) + pr_debug("block_index[%d] = %d\n", i, block_index[i]); + + sg = table->sgl; + list_for_each_entry_safe(page, tmp_page, &pages, lru) { + sg_set_page(sg, page, page_size(page), 0); + sg = sg_next(sg); + list_del(&page->lru); + } + + for (i = 0; i < maximum; i++) { + for (j = 0; j < 8; j++) { + if (!list_empty(&lists[j])) { + page = list_first_entry(&lists[j], struct page, + lru); + sg_set_page(sg, page, PAGE_SIZE, 0); + sg = sg_next(sg); + list_del(&page->lru); + } + } + } + + buffer->sg_table = table; + + ion_buffer_prep_noncached(buffer); + + return 0; + +free_table: + kfree(table); +free_pages: + list_for_each_entry_safe(page, tmp_page, &pages, lru) + free_buffer_page(sys_heap, buffer, page); + + for (i = 0; i < 8; i++) { + list_for_each_entry_safe(page, tmp_page, &lists[i], lru) + free_buffer_page(sys_heap, buffer, page); + } + return -ENOMEM; +} + +static void ion_system_heap_free(struct ion_buffer *buffer) +{ + struct ion_system_heap *sys_heap = container_of(buffer->heap, + struct ion_system_heap, + heap); + struct sg_table *table = buffer->sg_table; + struct scatterlist *sg; + int i; + + /* zero the buffer before goto page pool */ + if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) + ion_buffer_zero(buffer); + + for_each_sgtable_sg(table, sg, i) + free_buffer_page(sys_heap, buffer, sg_page(sg)); + sg_free_table(table); + kfree(table); +} + +static int ion_system_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, + int nr_to_scan) +{ + struct ion_page_pool *pool; + struct ion_system_heap *sys_heap; + int nr_total = 0; + int i, nr_freed; + int only_scan = 0; + + sys_heap = container_of(heap, struct ion_system_heap, heap); + + if (!nr_to_scan) + only_scan = 1; + + for (i = 0; i < NUM_ORDERS; i++) { + pool = sys_heap->pools[i]; + + if (only_scan) { + nr_total += ion_page_pool_shrink(pool, + gfp_mask, + nr_to_scan); + + } else { + nr_freed = ion_page_pool_shrink(pool, + gfp_mask, + nr_to_scan); + nr_to_scan -= nr_freed; + nr_total += nr_freed; + if (nr_to_scan <= 0) + break; + } + } + return nr_total; +} + +static long ion_system_get_pool_size(struct ion_heap *heap) +{ + struct ion_system_heap *sys_heap; + long total_pages = 0; + int i; + + sys_heap = container_of(heap, struct ion_system_heap, heap); + for (i = 0; i < NUM_ORDERS; i++) + total_pages += ion_page_pool_nr_pages(sys_heap->pools[i]); + + return total_pages; +} + +static void ion_system_heap_destroy_pools(struct ion_page_pool **pools) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (pools[i]) + ion_page_pool_destroy(pools[i]); +} + +static int ion_system_heap_create_pools(struct ion_page_pool **pools) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + struct ion_page_pool *pool; + gfp_t gfp_flags = low_order_gfp_flags; + + if (orders[i] > 4) + gfp_flags = high_order_gfp_flags; + + pool = ion_page_pool_create(gfp_flags, orders[i]); + if (!pool) + goto err_create_pool; + pools[i] = pool; + } + + return 0; + +err_create_pool: + ion_system_heap_destroy_pools(pools); + return -ENOMEM; +} + +static struct ion_heap_ops system_heap_ops = { + .allocate = ion_system_heap_allocate, + .free = ion_system_heap_free, + .shrink = ion_system_heap_shrink, + .get_pool_size = ion_system_get_pool_size, +}; + +static struct ion_system_heap system_heap = { + .heap = { + .ops = &system_heap_ops, + .type = ION_HEAP_TYPE_SYSTEM, + .flags = ION_HEAP_FLAG_DEFER_FREE, + .name = "ion_system_heap", + } +}; + +static int __init ion_system_heap_init(void) +{ + int ret = ion_system_heap_create_pools(system_heap.pools); + if (ret) + return ret; + + return ion_device_add_heap(&system_heap.heap); +} + +static void __exit ion_system_heap_exit(void) +{ + ion_device_remove_heap(&system_heap.heap); + ion_system_heap_destroy_pools(system_heap.pools); +} + +module_init(ion_system_heap_init); +module_exit(ion_system_heap_exit); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/staging/android/ion/ion.c b/drivers/staging/android/ion/ion.c new file mode 100644 index 000000000000..3d9ec5f1045c --- /dev/null +++ b/drivers/staging/android/ion/ion.c @@ -0,0 +1,518 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator + * + * Copyright (C) 2011 Google, Inc. + * Copyright (c) 2019, The Linux Foundation. All rights reserved. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ion_private.h" + +#define ION_CURRENT_ABI_VERSION 2 + +static struct ion_device *internal_dev; + +/* Entry into ION allocator for rest of the kernel */ +struct dma_buf *ion_alloc(size_t len, unsigned int heap_id_mask, + unsigned int flags) +{ + return ion_dmabuf_alloc(internal_dev, len, heap_id_mask, flags); +} +EXPORT_SYMBOL_GPL(ion_alloc); + +int ion_free(struct ion_buffer *buffer) +{ + return ion_buffer_destroy(internal_dev, buffer); +} +EXPORT_SYMBOL_GPL(ion_free); + +static int ion_alloc_fd(size_t len, unsigned int heap_id_mask, + unsigned int flags) +{ + int fd; + struct dma_buf *dmabuf; + + dmabuf = ion_dmabuf_alloc(internal_dev, len, heap_id_mask, flags); + if (IS_ERR(dmabuf)) + return PTR_ERR(dmabuf); + + fd = dma_buf_fd(dmabuf, O_CLOEXEC); + if (fd < 0) + dma_buf_put(dmabuf); + + return fd; +} + +size_t ion_query_heaps_kernel(struct ion_heap_data *hdata, size_t size) +{ + struct ion_device *dev = internal_dev; + size_t i = 0, num_heaps = 0; + struct ion_heap *heap; + + down_read(&dev->lock); + + // If size is 0, return without updating hdata. + if (size == 0) { + num_heaps = dev->heap_cnt; + goto out; + } + + plist_for_each_entry(heap, &dev->heaps, node) { + strncpy(hdata[i].name, heap->name, MAX_HEAP_NAME); + hdata[i].name[MAX_HEAP_NAME - 1] = '\0'; + hdata[i].type = heap->type; + hdata[i].heap_id = heap->id; + + i++; + if (i >= size) + break; + } + + num_heaps = i; +out: + up_read(&dev->lock); + return num_heaps; +} +EXPORT_SYMBOL_GPL(ion_query_heaps_kernel); + +static int ion_query_heaps(struct ion_heap_query *query) +{ + struct ion_device *dev = internal_dev; + struct ion_heap_data __user *buffer = u64_to_user_ptr(query->heaps); + int ret = -EINVAL, cnt = 0, max_cnt; + struct ion_heap *heap; + struct ion_heap_data hdata; + + memset(&hdata, 0, sizeof(hdata)); + + down_read(&dev->lock); + if (!buffer) { + query->cnt = dev->heap_cnt; + ret = 0; + goto out; + } + + if (query->cnt <= 0) + goto out; + + max_cnt = query->cnt; + + plist_for_each_entry(heap, &dev->heaps, node) { + strncpy(hdata.name, heap->name, MAX_HEAP_NAME); + hdata.name[sizeof(hdata.name) - 1] = '\0'; + hdata.type = heap->type; + hdata.heap_id = heap->id; + + if (copy_to_user(&buffer[cnt], &hdata, sizeof(hdata))) { + ret = -EFAULT; + goto out; + } + + cnt++; + if (cnt >= max_cnt) + break; + } + + query->cnt = cnt; + ret = 0; +out: + up_read(&dev->lock); + return ret; +} + +union ion_ioctl_arg { + struct ion_allocation_data allocation; + struct ion_heap_query query; + u32 ion_abi_version; +}; + +static int validate_ioctl_arg(unsigned int cmd, union ion_ioctl_arg *arg) +{ + switch (cmd) { + case ION_IOC_HEAP_QUERY: + if (arg->query.reserved0 || + arg->query.reserved1 || + arg->query.reserved2) + return -EINVAL; + break; + default: + break; + } + + return 0; +} + +static long ion_ioctl(struct file *filp, unsigned int cmd, unsigned long arg) +{ + int ret = 0; + union ion_ioctl_arg data; + + if (_IOC_SIZE(cmd) > sizeof(data)) + return -EINVAL; + + /* + * The copy_from_user is unconditional here for both read and write + * to do the validate. If there is no write for the ioctl, the + * buffer is cleared + */ + if (copy_from_user(&data, (void __user *)arg, _IOC_SIZE(cmd))) + return -EFAULT; + + ret = validate_ioctl_arg(cmd, &data); + if (ret) { + pr_warn_once("%s: ioctl validate failed\n", __func__); + return ret; + } + + if (!(_IOC_DIR(cmd) & _IOC_WRITE)) + memset(&data, 0, sizeof(data)); + + switch (cmd) { + case ION_IOC_ALLOC: + { + int fd; + + fd = ion_alloc_fd(data.allocation.len, + data.allocation.heap_id_mask, + data.allocation.flags); + if (fd < 0) + return fd; + + data.allocation.fd = fd; + + break; + } + case ION_IOC_HEAP_QUERY: + ret = ion_query_heaps(&data.query); + break; + case ION_IOC_ABI_VERSION: + data.ion_abi_version = ION_CURRENT_ABI_VERSION; + break; + default: + return -ENOTTY; + } + + if (_IOC_DIR(cmd) & _IOC_READ) { + if (copy_to_user((void __user *)arg, &data, _IOC_SIZE(cmd))) + return -EFAULT; + } + return ret; +} + +static const struct file_operations ion_fops = { + .owner = THIS_MODULE, + .unlocked_ioctl = ion_ioctl, + .compat_ioctl = compat_ptr_ioctl, +}; + +static int debug_shrink_set(void *data, u64 val) +{ + struct ion_heap *heap = data; + struct shrink_control sc; + int objs; + + sc.gfp_mask = GFP_HIGHUSER; + sc.nr_to_scan = val; + + if (!val) { + objs = heap->shrinker.count_objects(&heap->shrinker, &sc); + sc.nr_to_scan = objs; + } + + heap->shrinker.scan_objects(&heap->shrinker, &sc); + return 0; +} + +static int debug_shrink_get(void *data, u64 *val) +{ + struct ion_heap *heap = data; + struct shrink_control sc; + int objs; + + sc.gfp_mask = GFP_HIGHUSER; + sc.nr_to_scan = 0; + + objs = heap->shrinker.count_objects(&heap->shrinker, &sc); + *val = objs; + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(debug_shrink_fops, debug_shrink_get, + debug_shrink_set, "%llu\n"); + +static int ion_assign_heap_id(struct ion_heap *heap, struct ion_device *dev) +{ + int id_bit = -EINVAL; + int start_bit = -1, end_bit = -1; + + switch (heap->type) { + case ION_HEAP_TYPE_SYSTEM: + id_bit = __ffs(ION_HEAP_SYSTEM); + break; + case ION_HEAP_TYPE_DMA: + start_bit = __ffs(ION_HEAP_DMA_START); + end_bit = __ffs(ION_HEAP_DMA_END); + break; + case ION_HEAP_TYPE_CUSTOM ... ION_HEAP_TYPE_MAX: + start_bit = __ffs(ION_HEAP_CUSTOM_START); + end_bit = __ffs(ION_HEAP_CUSTOM_END); + break; + default: + return -EINVAL; + } + + /* For carveout, dma & custom heaps, we first let the heaps choose their + * own IDs. This allows the old behaviour of knowing the heap ids + * of these type of heaps in advance in user space. If a heap with + * that ID already exists, it is an error. + * + * If the heap hasn't picked an id by itself, then we assign it + * one. + */ + if (id_bit < 0) { + if (heap->id) { + id_bit = __ffs(heap->id); + if (id_bit < start_bit || id_bit > end_bit) + return -EINVAL; + } else { + id_bit = find_next_zero_bit(dev->heap_ids, end_bit + 1, + start_bit); + if (id_bit > end_bit) + return -ENOSPC; + } + } + + if (test_and_set_bit(id_bit, dev->heap_ids)) + return -EEXIST; + heap->id = id_bit; + dev->heap_cnt++; + + return 0; +} + +int __ion_device_add_heap(struct ion_heap *heap, struct module *owner) +{ + struct ion_device *dev = internal_dev; + int ret; + struct dentry *heap_root; + char debug_name[64]; + + if (!heap || !heap->ops || !heap->ops->allocate || !heap->ops->free) { + pr_err("%s: invalid heap or heap_ops\n", __func__); + ret = -EINVAL; + goto out; + } + + heap->owner = owner; + spin_lock_init(&heap->free_lock); + spin_lock_init(&heap->stat_lock); + heap->free_list_size = 0; + + if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) { + ret = ion_heap_init_deferred_free(heap); + if (ret) + goto out_heap_cleanup; + } + + if ((heap->flags & ION_HEAP_FLAG_DEFER_FREE) || heap->ops->shrink) { + ret = ion_heap_init_shrinker(heap); + if (ret) { + pr_err("%s: Failed to register shrinker\n", __func__); + goto out_heap_cleanup; + } + } + + heap->num_of_buffers = 0; + heap->num_of_alloc_bytes = 0; + heap->alloc_bytes_wm = 0; + + heap_root = debugfs_create_dir(heap->name, dev->debug_root); + debugfs_create_u64("num_of_buffers", + 0444, heap_root, + &heap->num_of_buffers); + debugfs_create_u64("num_of_alloc_bytes", + 0444, + heap_root, + &heap->num_of_alloc_bytes); + debugfs_create_u64("alloc_bytes_wm", + 0444, + heap_root, + &heap->alloc_bytes_wm); + + if (heap->shrinker.count_objects && + heap->shrinker.scan_objects) { + snprintf(debug_name, 64, "%s_shrink", heap->name); + debugfs_create_file(debug_name, + 0644, + heap_root, + heap, + &debug_shrink_fops); + } + + heap->debugfs_dir = heap_root; + down_write(&dev->lock); + ret = ion_assign_heap_id(heap, dev); + if (ret) { + pr_err("%s: Failed to assign heap id for heap type %x\n", + __func__, heap->type); + up_write(&dev->lock); + goto out_debugfs_cleanup; + } + + /* + * use negative heap->id to reverse the priority -- when traversing + * the list later attempt higher id numbers first + */ + plist_node_init(&heap->node, -heap->id); + plist_add(&heap->node, &dev->heaps); + + up_write(&dev->lock); + + return 0; + +out_debugfs_cleanup: + debugfs_remove_recursive(heap->debugfs_dir); +out_heap_cleanup: + ion_heap_cleanup(heap); +out: + return ret; +} +EXPORT_SYMBOL_GPL(__ion_device_add_heap); + +void ion_device_remove_heap(struct ion_heap *heap) +{ + struct ion_device *dev = internal_dev; + + if (!heap) { + pr_err("%s: Invalid argument\n", __func__); + return; + } + + // take semaphore and remove the heap from dev->heap list + down_write(&dev->lock); + /* So no new allocations can happen from this heap */ + plist_del(&heap->node, &dev->heaps); + if (ion_heap_cleanup(heap) != 0) { + pr_warn("%s: failed to cleanup heap (%s)\n", + __func__, heap->name); + } + debugfs_remove_recursive(heap->debugfs_dir); + clear_bit(heap->id, dev->heap_ids); + dev->heap_cnt--; + up_write(&dev->lock); +} +EXPORT_SYMBOL_GPL(ion_device_remove_heap); + +static ssize_t +total_heaps_kb_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + return sprintf(buf, "%llu\n", + div_u64(ion_get_total_heap_bytes(), 1024)); +} + +static ssize_t +total_pools_kb_show(struct kobject *kobj, struct kobj_attribute *attr, + char *buf) +{ + struct ion_device *dev = internal_dev; + struct ion_heap *heap; + u64 total_pages = 0; + + down_read(&dev->lock); + plist_for_each_entry(heap, &dev->heaps, node) + if (heap->ops->get_pool_size) + total_pages += heap->ops->get_pool_size(heap); + up_read(&dev->lock); + + return sprintf(buf, "%llu\n", total_pages * (PAGE_SIZE / 1024)); +} + +static struct kobj_attribute total_heaps_kb_attr = + __ATTR_RO(total_heaps_kb); + +static struct kobj_attribute total_pools_kb_attr = + __ATTR_RO(total_pools_kb); + +static struct attribute *ion_device_attrs[] = { + &total_heaps_kb_attr.attr, + &total_pools_kb_attr.attr, + NULL, +}; + +ATTRIBUTE_GROUPS(ion_device); + +static int ion_init_sysfs(void) +{ + struct kobject *ion_kobj; + int ret; + + ion_kobj = kobject_create_and_add("ion", kernel_kobj); + if (!ion_kobj) + return -ENOMEM; + + ret = sysfs_create_groups(ion_kobj, ion_device_groups); + if (ret) { + kobject_put(ion_kobj); + return ret; + } + + return 0; +} + +static int ion_device_create(void) +{ + struct ion_device *idev; + int ret; + + idev = kzalloc(sizeof(*idev), GFP_KERNEL); + if (!idev) + return -ENOMEM; + + idev->dev.minor = MISC_DYNAMIC_MINOR; + idev->dev.name = "ion"; + idev->dev.fops = &ion_fops; + idev->dev.parent = NULL; + ret = misc_register(&idev->dev); + if (ret) { + pr_err("ion: failed to register misc device.\n"); + goto err_reg; + } + + ret = ion_init_sysfs(); + if (ret) { + pr_err("ion: failed to add sysfs attributes.\n"); + goto err_sysfs; + } + + idev->debug_root = debugfs_create_dir("ion", NULL); + init_rwsem(&idev->lock); + plist_head_init(&idev->heaps); + internal_dev = idev; + return 0; + +err_sysfs: + misc_deregister(&idev->dev); +err_reg: + kfree(idev); + return ret; +} +subsys_initcall(ion_device_create); diff --git a/drivers/staging/android/ion/ion_buffer.c b/drivers/staging/android/ion/ion_buffer.c new file mode 100644 index 000000000000..9baca1a472b6 --- /dev/null +++ b/drivers/staging/android/ion/ion_buffer.c @@ -0,0 +1,278 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator - buffer interface + * + * Copyright (c) 2019, Google, Inc. + */ + +#include +#include +#include +#include +#include + +#define CREATE_TRACE_POINTS +#include "ion_trace.h" +#include "ion_private.h" + +static atomic_long_t total_heap_bytes; + +static void track_buffer_created(struct ion_buffer *buffer) +{ + long total = atomic_long_add_return(buffer->size, &total_heap_bytes); + + trace_ion_stat(buffer->sg_table, buffer->size, total); +} + +static void track_buffer_destroyed(struct ion_buffer *buffer) +{ + long total = atomic_long_sub_return(buffer->size, &total_heap_bytes); + + trace_ion_stat(buffer->sg_table, -buffer->size, total); +} + +/* this function should only be called while dev->lock is held */ +static struct ion_buffer *ion_buffer_create(struct ion_heap *heap, + struct ion_device *dev, + unsigned long len, + unsigned long flags) +{ + struct ion_buffer *buffer; + int ret; + + buffer = kzalloc(sizeof(*buffer), GFP_KERNEL); + if (!buffer) + return ERR_PTR(-ENOMEM); + + buffer->heap = heap; + buffer->flags = flags; + buffer->size = len; + + ret = heap->ops->allocate(heap, buffer, len, flags); + + if (ret) { + if (!(heap->flags & ION_HEAP_FLAG_DEFER_FREE)) + goto err2; + + ion_heap_freelist_drain(heap, 0); + ret = heap->ops->allocate(heap, buffer, len, flags); + if (ret) + goto err2; + } + + if (!buffer->sg_table) { + WARN_ONCE(1, "This heap needs to set the sgtable"); + ret = -EINVAL; + goto err1; + } + + spin_lock(&heap->stat_lock); + heap->num_of_buffers++; + heap->num_of_alloc_bytes += len; + if (heap->num_of_alloc_bytes > heap->alloc_bytes_wm) + heap->alloc_bytes_wm = heap->num_of_alloc_bytes; + if (heap->num_of_buffers == 1) { + /* This module reference lasts as long as at least one + * buffer is allocated from the heap. We are protected + * against ion_device_remove_heap() with dev->lock, so we can + * safely assume the module reference is going to* succeed. + */ + __module_get(heap->owner); + } + spin_unlock(&heap->stat_lock); + + INIT_LIST_HEAD(&buffer->attachments); + mutex_init(&buffer->lock); + track_buffer_created(buffer); + return buffer; + +err1: + heap->ops->free(buffer); +err2: + kfree(buffer); + return ERR_PTR(ret); +} + +static int ion_clear_pages(struct page **pages, int num, pgprot_t pgprot) +{ + void *addr = vmap(pages, num, VM_MAP, pgprot); + + if (!addr) + return -ENOMEM; + memset(addr, 0, PAGE_SIZE * num); + vunmap(addr); + + return 0; +} + +static int ion_sglist_zero(struct scatterlist *sgl, unsigned int nents, + pgprot_t pgprot) +{ + int p = 0; + int ret = 0; + struct sg_page_iter piter; + struct page *pages[32]; + + for_each_sg_page(sgl, &piter, nents, 0) { + pages[p++] = sg_page_iter_page(&piter); + if (p == ARRAY_SIZE(pages)) { + ret = ion_clear_pages(pages, p, pgprot); + if (ret) + return ret; + p = 0; + } + } + if (p) + ret = ion_clear_pages(pages, p, pgprot); + + return ret; +} + +struct ion_buffer *ion_buffer_alloc(struct ion_device *dev, size_t len, + unsigned int heap_id_mask, + unsigned int flags) +{ + struct ion_buffer *buffer = NULL; + struct ion_heap *heap; + + if (!dev || !len) { + return ERR_PTR(-EINVAL); + } + + /* + * traverse the list of heaps available in this system in priority + * order. If the heap type is supported by the client, and matches the + * request of the caller allocate from it. Repeat until allocate has + * succeeded or all heaps have been tried + */ + len = PAGE_ALIGN(len); + if (!len) + return ERR_PTR(-EINVAL); + + down_read(&dev->lock); + plist_for_each_entry(heap, &dev->heaps, node) { + /* if the caller didn't specify this heap id */ + if (!((1 << heap->id) & heap_id_mask)) + continue; + buffer = ion_buffer_create(heap, dev, len, flags); + if (!IS_ERR(buffer)) + break; + } + up_read(&dev->lock); + + if (!buffer) + return ERR_PTR(-ENODEV); + + if (IS_ERR(buffer)) + return ERR_CAST(buffer); + + return buffer; +} + +int ion_buffer_zero(struct ion_buffer *buffer) +{ + struct sg_table *table; + pgprot_t pgprot; + + if (!buffer) + return -EINVAL; + + table = buffer->sg_table; + if (buffer->flags & ION_FLAG_CACHED) + pgprot = PAGE_KERNEL; + else + pgprot = pgprot_writecombine(PAGE_KERNEL); + + return ion_sglist_zero(table->sgl, table->nents, pgprot); +} +EXPORT_SYMBOL_GPL(ion_buffer_zero); + +void ion_buffer_prep_noncached(struct ion_buffer *buffer) +{ + struct scatterlist *sg; + struct sg_table *table; + int i; + + if (WARN_ONCE(!buffer || !buffer->sg_table, + "%s needs a buffer and a sg_table", __func__) || + buffer->flags & ION_FLAG_CACHED) + return; + + table = buffer->sg_table; + + for_each_sg(table->sgl, sg, table->orig_nents, i) + arch_dma_prep_coherent(sg_page(sg), sg->length); +} +EXPORT_SYMBOL_GPL(ion_buffer_prep_noncached); + +void ion_buffer_release(struct ion_buffer *buffer) +{ + if (buffer->kmap_cnt > 0) { + pr_warn_once("%s: buffer still mapped in the kernel\n", + __func__); + ion_heap_unmap_kernel(buffer->heap, buffer); + } + buffer->heap->ops->free(buffer); + spin_lock(&buffer->heap->stat_lock); + buffer->heap->num_of_buffers--; + buffer->heap->num_of_alloc_bytes -= buffer->size; + if (buffer->heap->num_of_buffers == 0) + module_put(buffer->heap->owner); + spin_unlock(&buffer->heap->stat_lock); + /* drop reference to the heap module */ + + kfree(buffer); +} + +int ion_buffer_destroy(struct ion_device *dev, struct ion_buffer *buffer) +{ + struct ion_heap *heap; + + if (!dev || !buffer) { + pr_warn("%s: invalid argument\n", __func__); + return -EINVAL; + } + + heap = buffer->heap; + track_buffer_destroyed(buffer); + + if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) + ion_heap_freelist_add(heap, buffer); + else + ion_buffer_release(buffer); + + return 0; +} + +void *ion_buffer_kmap_get(struct ion_buffer *buffer) +{ + void *vaddr; + + if (buffer->kmap_cnt) { + buffer->kmap_cnt++; + return buffer->vaddr; + } + vaddr = ion_heap_map_kernel(buffer->heap, buffer); + if (WARN_ONCE(!vaddr, + "heap->ops->map_kernel should return ERR_PTR on error")) + return ERR_PTR(-EINVAL); + if (IS_ERR(vaddr)) + return vaddr; + buffer->vaddr = vaddr; + buffer->kmap_cnt++; + return vaddr; +} + +void ion_buffer_kmap_put(struct ion_buffer *buffer) +{ + buffer->kmap_cnt--; + if (!buffer->kmap_cnt) { + ion_heap_unmap_kernel(buffer->heap, buffer); + buffer->vaddr = NULL; + } +} + +u64 ion_get_total_heap_bytes(void) +{ + return atomic_long_read(&total_heap_bytes); +} diff --git a/drivers/staging/android/ion/ion_dma_buf.c b/drivers/staging/android/ion/ion_dma_buf.c new file mode 100644 index 000000000000..47ae54af5ccf --- /dev/null +++ b/drivers/staging/android/ion/ion_dma_buf.c @@ -0,0 +1,359 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator - dmabuf interface + * + * Copyright (c) 2019, Google, Inc. + */ + +#include +#include +#include +#include +#include + +#include "ion_private.h" + +static struct sg_table *dup_sg_table(struct sg_table *table) +{ + struct sg_table *new_table; + int ret, i; + struct scatterlist *sg, *new_sg; + + new_table = kzalloc(sizeof(*new_table), GFP_KERNEL); + if (!new_table) + return ERR_PTR(-ENOMEM); + + ret = sg_alloc_table(new_table, table->nents, GFP_KERNEL); + if (ret) { + kfree(new_table); + return ERR_PTR(-ENOMEM); + } + + new_sg = new_table->sgl; + for_each_sg(table->sgl, sg, table->nents, i) { + memcpy(new_sg, sg, sizeof(*sg)); + new_sg->dma_address = 0; + new_sg = sg_next(new_sg); + } + + return new_table; +} + +static void free_duped_table(struct sg_table *table) +{ + sg_free_table(table); + kfree(table); +} + +static int ion_dma_buf_attach(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct ion_dma_buf_attachment *a; + struct sg_table *table; + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + + if (heap->buf_ops.attach) + return heap->buf_ops.attach(dmabuf, attachment); + + a = kzalloc(sizeof(*a), GFP_KERNEL); + if (!a) + return -ENOMEM; + + table = dup_sg_table(buffer->sg_table); + if (IS_ERR(table)) { + kfree(a); + return -ENOMEM; + } + + a->table = table; + a->dev = attachment->dev; + INIT_LIST_HEAD(&a->list); + a->mapped = false; + + attachment->priv = a; + + mutex_lock(&buffer->lock); + list_add(&a->list, &buffer->attachments); + mutex_unlock(&buffer->lock); + + return 0; +} + +static void ion_dma_buf_detatch(struct dma_buf *dmabuf, + struct dma_buf_attachment *attachment) +{ + struct ion_dma_buf_attachment *a = attachment->priv; + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + + if (heap->buf_ops.detach) + return heap->buf_ops.detach(dmabuf, attachment); + + mutex_lock(&buffer->lock); + list_del(&a->list); + mutex_unlock(&buffer->lock); + free_duped_table(a->table); + + kfree(a); +} + +static struct sg_table *ion_map_dma_buf(struct dma_buf_attachment *attachment, + enum dma_data_direction direction) +{ + struct ion_buffer *buffer = attachment->dmabuf->priv; + struct ion_heap *heap = buffer->heap; + struct ion_dma_buf_attachment *a; + struct sg_table *table; + unsigned long attrs = attachment->dma_map_attrs; + + if (heap->buf_ops.map_dma_buf) + return heap->buf_ops.map_dma_buf(attachment, direction); + + a = attachment->priv; + table = a->table; + + if (!(buffer->flags & ION_FLAG_CACHED)) + attrs |= DMA_ATTR_SKIP_CPU_SYNC; + + if (!dma_map_sg_attrs(attachment->dev, table->sgl, table->nents, + direction, attrs)) + return ERR_PTR(-ENOMEM); + + a->mapped = true; + + return table; +} + +static void ion_unmap_dma_buf(struct dma_buf_attachment *attachment, + struct sg_table *table, + enum dma_data_direction direction) +{ + struct ion_buffer *buffer = attachment->dmabuf->priv; + struct ion_heap *heap = buffer->heap; + struct ion_dma_buf_attachment *a = attachment->priv; + unsigned long attrs = attachment->dma_map_attrs; + + if (heap->buf_ops.unmap_dma_buf) + return heap->buf_ops.unmap_dma_buf(attachment, table, + direction); + + a->mapped = false; + + if (!(buffer->flags & ION_FLAG_CACHED)) + attrs |= DMA_ATTR_SKIP_CPU_SYNC; + + dma_unmap_sg_attrs(attachment->dev, table->sgl, table->nents, + direction, attrs); +} + +static void ion_dma_buf_release(struct dma_buf *dmabuf) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + + if (heap->buf_ops.release) + return heap->buf_ops.release(dmabuf); + + ion_free(buffer); +} + +static int ion_dma_buf_begin_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + struct ion_dma_buf_attachment *a; + + if (heap->buf_ops.begin_cpu_access) + return heap->buf_ops.begin_cpu_access(dmabuf, direction); + + mutex_lock(&buffer->lock); + if (!(buffer->flags & ION_FLAG_CACHED)) + goto unlock; + + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sg_for_cpu(a->dev, a->table->sgl, a->table->nents, + direction); + } + +unlock: + mutex_unlock(&buffer->lock); + return 0; +} + +static int +ion_dma_buf_begin_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction direction, + unsigned int offset, unsigned int len) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + + /* This is done to make sure partial buffer cache flush / invalidate is + * allowed. The implementation may be vendor specific in this case, so + * ion core does not provide a default implementation + */ + if (!heap->buf_ops.begin_cpu_access_partial) + return -EOPNOTSUPP; + + return heap->buf_ops.begin_cpu_access_partial(dmabuf, direction, offset, + len); +} + +static int ion_dma_buf_end_cpu_access(struct dma_buf *dmabuf, + enum dma_data_direction direction) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + struct ion_dma_buf_attachment *a; + + if (heap->buf_ops.end_cpu_access) + return heap->buf_ops.end_cpu_access(dmabuf, direction); + + mutex_lock(&buffer->lock); + if (!(buffer->flags & ION_FLAG_CACHED)) + goto unlock; + + list_for_each_entry(a, &buffer->attachments, list) { + if (!a->mapped) + continue; + dma_sync_sg_for_device(a->dev, a->table->sgl, a->table->nents, + direction); + } +unlock: + mutex_unlock(&buffer->lock); + + return 0; +} + +static int ion_dma_buf_end_cpu_access_partial(struct dma_buf *dmabuf, + enum dma_data_direction direction, + unsigned int offset, + unsigned int len) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + + /* This is done to make sure partial buffer cache flush / invalidate is + * allowed. The implementation may be vendor specific in this case, so + * ion core does not provide a default implementation + */ + if (!heap->buf_ops.end_cpu_access_partial) + return -EOPNOTSUPP; + + return heap->buf_ops.end_cpu_access_partial(dmabuf, direction, offset, + len); +} + +static int ion_dma_buf_mmap(struct dma_buf *dmabuf, struct vm_area_struct *vma) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + int ret; + + /* now map it to userspace */ + if (heap->buf_ops.mmap) { + ret = heap->buf_ops.mmap(dmabuf, vma); + } else { + mutex_lock(&buffer->lock); + if (!(buffer->flags & ION_FLAG_CACHED)) + vma->vm_page_prot = + pgprot_writecombine(vma->vm_page_prot); + + ret = ion_heap_map_user(heap, buffer, vma); + mutex_unlock(&buffer->lock); + } + + if (ret) + pr_err("%s: failure mapping buffer to userspace\n", __func__); + + return ret; +} + +static void *ion_dma_buf_vmap(struct dma_buf *dmabuf) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + void *vaddr; + + if (heap->buf_ops.vmap) + return heap->buf_ops.vmap(dmabuf); + + mutex_lock(&buffer->lock); + vaddr = ion_buffer_kmap_get(buffer); + mutex_unlock(&buffer->lock); + + return vaddr; +} + +static void ion_dma_buf_vunmap(struct dma_buf *dmabuf, void *vaddr) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + + if (heap->buf_ops.vunmap) { + heap->buf_ops.vunmap(dmabuf, vaddr); + return; + } + + mutex_lock(&buffer->lock); + ion_buffer_kmap_put(buffer); + mutex_unlock(&buffer->lock); +} + +static int ion_dma_buf_get_flags(struct dma_buf *dmabuf, unsigned long *flags) +{ + struct ion_buffer *buffer = dmabuf->priv; + struct ion_heap *heap = buffer->heap; + + if (!heap->buf_ops.get_flags) + return -EOPNOTSUPP; + + return heap->buf_ops.get_flags(dmabuf, flags); +} + +static const struct dma_buf_ops dma_buf_ops = { + .attach = ion_dma_buf_attach, + .detach = ion_dma_buf_detatch, + .map_dma_buf = ion_map_dma_buf, + .unmap_dma_buf = ion_unmap_dma_buf, + .release = ion_dma_buf_release, + .begin_cpu_access = ion_dma_buf_begin_cpu_access, + .begin_cpu_access_partial = ion_dma_buf_begin_cpu_access_partial, + .end_cpu_access = ion_dma_buf_end_cpu_access, + .end_cpu_access_partial = ion_dma_buf_end_cpu_access_partial, + .mmap = ion_dma_buf_mmap, + .vmap = ion_dma_buf_vmap, + .vunmap = ion_dma_buf_vunmap, + .get_flags = ion_dma_buf_get_flags, +}; + +struct dma_buf *ion_dmabuf_alloc(struct ion_device *dev, size_t len, + unsigned int heap_id_mask, + unsigned int flags) +{ + struct ion_buffer *buffer; + DEFINE_DMA_BUF_EXPORT_INFO(exp_info); + struct dma_buf *dmabuf; + + pr_debug("%s: len %zu heap_id_mask %u flags %x\n", __func__, + len, heap_id_mask, flags); + + buffer = ion_buffer_alloc(dev, len, heap_id_mask, flags); + if (IS_ERR(buffer)) + return ERR_CAST(buffer); + + exp_info.ops = &dma_buf_ops; + exp_info.size = buffer->size; + exp_info.flags = O_RDWR; + exp_info.priv = buffer; + + dmabuf = dma_buf_export(&exp_info); + if (IS_ERR(dmabuf)) + ion_buffer_destroy(dev, buffer); + + return dmabuf; +} diff --git a/drivers/staging/android/ion/ion_heap.c b/drivers/staging/android/ion/ion_heap.c new file mode 100644 index 000000000000..aa29a4ee48a3 --- /dev/null +++ b/drivers/staging/android/ion/ion_heap.c @@ -0,0 +1,291 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ION Memory Allocator generic heap helpers + * + * Copyright (C) 2011 Google, Inc. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "ion_private.h" + +static unsigned long ion_heap_shrink_count(struct shrinker *shrinker, + struct shrink_control *sc) +{ + struct ion_heap *heap = container_of(shrinker, struct ion_heap, + shrinker); + int total = 0; + + total = ion_heap_freelist_size(heap) / PAGE_SIZE; + + if (heap->ops->shrink) + total += heap->ops->shrink(heap, sc->gfp_mask, 0); + + return total; +} + +static unsigned long ion_heap_shrink_scan(struct shrinker *shrinker, + struct shrink_control *sc) +{ + struct ion_heap *heap = container_of(shrinker, struct ion_heap, + shrinker); + int freed = 0; + int to_scan = sc->nr_to_scan; + + if (to_scan == 0) + return 0; + + /* + * shrink the free list first, no point in zeroing the memory if we're + * just going to reclaim it. Also, skip any possible page pooling. + */ + if (heap->flags & ION_HEAP_FLAG_DEFER_FREE) + freed = ion_heap_freelist_shrink(heap, to_scan * PAGE_SIZE) / + PAGE_SIZE; + + to_scan -= freed; + if (to_scan <= 0) + return freed; + + if (heap->ops->shrink) + freed += heap->ops->shrink(heap, sc->gfp_mask, to_scan); + + return freed; +} + +static size_t _ion_heap_freelist_drain(struct ion_heap *heap, size_t size, + bool skip_pools) +{ + struct ion_buffer *buffer; + size_t total_drained = 0; + + if (ion_heap_freelist_size(heap) == 0) + return 0; + + spin_lock(&heap->free_lock); + if (size == 0) + size = heap->free_list_size; + + while (!list_empty(&heap->free_list)) { + if (total_drained >= size) + break; + buffer = list_first_entry(&heap->free_list, struct ion_buffer, + list); + list_del(&buffer->list); + heap->free_list_size -= buffer->size; + if (skip_pools) + buffer->private_flags |= ION_PRIV_FLAG_SHRINKER_FREE; + total_drained += buffer->size; + spin_unlock(&heap->free_lock); + ion_buffer_release(buffer); + spin_lock(&heap->free_lock); + } + spin_unlock(&heap->free_lock); + + return total_drained; +} + +static int ion_heap_deferred_free(void *data) +{ + struct ion_heap *heap = data; + + while (true) { + struct ion_buffer *buffer; + + wait_event_freezable(heap->waitqueue, + (ion_heap_freelist_size(heap) > 0 || + kthread_should_stop())); + + spin_lock(&heap->free_lock); + if (list_empty(&heap->free_list)) { + spin_unlock(&heap->free_lock); + if (!kthread_should_stop()) + continue; + break; + } + buffer = list_first_entry(&heap->free_list, struct ion_buffer, + list); + list_del(&buffer->list); + heap->free_list_size -= buffer->size; + spin_unlock(&heap->free_lock); + ion_buffer_release(buffer); + } + + return 0; +} + +void *ion_heap_map_kernel(struct ion_heap *heap, + struct ion_buffer *buffer) +{ + struct scatterlist *sg; + int i, j; + void *vaddr; + pgprot_t pgprot; + struct sg_table *table = buffer->sg_table; + int npages = PAGE_ALIGN(buffer->size) / PAGE_SIZE; + struct page **pages = vmalloc(array_size(npages, + sizeof(struct page *))); + struct page **tmp = pages; + + if (!pages) + return ERR_PTR(-ENOMEM); + + if (buffer->flags & ION_FLAG_CACHED) + pgprot = PAGE_KERNEL; + else + pgprot = pgprot_writecombine(PAGE_KERNEL); + + for_each_sg(table->sgl, sg, table->nents, i) { + int npages_this_entry = PAGE_ALIGN(sg->length) / PAGE_SIZE; + struct page *page = sg_page(sg); + + BUG_ON(i >= npages); + for (j = 0; j < npages_this_entry; j++) + *(tmp++) = page++; + } + vaddr = vmap(pages, npages, VM_MAP, pgprot); + vfree(pages); + + if (!vaddr) + return ERR_PTR(-ENOMEM); + + return vaddr; +} +EXPORT_SYMBOL_GPL(ion_heap_map_kernel); + +void ion_heap_unmap_kernel(struct ion_heap *heap, + struct ion_buffer *buffer) +{ + vunmap(buffer->vaddr); +} +EXPORT_SYMBOL_GPL(ion_heap_unmap_kernel); + +int ion_heap_map_user(struct ion_heap *heap, struct ion_buffer *buffer, + struct vm_area_struct *vma) +{ + struct sg_table *table = buffer->sg_table; + unsigned long addr = vma->vm_start; + unsigned long offset = vma->vm_pgoff * PAGE_SIZE; + struct scatterlist *sg; + int i; + int ret; + + for_each_sg(table->sgl, sg, table->nents, i) { + struct page *page = sg_page(sg); + unsigned long remainder = vma->vm_end - addr; + unsigned long len = sg->length; + + if (offset >= sg->length) { + offset -= sg->length; + continue; + } else if (offset) { + page += offset / PAGE_SIZE; + len = sg->length - offset; + offset = 0; + } + len = min(len, remainder); + ret = remap_pfn_range(vma, addr, page_to_pfn(page), len, + vma->vm_page_prot); + if (ret) + return ret; + addr += len; + if (addr >= vma->vm_end) + return 0; + } + + return 0; +} +EXPORT_SYMBOL_GPL(ion_heap_map_user); + +void ion_heap_freelist_add(struct ion_heap *heap, struct ion_buffer *buffer) +{ + spin_lock(&heap->free_lock); + list_add(&buffer->list, &heap->free_list); + heap->free_list_size += buffer->size; + spin_unlock(&heap->free_lock); + wake_up(&heap->waitqueue); +} + +size_t ion_heap_freelist_size(struct ion_heap *heap) +{ + size_t size; + + spin_lock(&heap->free_lock); + size = heap->free_list_size; + spin_unlock(&heap->free_lock); + + return size; +} + +size_t ion_heap_freelist_drain(struct ion_heap *heap, size_t size) +{ + return _ion_heap_freelist_drain(heap, size, false); +} + +size_t ion_heap_freelist_shrink(struct ion_heap *heap, size_t size) +{ + return _ion_heap_freelist_drain(heap, size, true); +} + +int ion_heap_init_deferred_free(struct ion_heap *heap) +{ + INIT_LIST_HEAD(&heap->free_list); + init_waitqueue_head(&heap->waitqueue); + heap->task = kthread_run(ion_heap_deferred_free, heap, + "%s", heap->name); + if (IS_ERR(heap->task)) { + pr_err("%s: creating thread for deferred free failed\n", + __func__); + return PTR_ERR_OR_ZERO(heap->task); + } + sched_set_normal(heap->task, 19); + + return 0; +} + +int ion_heap_init_shrinker(struct ion_heap *heap) +{ + heap->shrinker.count_objects = ion_heap_shrink_count; + heap->shrinker.scan_objects = ion_heap_shrink_scan; + heap->shrinker.seeks = DEFAULT_SEEKS; + heap->shrinker.batch = 0; + + return register_shrinker(&heap->shrinker); +} + +int ion_heap_cleanup(struct ion_heap *heap) +{ + int ret; + + if (heap->flags & ION_HEAP_FLAG_DEFER_FREE && + !IS_ERR_OR_NULL(heap->task)) { + size_t free_list_size = ion_heap_freelist_size(heap); + size_t total_drained = ion_heap_freelist_drain(heap, 0); + + if (total_drained != free_list_size) { + pr_err("%s: %s heap drained %zu bytes, requested %zu\n", + __func__, heap->name, free_list_size, + total_drained); + return -EBUSY; + } + ret = kthread_stop(heap->task); + if (ret < 0) { + pr_err("%s: failed to stop heap free thread\n", + __func__); + return ret; + } + } + + if ((heap->flags & ION_HEAP_FLAG_DEFER_FREE) || heap->ops->shrink) + unregister_shrinker(&heap->shrinker); + + return 0; +} diff --git a/drivers/staging/android/ion/ion_private.h b/drivers/staging/android/ion/ion_private.h new file mode 100644 index 000000000000..db4e90683f4c --- /dev/null +++ b/drivers/staging/android/ion/ion_private.h @@ -0,0 +1,58 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * ION Memory Allocator - Internal header + * + * Copyright (C) 2019 Google, Inc. + */ + +#ifndef _ION_PRIVATE_H +#define _ION_PRIVATE_H + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * struct ion_device - the metadata of the ion device node + * @dev: the actual misc device + * @lock: rwsem protecting the tree of heaps, heap_bitmap and + * clients + * @heap_ids: bitmap of register heap ids + */ +struct ion_device { + struct miscdevice dev; + struct rw_semaphore lock; + DECLARE_BITMAP(heap_ids, ION_NUM_MAX_HEAPS); + struct plist_head heaps; + struct dentry *debug_root; + int heap_cnt; +}; + +/* ion_buffer manipulators */ +extern struct ion_buffer *ion_buffer_alloc(struct ion_device *dev, size_t len, + unsigned int heap_id_mask, + unsigned int flags); +extern void ion_buffer_release(struct ion_buffer *buffer); +extern int ion_buffer_destroy(struct ion_device *dev, + struct ion_buffer *buffer); +extern void *ion_buffer_kmap_get(struct ion_buffer *buffer); +extern void ion_buffer_kmap_put(struct ion_buffer *buffer); + +/* ion dmabuf allocator */ +extern struct dma_buf *ion_dmabuf_alloc(struct ion_device *dev, size_t len, + unsigned int heap_id_mask, + unsigned int flags); +extern int ion_free(struct ion_buffer *buffer); + +/* ion heap helpers */ +extern int ion_heap_cleanup(struct ion_heap *heap); + +u64 ion_get_total_heap_bytes(void); + +#endif /* _ION_PRIVATE_H */ diff --git a/drivers/staging/android/ion/ion_protected_heap.c b/drivers/staging/android/ion/ion_protected_heap.c new file mode 100644 index 000000000000..c7f25373a744 --- /dev/null +++ b/drivers/staging/android/ion/ion_protected_heap.c @@ -0,0 +1,526 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (c) 2021 Rockchip Electronics Co. Ltd. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "ion.h" + +#define ION_HEAP_TYPE_PROTECTED (ION_HEAP_TYPE_CUSTOM + 1) + +#define NUM_ORDERS ARRAY_SIZE(orders) + +static unsigned int orders[] = {8, 4, 0}; + +static struct reserved_mem *protected_reserved_memory; + +#ifdef CONFIG_OF_RESERVED_MEM +static int __init protected_dma_setup(struct reserved_mem *rmem) +{ + protected_reserved_memory = rmem; + + pr_info("ION: created protected pool at %pa, size %ld MiB\n", + &rmem->base, (unsigned long)rmem->size / SZ_1M); + + return 0; +} + +RESERVEDMEM_OF_DECLARE(protected, "protected-dma-pool", protected_dma_setup); +#endif + +static int order_to_index(unsigned int order) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (order == orders[i]) + return i; + + return 0; +} + +static inline unsigned int order_to_size(int order) +{ + return PAGE_SIZE << order; +} + +static void protected_pool_add(struct ion_page_pool *pool, struct page *page) +{ + mutex_lock(&pool->mutex); + if (PageHighMem(page)) { + list_add_tail(&page->lru, &pool->high_items); + pool->high_count++; + } else { + list_add_tail(&page->lru, &pool->low_items); + pool->low_count++; + } + mutex_unlock(&pool->mutex); +} + +static struct page *protected_pool_remove(struct ion_page_pool *pool, + bool high) +{ + struct page *page; + + if (high) { + page = list_first_entry(&pool->high_items, struct page, lru); + pool->high_count--; + } else { + page = list_first_entry(&pool->low_items, struct page, lru); + pool->low_count--; + } + list_del(&page->lru); + + return page; +} + +static struct page *protected_pool_alloc(struct ion_page_pool *pool) +{ + struct page *page = NULL; + + mutex_lock(&pool->mutex); + if (pool->high_count) + page = protected_pool_remove(pool, true); + else if (pool->low_count) + page = protected_pool_remove(pool, false); + mutex_unlock(&pool->mutex); + + return page; +} + +static void protected_pool_free(struct ion_page_pool *pool, + struct page *page) +{ + protected_pool_add(pool, page); +} + +static int protected_pool_total(struct ion_page_pool *pool) +{ + return (pool->low_count + pool->high_count) << pool->order; +} + +static int protected_pool_shrink(struct ion_page_pool *pool, + struct gen_pool *rmem, + int nr_to_scan) +{ + int freed = 0; + + if (nr_to_scan == 0) + return protected_pool_total(pool); + + while (freed < nr_to_scan) { + struct page *page; + + mutex_lock(&pool->mutex); + if (pool->low_count) { + page = protected_pool_remove(pool, false); + } else if (pool->high_count) { + page = protected_pool_remove(pool, true); + } else { + mutex_unlock(&pool->mutex); + break; + } + mutex_unlock(&pool->mutex); + gen_pool_free(rmem, page_to_phys(page), + order_to_size(pool->order)); + freed += (1 << pool->order); + } + + return freed; +} + +static struct ion_page_pool *protected_pool_create(unsigned int order) +{ + struct ion_page_pool *pool = kzalloc(sizeof(*pool), GFP_KERNEL); + + if (!pool) + return NULL; + + pool->high_count = 0; + pool->low_count = 0; + INIT_LIST_HEAD(&pool->low_items); + INIT_LIST_HEAD(&pool->high_items); + pool->order = order; + mutex_init(&pool->mutex); + plist_node_init(&pool->list, order); + + return pool; +} + +static void protected_pool_destroy(struct ion_page_pool *pool) +{ + kfree(pool); +} + +struct ion_protected_heap { + struct ion_heap heap; + struct gen_pool *rmem; + struct ion_page_pool *pools[NUM_ORDERS]; +}; + +struct page_info { + struct page *page; + struct list_head list; + unsigned long order; +}; + +static void free_buffer_page(struct ion_heap *heap, + struct ion_buffer *buffer, + struct page *page, + unsigned long order) +{ + struct ion_page_pool *pool; + struct ion_protected_heap *pheap; + + pheap = container_of(heap, struct ion_protected_heap, heap); + if (buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE) { + gen_pool_free(pheap->rmem, page_to_phys(page), + order_to_size(order)); + return; + } + + pool = pheap->pools[order_to_index(order)]; + protected_pool_free(pool, page); +} + +static struct page *alloc_buffer_page(struct ion_heap *heap, + struct ion_buffer *buffer, + unsigned long order) +{ + struct ion_page_pool *pool; + struct ion_protected_heap *pheap; + struct page *page; + unsigned long paddr; + + pheap = container_of(heap, struct ion_protected_heap, heap); + pool = pheap->pools[order_to_index(order)]; + page = protected_pool_alloc(pool); + if (!page) { + paddr = gen_pool_alloc(pheap->rmem, order_to_size(order)); + if (WARN_ON(!paddr)) + return NULL; + page = phys_to_page(paddr); + } + + return page; +} + +static struct page_info *alloc_largest_available(struct ion_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned int max_order) +{ + struct page_info *info; + struct page *page; + int i; + + info = kmalloc(sizeof(*info), GFP_KERNEL); + if (!info) + return NULL; + + for (i = 0; i < NUM_ORDERS; i++) { + if (size < order_to_size(orders[i])) + continue; + if (max_order < orders[i]) + continue; + + page = alloc_buffer_page(heap, buffer, orders[i]); + if (IS_ERR(page)) + continue; + + info->page = page; + info->order = orders[i]; + INIT_LIST_HEAD(&info->list); + return info; + } + kfree(info); + + return NULL; +} + +static int ion_protected_heap_allocate(struct ion_heap *heap, + struct ion_buffer *buffer, + unsigned long size, + unsigned long flags) +{ + struct list_head pages; + struct list_head lists[8]; + struct page_info *info, *tmp; + struct scatterlist *sg; + struct sg_table *table; + unsigned int block_index[8] = {0}; + unsigned int max_order = orders[0], maximum; + unsigned long size_remaining = PAGE_ALIGN(size); + int i, j; + + if (size / PAGE_SIZE > totalram_pages / 2) + return -ENOMEM; + + INIT_LIST_HEAD(&pages); + for (i = 0; i < 8; i++) + INIT_LIST_HEAD(&lists[i]); + + i = 0; + while (size_remaining > 0) { + info = alloc_largest_available(heap, buffer, size_remaining, + max_order); + if (!info) + goto free_pages; + + size_remaining -= PAGE_SIZE << info->order; + max_order = info->order; + if (max_order) { + list_add_tail(&info->list, &pages); + } else { + dma_addr_t phys = page_to_phys(info->page); + unsigned int bit12_14 = (phys >> 12) & 0x7; + + list_add_tail(&info->list, &lists[bit12_14]); + block_index[bit12_14]++; + } + + i++; + } + + table = kmalloc(sizeof(*table), GFP_KERNEL); + if (!table) + goto free_pages; + + if (sg_alloc_table(table, i, GFP_KERNEL)) + goto free_table; + + maximum = block_index[0]; + for (i = 1; i < 8; i++) + maximum = max(maximum, block_index[i]); + + sg = table->sgl; + list_for_each_entry_safe(info, tmp, &pages, list) { + sg_set_page(sg, info->page, PAGE_SIZE << info->order, 0); + sg = sg_next(sg); + list_del(&info->list); + } + + for (i = 0; i < maximum; i++) { + for (j = 0; j < 8; j++) { + if (list_empty(&lists[j])) + continue; + + info = list_first_entry(&lists[j], struct page_info, + list); + sg_set_page(sg, info->page, PAGE_SIZE, 0); + sg = sg_next(sg); + list_del(&info->list); + } + } + buffer->sg_table = table; + + return 0; +free_table: + kfree(table); +free_pages: + list_for_each_entry_safe(info, tmp, &pages, list) + free_buffer_page(heap, buffer, info->page, info->order); + + for (i = 0; i < 8; i++) { + list_for_each_entry_safe(info, tmp, &lists[i], list) + free_buffer_page(heap, buffer, info->page, info->order); + } + + return -ENOMEM; +} + +static void ion_protected_heap_free(struct ion_buffer *buffer) +{ + struct sg_table *table = buffer->sg_table; + struct scatterlist *sg; + int i; + + /* zero the buffer before goto page pool */ + if (!(buffer->private_flags & ION_PRIV_FLAG_SHRINKER_FREE)) + ion_heap_buffer_zero(buffer); + + for_each_sg(table->sgl, sg, table->nents, i) + free_buffer_page(buffer->heap, buffer, sg_page(sg), + get_order(sg->length)); + sg_free_table(table); + kfree(table); +} + +static int ion_protected_heap_shrink(struct ion_heap *heap, gfp_t gfp_mask, + int nr_to_scan) +{ + struct ion_page_pool *pool; + struct ion_protected_heap *pheap; + int nr_total = 0; + int i, nr_freed; + int only_scan = 0; + + pheap = container_of(heap, struct ion_protected_heap, heap); + if (!nr_to_scan) + only_scan = 1; + + for (i = 0; i < NUM_ORDERS; i++) { + pool = pheap->pools[i]; + + if (only_scan) { + nr_total += protected_pool_shrink(pool, + pheap->rmem, + nr_to_scan); + + } else { + nr_freed = protected_pool_shrink(pool, + pheap->rmem, + nr_to_scan); + nr_to_scan -= nr_freed; + nr_total += nr_freed; + if (nr_to_scan <= 0) + break; + } + } + + return nr_total; +} + +static struct ion_heap_ops protected_heap_ops = { + .allocate = ion_protected_heap_allocate, + .free = ion_protected_heap_free, + .map_kernel = ion_heap_map_kernel, + .unmap_kernel = ion_heap_unmap_kernel, + .map_user = ion_heap_map_user, + .shrink = ion_protected_heap_shrink, +}; + +static int ion_protected_heap_debug_show(struct ion_heap *heap, + struct seq_file *s, + void *unused) +{ + struct ion_protected_heap *pheap; + struct ion_page_pool *pool; + int i; + + pheap = container_of(heap, struct ion_protected_heap, heap); + for (i = 0; i < NUM_ORDERS; i++) { + pool = pheap->pools[i]; + + seq_printf(s, "%d order %u highmem pages %lu total\n", + pool->high_count, pool->order, + (PAGE_SIZE << pool->order) * pool->high_count); + seq_printf(s, "%d order %u lowmem pages %lu total\n", + pool->low_count, pool->order, + (PAGE_SIZE << pool->order) * pool->low_count); + } + + return 0; +} + +static void ion_protected_heap_destroy_pools(struct ion_page_pool **pools) +{ + int i; + + for (i = 0; i < NUM_ORDERS; i++) + if (pools[i]) + protected_pool_destroy(pools[i]); +} + +static int ion_protected_heap_create_pools(struct ion_page_pool **pools) +{ + struct ion_page_pool *pool; + int i; + + for (i = 0; i < NUM_ORDERS; i++) { + pool = protected_pool_create(orders[i]); + if (!pool) + goto err_create_pool; + + pools[i] = pool; + } + + return 0; +err_create_pool: + ion_protected_heap_destroy_pools(pools); + + return -ENOMEM; +} + +static int ion_protected_heap_create_rmem(struct gen_pool **pool) +{ + struct gen_pool *mpool; + struct reserved_mem *rmem = protected_reserved_memory; + int ret; + + if (!rmem) + return -ENOENT; + + mpool = gen_pool_create(PAGE_SHIFT, -1); + if (!mpool) + return -ENOMEM; + + ret = gen_pool_add(mpool, rmem->base, rmem->size, -1); + if (ret) { + gen_pool_destroy(mpool); + return ret; + } + *pool = mpool; + + return 0; +} + +static struct ion_heap *__ion_protected_heap_create(void) +{ + struct ion_protected_heap *heap; + + heap = kzalloc(sizeof(*heap), GFP_KERNEL); + if (!heap) + return ERR_PTR(-ENOMEM); + + heap->heap.ops = &protected_heap_ops; + heap->heap.type = ION_HEAP_TYPE_PROTECTED; + heap->heap.flags = ION_HEAP_FLAG_DEFER_FREE; + heap->heap.name = "ion_protected_heap"; + + if (ion_protected_heap_create_pools(heap->pools)) + goto free_heap; + + if (ion_protected_heap_create_rmem(&heap->rmem)) + goto destroy_pool; + + heap->heap.debug_show = ion_protected_heap_debug_show; + + return &heap->heap; +destroy_pool: + ion_protected_heap_destroy_pools(heap->pools); +free_heap: + kfree(heap); + + return ERR_PTR(-ENOMEM); +} + +int ion_protected_heap_create(void) +{ + struct ion_heap *heap; + + heap = __ion_protected_heap_create(); + if (IS_ERR(heap)) + return PTR_ERR(heap); + + ion_device_add_heap(heap); + + return 0; +} + +#ifndef CONFIG_ION_MODULE +device_initcall(ion_protected_heap_create); +#endif diff --git a/drivers/staging/android/ion/ion_trace.h b/drivers/staging/android/ion/ion_trace.h new file mode 100644 index 000000000000..eacb47d4f135 --- /dev/null +++ b/drivers/staging/android/ion/ion_trace.h @@ -0,0 +1,55 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * drivers/staging/android/ion/ion-trace.h + * + * Copyright (C) 2020 Google, Inc. + */ + +#undef TRACE_SYSTEM +#define TRACE_SYSTEM ion + +#if !defined(_ION_TRACE_H) || defined(TRACE_HEADER_MULTI_READ) +#define _ION_TRACE_H + +#include + +#ifndef __ION_PTR_TO_HASHVAL +static unsigned int __maybe_unused __ion_ptr_to_hash(const void *ptr) +{ + unsigned long hashval; + + if (ptr_to_hashval(ptr, &hashval)) + return 0; + + /* The hashed value is only 32-bit */ + return (unsigned int)hashval; +} + +#define __ION_PTR_TO_HASHVAL +#endif + +TRACE_EVENT(ion_stat, + TP_PROTO(const void *addr, long len, + unsigned long total_allocated), + TP_ARGS(addr, len, total_allocated), + TP_STRUCT__entry(__field(unsigned int, buffer_id) + __field(long, len) + __field(unsigned long, total_allocated) + ), + TP_fast_assign(__entry->buffer_id = __ion_ptr_to_hash(addr); + __entry->len = len; + __entry->total_allocated = total_allocated; + ), + TP_printk("buffer_id=%u len=%ldB total_allocated=%ldB", + __entry->buffer_id, + __entry->len, + __entry->total_allocated) + ); + +#endif /* _ION_TRACE_H */ + +/* This part must be outside protection */ +#undef TRACE_INCLUDE_PATH +#define TRACE_INCLUDE_PATH . +#define TRACE_INCLUDE_FILE ion_trace +#include diff --git a/drivers/staging/android/uapi/ashmem.h b/drivers/staging/android/uapi/ashmem.h new file mode 100644 index 000000000000..2d4673b609d6 --- /dev/null +++ b/drivers/staging/android/uapi/ashmem.h @@ -0,0 +1,54 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Apache-2.0 */ +/* + * drivers/staging/android/uapi/ashmem.h + * + * Copyright 2008 Google Inc. + * Author: Robert Love + */ + +#ifndef _UAPI_LINUX_ASHMEM_H +#define _UAPI_LINUX_ASHMEM_H + +#include +#include + +#define ASHMEM_NAME_LEN 256 + +#define ASHMEM_NAME_DEF "dev/ashmem" + +/* Return values from ASHMEM_PIN: Was the mapping purged while unpinned? */ +#define ASHMEM_NOT_PURGED 0 +#define ASHMEM_WAS_PURGED 1 + +/* Return values from ASHMEM_GET_PIN_STATUS: Is the mapping pinned? */ +#define ASHMEM_IS_UNPINNED 0 +#define ASHMEM_IS_PINNED 1 + +struct ashmem_pin { + __u32 offset; /* offset into region, in bytes, page-aligned */ + __u32 len; /* length forward from offset, in bytes, page-aligned */ +}; + +#define __ASHMEMIOC 0x77 + +#define ASHMEM_SET_NAME _IOW(__ASHMEMIOC, 1, char[ASHMEM_NAME_LEN]) +#define ASHMEM_GET_NAME _IOR(__ASHMEMIOC, 2, char[ASHMEM_NAME_LEN]) +#define ASHMEM_SET_SIZE _IOW(__ASHMEMIOC, 3, size_t) +#define ASHMEM_GET_SIZE _IO(__ASHMEMIOC, 4) +#define ASHMEM_SET_PROT_MASK _IOW(__ASHMEMIOC, 5, unsigned long) +#define ASHMEM_GET_PROT_MASK _IO(__ASHMEMIOC, 6) +#define ASHMEM_PIN _IOW(__ASHMEMIOC, 7, struct ashmem_pin) +#define ASHMEM_UNPIN _IOW(__ASHMEMIOC, 8, struct ashmem_pin) +#define ASHMEM_GET_PIN_STATUS _IO(__ASHMEMIOC, 9) +#define ASHMEM_PURGE_ALL_CACHES _IO(__ASHMEMIOC, 10) + +#ifdef CONFIG_PURGEABLE_ASHMEM +#define PM_SUCCESS 0 +#define PM_FAIL 1 + +#define ASHMEM_SET_PURGEABLE _IO(__ASHMEMIOC, 11) +#define ASHMEM_GET_PURGEABLE _IO(__ASHMEMIOC, 12) +#define PURGEABLE_ASHMEM_IS_PURGED _IO(__ASHMEMIOC, 13) +#define PURGEABLE_ASHMEM_REBUILD_SUCCESS _IO(__ASHMEMIOC, 14) +#endif +#endif /* _UAPI_LINUX_ASHMEM_H */ diff --git a/drivers/staging/android/uapi/ion.h b/drivers/staging/android/uapi/ion.h new file mode 100644 index 000000000000..46c93fcb46d6 --- /dev/null +++ b/drivers/staging/android/uapi/ion.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note */ +/* + * drivers/staging/android/uapi/ion.h + * + * Copyright (C) 2011 Google, Inc. + */ + +#ifndef _UAPI_LINUX_ION_H +#define _UAPI_LINUX_ION_H + +#include +#include + +/** + * enum ion_heap_types - list of all possible types of heaps + * @ION_HEAP_TYPE_SYSTEM: memory allocated via vmalloc + * @ION_HEAP_TYPE_SYSTEM_CONTIG: memory allocated via kmalloc + * @ION_HEAP_TYPE_CARVEOUT: memory allocated from a prereserved + * carveout heap, allocations are physically + * contiguous + * @ION_HEAP_TYPE_DMA: memory allocated via DMA API + * @ION_NUM_HEAPS: helper for iterating over heaps, a bit mask + * is used to identify the heaps, so only 32 + * total heap types are supported + */ +enum ion_heap_type { + ION_HEAP_TYPE_SYSTEM, + ION_HEAP_TYPE_SYSTEM_CONTIG, + ION_HEAP_TYPE_CARVEOUT, + ION_HEAP_TYPE_CHUNK, + ION_HEAP_TYPE_DMA, + ION_HEAP_TYPE_CUSTOM, /* + * must be last so device specific heaps always + * are at the end of this enum + */ +}; + +#define ION_NUM_HEAP_IDS (sizeof(unsigned int) * 8) + +/** + * allocation flags - the lower 16 bits are used by core ion, the upper 16 + * bits are reserved for use by the heaps themselves. + */ + +/* + * mappings of this buffer should be cached, ion will do cache maintenance + * when the buffer is mapped for dma + */ +#define ION_FLAG_CACHED 1 + +/** + * DOC: Ion Userspace API + * + * create a client by opening /dev/ion + * most operations handled via following ioctls + * + */ + +/** + * struct ion_allocation_data - metadata passed from userspace for allocations + * @len: size of the allocation + * @heap_id_mask: mask of heap ids to allocate from + * @flags: flags passed to heap + * @handle: pointer that will be populated with a cookie to use to + * refer to this allocation + * + * Provided by userspace as an argument to the ioctl + */ +struct ion_allocation_data { + __u64 len; + __u32 heap_id_mask; + __u32 flags; + __u32 fd; + __u32 unused; +}; + +#define MAX_HEAP_NAME 32 + +/** + * struct ion_heap_data - data about a heap + * @name - first 32 characters of the heap name + * @type - heap type + * @heap_id - heap id for the heap + */ +struct ion_heap_data { + char name[MAX_HEAP_NAME]; + __u32 type; + __u32 heap_id; + __u32 reserved0; + __u32 reserved1; + __u32 reserved2; +}; + +/** + * struct ion_heap_query - collection of data about all heaps + * @cnt - total number of heaps to be copied + * @heaps - buffer to copy heap data + */ +struct ion_heap_query { + __u32 cnt; /* Total number of heaps to be copied */ + __u32 reserved0; /* align to 64bits */ + __u64 heaps; /* buffer to be populated */ + __u32 reserved1; + __u32 reserved2; +}; + +#define ION_IOC_MAGIC 'I' + +/** + * DOC: ION_IOC_ALLOC - allocate memory + * + * Takes an ion_allocation_data struct and returns it with the handle field + * populated with the opaque handle for the allocation. + */ +#define ION_IOC_ALLOC _IOWR(ION_IOC_MAGIC, 0, \ + struct ion_allocation_data) + +/** + * DOC: ION_IOC_HEAP_QUERY - information about available heaps + * + * Takes an ion_heap_query structure and populates information about + * available Ion heaps. + */ +#define ION_IOC_HEAP_QUERY _IOWR(ION_IOC_MAGIC, 8, \ + struct ion_heap_query) + +#endif /* _UAPI_LINUX_ION_H */ -- Gitee