diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 98bf88a2e7f7bf615c5bb5b81003d0220a5f15ba..83697bf28c02a45e4c5c07c91ab34a935684cb9c 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -6728,8 +6728,9 @@ CONFIG_SMMU_BYPASS_DEV=y # CONFIG_HISI_HBMDEV is not set # CONFIG_HISI_HBMCACHE is not set CONFIG_KUNPENG_HCCS=m -CONFIG_HISI_SOC_CACHE=m +CONFIG_HISI_SOC_CACHE=y CONFIG_HISI_SOC_HHA=m +CONFIG_HISI_SOC_L3C=m # end of Hisilicon SoC drivers # diff --git a/drivers/soc/hisilicon/Kconfig b/drivers/soc/hisilicon/Kconfig index cb5976a5f02b37b9562d1c277eaf900dd2da4cc8..56a6360e0c53c31a3e4c21be6e5324f24081b9f3 100644 --- a/drivers/soc/hisilicon/Kconfig +++ b/drivers/soc/hisilicon/Kconfig @@ -58,7 +58,7 @@ config KUNPENG_HCCS power consumption on Kunpeng SoC. config HISI_SOC_CACHE - tristate "HiSilicon Cache driver for Kunpeng SoC" + bool "HiSilicon Cache driver for Kunpeng SoC" depends on ARCH_HISI help This driver provides the basic utilities for drivers of diff --git a/drivers/soc/hisilicon/hisi_soc_cache_framework.c b/drivers/soc/hisilicon/hisi_soc_cache_framework.c index e74037e1ef9099d9126882e548a1d40351312584..cceeef1b3b12eed0419bb019cd40320576174d52 100644 --- a/drivers/soc/hisilicon/hisi_soc_cache_framework.c +++ b/drivers/soc/hisilicon/hisi_soc_cache_framework.c @@ -26,6 +26,19 @@ #include "hisi_soc_cache_framework.h" +struct hisi_soc_cache_lock_region { + /* physical address of the arena allocated for aligned address */ + unsigned long arena_start; + /* VMA region of locked memory for future release */ + unsigned long vm_start; + unsigned long vm_end; + phys_addr_t addr; + size_t size; + /* Return value of cache lock call */ + int status; + int cpu; +}; + struct hisi_soc_comp_inst { struct list_head node; struct hisi_soc_comp *comp; @@ -40,6 +53,72 @@ struct hisi_soc_comp_list { static struct hisi_soc_comp_list soc_cache_devs[SOC_COMP_TYPE_MAX]; +static int hisi_soc_cache_lock(int cpu, phys_addr_t addr, size_t size) +{ + struct hisi_soc_comp_inst *inst; + struct list_head *head; + int ret = -ENOMEM; + + /* Avoid null pointer when there is no instance onboard. */ + if (soc_cache_devs[HISI_SOC_L3C].inst_num <= 0) + return ret; + + guard(spinlock)(&soc_cache_devs[HISI_SOC_L3C].lock); + + /* Iterate L3C instances to perform operation, break loop once found. */ + head = &soc_cache_devs[HISI_SOC_L3C].node; + list_for_each_entry(inst, head, node) { + if (!cpumask_test_cpu(cpu, &inst->comp->affinity_mask)) + continue; + ret = inst->comp->ops->do_lock(inst->comp, addr, size); + if (ret) + return ret; + } + + list_for_each_entry(inst, head, node) { + if (!cpumask_test_cpu(cpu, &inst->comp->affinity_mask)) + continue; + ret = inst->comp->ops->poll_lock_done(inst->comp, addr, size); + if (ret) + return ret; + } + + return ret; +} + +static int hisi_soc_cache_unlock(int cpu, phys_addr_t addr) +{ + struct hisi_soc_comp_inst *inst; + struct list_head *head; + int ret = 0; + + /* Avoid null pointer when there is no instance onboard. */ + if (soc_cache_devs[HISI_SOC_L3C].inst_num <= 0) + return ret; + + guard(spinlock)(&soc_cache_devs[HISI_SOC_L3C].lock); + + /* Iterate L3C instances to perform operation, break loop once found. */ + head = &soc_cache_devs[HISI_SOC_L3C].node; + list_for_each_entry(inst, head, node) { + if (!cpumask_test_cpu(cpu, &inst->comp->affinity_mask)) + continue; + ret = inst->comp->ops->do_unlock(inst->comp, addr); + if (ret) + return ret; + } + + list_for_each_entry(inst, head, node) { + if (!cpumask_test_cpu(cpu, &inst->comp->affinity_mask)) + continue; + ret = inst->comp->ops->poll_unlock_done(inst->comp, addr); + if (ret) + return ret; + } + + return ret; +} + int hisi_soc_cache_maintain(phys_addr_t addr, size_t size, enum hisi_soc_cache_maint_type mnt_type) { @@ -98,8 +177,15 @@ static const struct mm_walk_ops hisi_soc_cache_maint_walk = { static int hisi_soc_cache_inst_check(const struct hisi_soc_comp *comp, enum hisi_soc_comp_type comp_type) { + struct hisi_soc_comp_ops *ops = comp->ops; + /* Different types of component could have different ops. */ switch (comp_type) { + case HISI_SOC_L3C: + if (!ops->do_lock || !ops->poll_lock_done + || !ops->do_unlock || !ops->poll_unlock_done) + return -EINVAL; + break; case HISI_SOC_HHA: if (!comp->ops->do_maintain || !comp->ops->poll_maintain_done) return -EINVAL; @@ -164,7 +250,7 @@ static void hisi_soc_cache_inst_del(struct hisi_soc_comp *comp, int hisi_soc_comp_inst_add(struct hisi_soc_comp *comp) { - int ret, i = 0; + int ret, i = HISI_SOC_L3C; if (!comp || !comp->ops || comp->comp_type == 0) return -EINVAL; @@ -193,6 +279,187 @@ int hisi_soc_comp_inst_del(struct hisi_soc_comp *comp) } EXPORT_SYMBOL_GPL(hisi_soc_comp_inst_del); +/** + * hisi_soc_cache_aligned_alloc - Allocate memory region to be locked and + * returns address that aligned to the requested + * size. + * @clr: The locked memory region to be allocated for. + * @size: Requested memory size. + * @addr: Pointer of the start physical address of the requested + * memory region. + * + * @return: + * - -ENOMEM: If allocation fails. + * - 0: If allocations succeeds. + * + * Physical address of allocated memory region is requested to be aligned to + * its size. In order to achieve that, add the order of requested memory size + * by 1 to double the size of allocated memory to ensure the existence of size- + * aligned address. After locating the aligned region, release the unused + * pages from both sides to avoid waste. + */ +static int hisi_soc_cache_aligned_alloc(struct hisi_soc_cache_lock_region *clr, + unsigned long size, + unsigned long *addr) +{ + int order = get_order(size) + 1; + unsigned long arena_start; + struct page *pg; + + pg = alloc_contig_pages(1 << order, GFP_KERNEL | __GFP_NOWARN | __GFP_ZERO, + cpu_to_node(smp_processor_id()), NULL); + if (!pg) + return -ENOMEM; + + arena_start = page_to_phys(pg); + + /* + * Align up the address by the requested size if the address is not + * naturally aligned to the size. + */ + *addr = arena_start % size == 0 + ? arena_start + : arena_start / size * size + size; + + clr->arena_start = arena_start; + + return 0; +} + +/** + * hisi_soc_cache_aligned_free - Free the aligned memory region allcated by + * hisi_soc_cache_aligned_alloc(). + * @clr: The allocated locked memory region. + * + * Since unused memory pages are release in hisi_soc_cache_aligned_alloc(), the + * memory region to be freed here may not be power of 2 numbers of pages. + * Thus split the memory by page order and release them accordingly. + */ +static void hisi_soc_cache_aligned_free(struct hisi_soc_cache_lock_region *clr) +{ + int order = get_order(clr->size) + 1; + + free_contig_range(PHYS_PFN(clr->arena_start), 1 << order); +} + +static void hisi_soc_cache_vm_open(struct vm_area_struct *vma) +{ + struct hisi_soc_cache_lock_region *clr = vma->vm_private_data; + + /* + * Only perform cache lock when the vma passed in is created + * in hisi_soc_cache_mmap. + */ + if (clr->vm_start != vma->vm_start || clr->vm_end != vma->vm_end) + return; + + clr->status = hisi_soc_cache_lock(clr->cpu, clr->addr, clr->size); +} + +static void hisi_soc_cache_vm_close(struct vm_area_struct *vma) +{ + struct hisi_soc_cache_lock_region *clr = vma->vm_private_data; + + /* + * Only perform cache unlock when the vma passed in is created + * in hisi_soc_cache_mmap. + */ + if (clr->vm_start != vma->vm_start || clr->vm_end != vma->vm_end) + return; + + hisi_soc_cache_unlock(clr->cpu, clr->addr); + + hisi_soc_cache_aligned_free(clr); + kfree(clr); + vma->vm_private_data = NULL; +} + +/* + * mremap operation is not supported for HiSilicon SoC cache. + */ +static int hisi_soc_cache_vm_mremap(struct vm_area_struct *vma) +{ + struct hisi_soc_cache_lock_region *clr = vma->vm_private_data; + + /* + * vma region size will be changed as requested by mremap despite the + * callback failure in this function. Thus, change the vma region + * stored in clr according to the parameters to verify if the pages + * should be freed when unmapping. + */ + clr->vm_end = clr->vm_start + (vma->vm_end - vma->vm_start); + pr_err("mremap for HiSilicon SoC locked cache is not supported\n"); + + return -EOPNOTSUPP; +} + +static int hisi_soc_cache_may_split(struct vm_area_struct *area, unsigned long addr) +{ + pr_err("HiSilicon SoC locked cache may not be split.\n"); + return -EINVAL; +} + +static const struct vm_operations_struct hisi_soc_cache_vm_ops = { + .open = hisi_soc_cache_vm_open, + .close = hisi_soc_cache_vm_close, + .may_split = hisi_soc_cache_may_split, + .mremap = hisi_soc_cache_vm_mremap, +}; + +static int hisi_soc_cache_mmap(struct file *file, struct vm_area_struct *vma) +{ + unsigned long size = vma->vm_end - vma->vm_start; + struct hisi_soc_cache_lock_region *clr; + unsigned long addr; + int ret; + + clr = kzalloc(sizeof(*clr), GFP_KERNEL); + if (!clr) + return -ENOMEM; + + ret = hisi_soc_cache_aligned_alloc(clr, size, &addr); + if (ret) + goto out_clr; + + if (vma->vm_pgoff > PAGE_SIZE) + return -EINVAL; + + ret = remap_pfn_range(vma, vma->vm_start, + (addr >> PAGE_SHIFT) + vma->vm_pgoff, + size, vma->vm_page_prot); + if (ret) + goto out_page; + + clr->addr = addr; + clr->size = size; + clr->cpu = smp_processor_id(); + vm_flags_set(vma, VM_DONTCOPY | VM_DONTEXPAND); + + /* + * The vma should not be moved throughout its lifetime, store the + * region for verification. + */ + clr->vm_start = vma->vm_start; + clr->vm_end = vma->vm_end; + + vma->vm_private_data = clr; + vma->vm_ops = &hisi_soc_cache_vm_ops; + hisi_soc_cache_vm_ops.open(vma); + + if (clr->status) { + ret = clr->status; + goto out_page; + } + + return 0; + +out_page: + hisi_soc_cache_aligned_free(clr); +out_clr: + kfree(clr); + return ret; +} + static int __hisi_soc_cache_maintain(unsigned long __user vaddr, size_t size, enum hisi_soc_cache_maint_type mnt_type) { @@ -261,6 +528,7 @@ static long hisi_soc_cache_mgmt_ioctl(struct file *file, u32 cmd, unsigned long static const struct file_operations soc_cache_dev_fops = { .owner = THIS_MODULE, .unlocked_ioctl = hisi_soc_cache_mgmt_ioctl, + .mmap = hisi_soc_cache_mmap, }; static struct miscdevice soc_cache_miscdev = { @@ -289,6 +557,7 @@ static void hisi_soc_cache_framework_data_init(void) } static const char *const hisi_soc_cache_item_str[SOC_COMP_TYPE_MAX] = { + "cache", "hha" }; diff --git a/drivers/soc/hisilicon/hisi_soc_cache_framework.h b/drivers/soc/hisilicon/hisi_soc_cache_framework.h index 9b3de4e5016129ce0ab7535153748bf95fe6dbdf..67ee9a33f382f448a38d90d0bf892702d76c4c01 100644 --- a/drivers/soc/hisilicon/hisi_soc_cache_framework.h +++ b/drivers/soc/hisilicon/hisi_soc_cache_framework.h @@ -17,6 +17,7 @@ #include enum hisi_soc_comp_type { + HISI_SOC_L3C, HISI_SOC_HHA, SOC_COMP_TYPE_MAX }; @@ -26,6 +27,12 @@ struct hisi_soc_comp; /** * struct hisi_soc_comp_ops - Callbacks for SoC cache drivers to handle * operation requests. + * + * @lock_enable: lock certain region of L3 cache from being evicted. + * @poll_lock_done: check if the lock operation has succeeded. + * @unlock_enable: unlock the locked region of L3 cache back to normal. + * @poll_unlock_done: check if the unlock operation has succeeded. + operation requests. * @maintain_enable: perform certain cache maintain operation on HHA. * @poll_maintain_done: check if the HHA maintain operation has succeeded. * @@ -41,6 +48,14 @@ struct hisi_soc_comp; * before corresponding done functions being called. */ struct hisi_soc_comp_ops { + int (*do_lock)(struct hisi_soc_comp *comp, + phys_addr_t addr, size_t size); + int (*poll_lock_done)(struct hisi_soc_comp *comp, + phys_addr_t addr, size_t size); + int (*do_unlock)(struct hisi_soc_comp *comp, + phys_addr_t addr); + int (*poll_unlock_done)(struct hisi_soc_comp *comp, + phys_addr_t addr); int (*do_maintain)(struct hisi_soc_comp *comp, phys_addr_t addr, size_t size, enum hisi_soc_cache_maint_type mnt_type); diff --git a/drivers/soc/hisilicon/hisi_soc_l3c.c b/drivers/soc/hisilicon/hisi_soc_l3c.c new file mode 100644 index 0000000000000000000000000000000000000000..a5a8ac6baeb09ada7d153b01d1a1996d54f7a222 --- /dev/null +++ b/drivers/soc/hisilicon/hisi_soc_l3c.c @@ -0,0 +1,610 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Driver for HiSilicon L3 cache. + * + * Copyright (c) 2024 HiSilicon Technologies Co., Ltd. + * Author: Yushan Wang + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "hisi_soc_cache_framework.h" + +#define HISI_L3C_LOCK_CTRL 0x0530 +#define HISI_L3C_LOCK_AREA 0x0534 +#define HISI_L3C_LOCK_START_L 0x0538 +#define HISI_L3C_LOCK_START_H 0x053C + +#define HISI_L3C_DYNAMIC_AUCTRL 0x0404 + +#define HISI_L3C_LOCK_CTRL_POLL_GAP_US 10 + +#define HISI_L3C_MAX_LOCKREGION_SIZE \ + "hisilicon,l3c-max-single-lockregion-size" +#define HISI_L3C_MAX_LOCKREGION_NUM \ + "hisilicon,l3c-lockregion-num" + +/* L3C control register bit definition */ +#define HISI_L3C_LOCK_CTRL_LOCK_EN BIT(0) +#define HISI_L3C_LOCK_CTRL_LOCK_DONE BIT(1) +#define HISI_L3C_LOCK_CTRL_UNLOCK_EN BIT(2) +#define HISI_L3C_LOCK_CTRL_UNLOCK_DONE BIT(3) + +#define HISI_L3C_LOCK_MIN_SIZE (1 * 1024 * 1024) +#define HISI_L3_CACHE_LINE_SIZE 64 + +/* Allow maximum 70% of cache locked. */ +#define HISI_L3C_MAX_LOCK_SIZE(size) ((size) / 10 * 7) + +#define l3c_lock_reg_offset(reg, set) ((reg) + 16 * (set)) + +#define l3c_lock_ctrl_mask(lock_ctrl, mask) ((lock_ctrl) & (mask)) + +#define to_hisi_l3c(p) container_of((p), struct hisi_soc_l3c, comp) + +static int hisi_l3c_cpuhp_state; + +struct hisi_soc_l3c { + struct hisi_soc_comp comp; + cpumask_t associated_cpus; + + /* Stores the first address locked by each register sets. */ + struct xarray lock_sets; + /* Stores if a set of lock control register has been used. */ + u32 reg_used_map; + /* Locks reg_used_map and lock_sets to forbid overlapping access. */ + spinlock_t reg_lock; + + /* Locked memory size range. */ + size_t max_lock_size; + size_t min_lock_size; + size_t locked_size; + + /* Maximum number of locked memory size. */ + int max_lock_num; + + struct hlist_node node; + void __iomem *base; + + /* ID of Super CPU cluster on where the L3 cache locates. */ + int sccl_id; + /* ID of CPU cluster where L3 cache is located. */ + int ccl_id; +}; + +struct hisi_soc_l3c_lock_region { + phys_addr_t addr; + size_t size; +}; + +/** + * hisi_soc_l3c_alloc_lock_reg_set - Allocate an available control register set + * of L3 cache for lock & unlock operations. + * @soc_l3c: The L3C instance on which the register set will be allocated. + * @addr: The address to be locked. + * @size: The size to be locked. + * + * @return: + * - -EBUSY: If there is no available register sets. + * - -ENOMEM: If there is no available memory for lock region struct. + * - -EINVAL: If there is no available cache size for lock. + * - 0: If allocation succeeds. + * + * Maintains the resource of control registers of L3 cache. On allocation, + * the index of a spare set of registers is returned, then the address is + * stored inside for future match of unlock operation. + */ +static int hisi_soc_l3c_alloc_lock_reg_set(struct hisi_soc_l3c *soc_l3c, + phys_addr_t addr, size_t size) +{ + struct hisi_soc_l3c_lock_region *lr; + unsigned long idx; + void *entry; + + if (size > soc_l3c->max_lock_size - soc_l3c->locked_size) + return -EINVAL; + + for (idx = 0; idx < soc_l3c->max_lock_num; ++idx) { + entry = xa_load(&soc_l3c->lock_sets, idx); + if (!entry) + break; + } + + if (idx >= soc_l3c->max_lock_num) + return -EBUSY; + + lr = kzalloc(sizeof(struct hisi_soc_l3c_lock_region), GFP_KERNEL); + if (!lr) + return -ENOMEM; + + lr->addr = addr; + lr->size = size; + + entry = xa_store(&soc_l3c->lock_sets, idx, lr, GFP_KERNEL); + if (xa_is_err(entry)) { + kfree(lr); + return xa_err(entry); + } + + soc_l3c->locked_size += size; + + return idx; +} + +/** + * hisi_soc_l3c_get_locked_reg_set - Get the index of an allocated register set + * by locked address. + * @soc_l3c: The L3C instance on which the register set is allocated. + * @addr: The locked address. + * + * @return: + * - >= 0: index of register set which controls locked memory region of @addr. + * - -EINVAL: If @addr is not locked in this cache. + */ +static int hisi_soc_l3c_get_locked_reg_set(struct hisi_soc_l3c *soc_l3c, + phys_addr_t addr) +{ + struct hisi_soc_l3c_lock_region *entry; + unsigned long idx; + + xa_for_each_range(&soc_l3c->lock_sets, idx, entry, 0, + soc_l3c->max_lock_num) { + if (entry->addr == addr) + return idx; + } + return -EINVAL; +} + +/** + * hisi_soc_l3c_free_lock_reg_set - Free an allocated register set by locked + * address. + * + * @soc_l3c: The L3C instance on which the register set is allocated. + * @regset: ID of Register set to be freed. + */ +static void hisi_soc_l3c_free_lock_reg_set(struct hisi_soc_l3c *soc_l3c, + int regset) +{ + struct hisi_soc_l3c_lock_region *entry; + + if (regset < 0) + return; + + entry = xa_erase(&soc_l3c->lock_sets, regset); + if (!entry) + return; + + soc_l3c->locked_size -= entry->size; + kfree(entry); +} + +static int hisi_l3c_lock_ctrl_wait_finished(struct hisi_soc_l3c *soc_l3c, + int regset, u32 mask) +{ + u32 reg_used_map = soc_l3c->reg_used_map; + void *base = soc_l3c->base; + u32 val; + + /* + * Each HiSilicon L3 cache instance will have lock/unlock done bit set + * to 0 when first put to use even if the device is available. + * A reg_used_map is proposed to record if an instance has been called + * to lock down, then we can determine if it is available by + * reading lock/unlock done bit. + */ + if (!(reg_used_map & BIT(regset))) { + reg_used_map |= BIT(regset); + return 1; + } + + return !readl_poll_timeout_atomic( + base + l3c_lock_reg_offset(HISI_L3C_LOCK_CTRL, regset), + val, l3c_lock_ctrl_mask(val, mask), + HISI_L3C_LOCK_CTRL_POLL_GAP_US, + jiffies_to_usecs(HZ)); +} + +static int hisi_soc_l3c_do_lock(struct hisi_soc_comp *l3c_comp, + phys_addr_t addr, size_t size) +{ + struct hisi_soc_l3c *soc_l3c = to_hisi_l3c(l3c_comp); + void *base = soc_l3c->base; + int regset; + u32 ctrl; + + if (soc_l3c->max_lock_num == 1 && addr % size != 0) + return -EINVAL; + + if (size < soc_l3c->min_lock_size) + return -EINVAL; + + guard(spinlock)(&soc_l3c->reg_lock); + + regset = hisi_soc_l3c_alloc_lock_reg_set(soc_l3c, addr, size); + if (regset < 0) + return -EBUSY; + + if (!hisi_l3c_lock_ctrl_wait_finished(soc_l3c, regset, + HISI_L3C_LOCK_CTRL_LOCK_DONE)) { + hisi_soc_l3c_free_lock_reg_set(soc_l3c, regset); + return -EBUSY; + } + + writel(lower_32_bits(addr), + base + l3c_lock_reg_offset(HISI_L3C_LOCK_START_L, regset)); + writel(upper_32_bits(addr), + base + l3c_lock_reg_offset(HISI_L3C_LOCK_START_H, regset)); + writel(size, base + l3c_lock_reg_offset(HISI_L3C_LOCK_AREA, regset)); + + ctrl = readl(base + HISI_L3C_DYNAMIC_AUCTRL); + ctrl |= BIT(regset); + writel(ctrl, base + HISI_L3C_DYNAMIC_AUCTRL); + + ctrl = readl(base + l3c_lock_reg_offset(HISI_L3C_LOCK_CTRL, regset)); + ctrl = (ctrl | HISI_L3C_LOCK_CTRL_LOCK_EN) & + ~HISI_L3C_LOCK_CTRL_UNLOCK_EN; + writel(ctrl, base + l3c_lock_reg_offset(HISI_L3C_LOCK_CTRL, regset)); + + return 0; +} + +static int hisi_soc_l3c_poll_lock_done(struct hisi_soc_comp *l3c_comp, + phys_addr_t addr, size_t size) +{ + struct hisi_soc_l3c *soc_l3c = to_hisi_l3c(l3c_comp); + int regset; + + guard(spinlock)(&soc_l3c->reg_lock); + + regset = hisi_soc_l3c_get_locked_reg_set(soc_l3c, addr); + if (regset < 0) + return -EINVAL; + + if (!hisi_l3c_lock_ctrl_wait_finished(soc_l3c, regset, + HISI_L3C_LOCK_CTRL_LOCK_DONE)) + return -ETIMEDOUT; + + return 0; +} + +static int hisi_soc_l3c_do_unlock(struct hisi_soc_comp *l3c_comp, + phys_addr_t addr) +{ + struct hisi_soc_l3c *soc_l3c = to_hisi_l3c(l3c_comp); + void *base = soc_l3c->base; + int regset; + u32 ctrl; + + guard(spinlock)(&soc_l3c->reg_lock); + + regset = hisi_soc_l3c_get_locked_reg_set(soc_l3c, addr); + if (regset < 0) + return -EINVAL; + + if (!hisi_l3c_lock_ctrl_wait_finished(soc_l3c, regset, + HISI_L3C_LOCK_CTRL_UNLOCK_DONE)) + return -EBUSY; + + ctrl = readl(base + HISI_L3C_DYNAMIC_AUCTRL); + ctrl &= ~BIT(regset); + writel(ctrl, base + HISI_L3C_DYNAMIC_AUCTRL); + + ctrl = readl(base + l3c_lock_reg_offset(HISI_L3C_LOCK_CTRL, regset)); + ctrl = (ctrl | HISI_L3C_LOCK_CTRL_UNLOCK_EN) & + ~HISI_L3C_LOCK_CTRL_LOCK_EN; + writel(ctrl, base + l3c_lock_reg_offset(HISI_L3C_LOCK_CTRL, regset)); + + return 0; +} + +static int hisi_soc_l3c_poll_unlock_done(struct hisi_soc_comp *l3c_comp, + phys_addr_t addr) +{ + struct hisi_soc_l3c *soc_l3c = to_hisi_l3c(l3c_comp); + int regset; + + guard(spinlock)(&soc_l3c->reg_lock); + + regset = hisi_soc_l3c_get_locked_reg_set(soc_l3c, addr); + if (regset < 0) + return -EINVAL; + + if (!hisi_l3c_lock_ctrl_wait_finished(soc_l3c, regset, + HISI_L3C_LOCK_CTRL_UNLOCK_DONE)) + return -ETIMEDOUT; + + hisi_soc_l3c_free_lock_reg_set(soc_l3c, regset); + + return 0; +} + +/** + * hisi_soc_l3c_remove_locks - Remove all cache locks when the driver exits. + * + * @soc_l3c: The L3C instance on which the cache locks should be removed. + */ +static void hisi_soc_l3c_remove_locks(struct hisi_soc_l3c *soc_l3c) +{ + + void *base = soc_l3c->base; + unsigned long regset; + int timeout; + void *entry; + u32 ctrl; + + guard(spinlock)(&soc_l3c->reg_lock); + + xa_for_each(&soc_l3c->lock_sets, regset, entry) { + timeout = hisi_l3c_lock_ctrl_wait_finished(soc_l3c, regset, + HISI_L3C_LOCK_CTRL_UNLOCK_DONE); + + ctrl = readl(base + l3c_lock_reg_offset(HISI_L3C_LOCK_CTRL, + regset)); + ctrl = (ctrl | HISI_L3C_LOCK_CTRL_UNLOCK_EN) & + ~HISI_L3C_LOCK_CTRL_LOCK_EN; + writel(ctrl, base + l3c_lock_reg_offset(HISI_L3C_LOCK_CTRL, + regset)); + + timeout = hisi_l3c_lock_ctrl_wait_finished(soc_l3c, regset, + HISI_L3C_LOCK_CTRL_UNLOCK_DONE); + + /* + * If cache lock remove fails, inform user since the removal of + * driver cannot fail. + */ + if (timeout) + pr_err("failed to remove %lu-th cache lock.\n", regset); + } +} + +static int hisi_soc_l3c_init_lock_capacity(struct hisi_soc_l3c *soc_l3c, + struct device *dev) +{ + int ret; + u32 val; + + ret = device_property_read_u32(dev, HISI_L3C_MAX_LOCKREGION_NUM, &val); + if (ret || val <= 0) + return -EINVAL; + + soc_l3c->max_lock_num = val; + + ret = device_property_read_u32(dev, HISI_L3C_MAX_LOCKREGION_SIZE, &val); + if (ret || val <= 0) + return -EINVAL; + + soc_l3c->max_lock_size = HISI_L3C_MAX_LOCK_SIZE(val); + + soc_l3c->min_lock_size = soc_l3c->max_lock_num == 1 + ? HISI_L3C_LOCK_MIN_SIZE + : HISI_L3_CACHE_LINE_SIZE; + + return 0; +} + +static int hisi_soc_l3c_init_topology(struct hisi_soc_l3c *soc_l3c, + struct device *dev) +{ + soc_l3c->sccl_id = -1; + soc_l3c->ccl_id = -1; + + if (device_property_read_u32(dev, "hisilicon,scl-id", &soc_l3c->sccl_id) + || soc_l3c->sccl_id < 0) + return -EINVAL; + + if (device_property_read_u32(dev, "hisilicon,ccl-id", &soc_l3c->ccl_id) + || soc_l3c->ccl_id < 0) + return -EINVAL; + + return 0; +} + +static void hisi_init_associated_cpus(struct hisi_soc_l3c *soc_l3c) +{ + if (!cpumask_empty(&soc_l3c->associated_cpus)) + return; + cpumask_clear(&soc_l3c->associated_cpus); + cpumask_copy(&soc_l3c->comp.affinity_mask, &soc_l3c->associated_cpus); +} + +static struct hisi_soc_comp_ops hisi_soc_l3c_comp_ops = { + .do_lock = hisi_soc_l3c_do_lock, + .poll_lock_done = hisi_soc_l3c_poll_lock_done, + .do_unlock = hisi_soc_l3c_do_unlock, + .poll_unlock_done = hisi_soc_l3c_poll_unlock_done, +}; + +static struct hisi_soc_comp hisi_soc_l3c_comp = { + .ops = &hisi_soc_l3c_comp_ops, + .comp_type = BIT(HISI_SOC_L3C), +}; + +static int hisi_soc_l3c_probe(struct platform_device *pdev) +{ + struct hisi_soc_l3c *soc_l3c; + struct resource *mem; + int ret = 0; + + soc_l3c = devm_kzalloc(&pdev->dev, sizeof(*soc_l3c), GFP_KERNEL); + if (!soc_l3c) + return -ENOMEM; + + platform_set_drvdata(pdev, soc_l3c); + + mem = platform_get_resource(pdev, IORESOURCE_MEM, 0); + if (!mem) + return -ENODEV; + + /* + * L3C cache driver share the same register region with L3C uncore PMU + * driver in hardware's perspective, none of them should reserve the + * resource to itself only. Here exclusive access verification is + * avoided by calling devm_ioremap instead of devm_ioremap_resource to + * allow both drivers to exist at the same time. + */ + soc_l3c->base = devm_ioremap(&pdev->dev, mem->start, + resource_size(mem)); + if (IS_ERR_OR_NULL(soc_l3c->base)) + return PTR_ERR(soc_l3c->base); + + soc_l3c->comp = hisi_soc_l3c_comp; + soc_l3c->locked_size = 0; + spin_lock_init(&soc_l3c->reg_lock); + xa_init(&soc_l3c->lock_sets); + + ret = hisi_soc_l3c_init_lock_capacity(soc_l3c, &pdev->dev); + if (ret) + goto err_xa; + + hisi_init_associated_cpus(soc_l3c); + + ret = hisi_soc_l3c_init_topology(soc_l3c, &pdev->dev); + if (ret) + goto err_xa; + + ret = cpuhp_state_add_instance(hisi_l3c_cpuhp_state, &soc_l3c->node); + if (ret) + goto err_xa; + + ret = hisi_soc_comp_inst_add(&soc_l3c->comp); + if (ret) + goto err_hotplug; + + return ret; + +err_hotplug: + cpuhp_state_remove_instance_nocalls(hisi_l3c_cpuhp_state, + &soc_l3c->node); + +err_xa: + xa_destroy(&soc_l3c->lock_sets); + return ret; +} + +static int hisi_soc_l3c_remove(struct platform_device *pdev) +{ + struct hisi_soc_l3c *soc_l3c = platform_get_drvdata(pdev); + unsigned long idx; + struct hisi_soc_l3c_lock_region *entry; + + hisi_soc_l3c_remove_locks(soc_l3c); + + hisi_soc_comp_inst_del(&soc_l3c->comp); + + cpuhp_state_remove_instance_nocalls(hisi_l3c_cpuhp_state, + &soc_l3c->node); + + xa_for_each(&soc_l3c->lock_sets, idx, entry) { + entry = xa_erase(&soc_l3c->lock_sets, idx); + kfree(entry); + } + + xa_destroy(&soc_l3c->lock_sets); + + return 0; +} + +static void hisi_read_sccl_and_ccl_id(int *scclp, int *cclp) +{ + u64 mpidr = read_cpuid_mpidr(); + int aff3 = MPIDR_AFFINITY_LEVEL(mpidr, 3); + int aff2 = MPIDR_AFFINITY_LEVEL(mpidr, 2); + int aff1 = MPIDR_AFFINITY_LEVEL(mpidr, 1); + int sccl, ccl; + + if (mpidr & MPIDR_MT_BITMASK) { + sccl = aff3; + ccl = aff2; + } else { + sccl = aff2; + ccl = aff1; + } + + *scclp = sccl; + *cclp = ccl; +} + +static bool hisi_soc_l3c_is_associated(struct hisi_soc_l3c *soc_l3c) +{ + int sccl_id, ccl_id; + + hisi_read_sccl_and_ccl_id(&sccl_id, &ccl_id); + return sccl_id == soc_l3c->sccl_id && ccl_id == soc_l3c->ccl_id; +} + +static int hisi_soc_l3c_online_cpu(unsigned int cpu, struct hlist_node *node) +{ + struct hisi_soc_l3c *soc_l3c = + hlist_entry_safe(node, struct hisi_soc_l3c, node); + + if (!cpumask_test_cpu(cpu, &soc_l3c->associated_cpus)) { + if (!(hisi_soc_l3c_is_associated(soc_l3c))) + return 0; + + cpumask_set_cpu(cpu, &soc_l3c->associated_cpus); + cpumask_copy(&soc_l3c->comp.affinity_mask, + &soc_l3c->associated_cpus); + } + return 0; +} + +static const struct acpi_device_id hisi_l3c_acpi_match[] = { + { "HISI0501", }, + { } +}; +MODULE_DEVICE_TABLE(acpi, hisi_l3c_acpi_match); + +static struct platform_driver hisi_soc_l3c_driver = { + .driver = { + .name = "hisi_soc_l3c", + .acpi_match_table = hisi_l3c_acpi_match, + }, + .probe = hisi_soc_l3c_probe, + .remove = hisi_soc_l3c_remove, +}; + +static int __init hisi_soc_l3c_init(void) +{ + int ret; + + ret = cpuhp_setup_state_multi(CPUHP_AP_ONLINE_DYN, "hisi_soc_l3c", + hisi_soc_l3c_online_cpu, NULL); + if (ret < 0) + return ret; + hisi_l3c_cpuhp_state = ret; + + ret = platform_driver_register(&hisi_soc_l3c_driver); + if (ret) + cpuhp_remove_multi_state(CPUHP_AP_ONLINE_DYN); + + return ret; +} +module_init(hisi_soc_l3c_init); + +static void __exit hisi_soc_l3c_exit(void) +{ + platform_driver_unregister(&hisi_soc_l3c_driver); + cpuhp_remove_multi_state(CPUHP_AP_ONLINE_DYN); +} +module_exit(hisi_soc_l3c_exit); + +MODULE_DESCRIPTION("Driver supporting cache lockdown for Hisilicon L3 cache"); +MODULE_AUTHOR("Yushan Wang "); +MODULE_LICENSE("GPL");